1 //
2 // Copyright 2017 gRPC authors.
3 //
4 // Licensed under the Apache License, Version 2.0 (the "License");
5 // you may not use this file except in compliance with the License.
6 // You may obtain a copy of the License at
7 //
8 // http://www.apache.org/licenses/LICENSE-2.0
9 //
10 // Unless required by applicable law or agreed to in writing, software
11 // distributed under the License is distributed on an "AS IS" BASIS,
12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 // See the License for the specific language governing permissions and
14 // limitations under the License.
15 //
16
17 // TODO(roth): Split this file up into a common test framework and a set
18 // of test files that use that framework. Need to figure out the best
19 // way to split up the tests. One option would be to split it up by xDS
20 // resource type; another approach would be to have all of the "core"
21 // xDS functionality in one file and then move specific features to
22 // their own files (e.g., mTLS security, fault injection, circuit
23 // breaking, etc).
24
25 #include <deque>
26 #include <memory>
27 #include <mutex>
28 #include <numeric>
29 #include <set>
30 #include <sstream>
31 #include <string>
32 #include <thread>
33 #include <vector>
34
35 #include <gmock/gmock.h>
36 #include <gtest/gtest.h>
37
38 #include "absl/functional/bind_front.h"
39 #include "absl/memory/memory.h"
40 #include "absl/strings/match.h"
41 #include "absl/strings/str_cat.h"
42 #include "absl/strings/str_format.h"
43 #include "absl/strings/str_join.h"
44 #include "absl/types/optional.h"
45
46 #include <grpc/grpc.h>
47 #include <grpc/grpc_security.h>
48 #include <grpc/support/alloc.h>
49 #include <grpc/support/log.h>
50 #include <grpc/support/time.h>
51 #include <grpcpp/channel.h>
52 #include <grpcpp/client_context.h>
53 #include <grpcpp/create_channel.h>
54 #include <grpcpp/security/tls_certificate_provider.h>
55 #include <grpcpp/server.h>
56 #include <grpcpp/server_builder.h>
57 #include <grpcpp/xds_server_builder.h>
58
59 #include "src/core/ext/filters/client_channel/backup_poller.h"
60 #include "src/core/ext/filters/client_channel/lb_policy/xds/xds_channel_args.h"
61 #include "src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h"
62 #include "src/core/ext/filters/client_channel/server_address.h"
63 #include "src/core/ext/xds/certificate_provider_registry.h"
64 #include "src/core/ext/xds/xds_api.h"
65 #include "src/core/ext/xds/xds_channel_args.h"
66 #include "src/core/ext/xds/xds_client.h"
67 #include "src/core/lib/address_utils/parse_address.h"
68 #include "src/core/lib/channel/channel_args.h"
69 #include "src/core/lib/gpr/env.h"
70 #include "src/core/lib/gpr/string.h"
71 #include "src/core/lib/gpr/time_precise.h"
72 #include "src/core/lib/gpr/tmpfile.h"
73 #include "src/core/lib/gprpp/ref_counted_ptr.h"
74 #include "src/core/lib/gprpp/sync.h"
75 #include "src/core/lib/gprpp/time_util.h"
76 #include "src/core/lib/iomgr/load_file.h"
77 #include "src/core/lib/iomgr/sockaddr.h"
78 #include "src/core/lib/security/credentials/fake/fake_credentials.h"
79 #include "src/cpp/client/secure_credentials.h"
80 #include "src/cpp/server/secure_server_credentials.h"
81 #include "src/proto/grpc/testing/echo.grpc.pb.h"
82 #include "src/proto/grpc/testing/xds/ads_for_test.grpc.pb.h"
83 #include "src/proto/grpc/testing/xds/cds_for_test.grpc.pb.h"
84 #include "src/proto/grpc/testing/xds/eds_for_test.grpc.pb.h"
85 #include "src/proto/grpc/testing/xds/lds_rds_for_test.grpc.pb.h"
86 #include "src/proto/grpc/testing/xds/lrs_for_test.grpc.pb.h"
87 #include "src/proto/grpc/testing/xds/v3/ads.grpc.pb.h"
88 #include "src/proto/grpc/testing/xds/v3/aggregate_cluster.grpc.pb.h"
89 #include "src/proto/grpc/testing/xds/v3/cluster.grpc.pb.h"
90 #include "src/proto/grpc/testing/xds/v3/discovery.grpc.pb.h"
91 #include "src/proto/grpc/testing/xds/v3/endpoint.grpc.pb.h"
92 #include "src/proto/grpc/testing/xds/v3/fault.grpc.pb.h"
93 #include "src/proto/grpc/testing/xds/v3/http_connection_manager.grpc.pb.h"
94 #include "src/proto/grpc/testing/xds/v3/listener.grpc.pb.h"
95 #include "src/proto/grpc/testing/xds/v3/lrs.grpc.pb.h"
96 #include "src/proto/grpc/testing/xds/v3/route.grpc.pb.h"
97 #include "src/proto/grpc/testing/xds/v3/router.grpc.pb.h"
98 #include "src/proto/grpc/testing/xds/v3/tls.grpc.pb.h"
99 #include "test/core/util/port.h"
100 #include "test/core/util/resolve_localhost_ip46.h"
101 #include "test/core/util/test_config.h"
102 #include "test/cpp/end2end/counted_service.h"
103 #include "test/cpp/end2end/test_service_impl.h"
104 #include "test/cpp/end2end/xds/xds_server.h"
105 #include "test/cpp/util/test_config.h"
106
107 #ifndef DISABLED_XDS_PROTO_IN_CC
108 #include "src/cpp/server/csds/csds.h"
109 #include "src/proto/grpc/testing/xds/v3/csds.grpc.pb.h"
110 #endif // DISABLED_XDS_PROTO_IN_CC
111
112 namespace grpc {
113 namespace testing {
114 namespace {
115
116 using std::chrono::system_clock;
117
118 #ifndef DISABLED_XDS_PROTO_IN_CC
119 using ::envoy::admin::v3::ClientResourceStatus;
120 #endif // DISABLED_XDS_PROTO_IN_CC
121 using ::envoy::config::cluster::v3::CircuitBreakers;
122 using ::envoy::config::cluster::v3::Cluster;
123 using ::envoy::config::cluster::v3::CustomClusterType;
124 using ::envoy::config::cluster::v3::RoutingPriority;
125 using ::envoy::config::endpoint::v3::ClusterLoadAssignment;
126 using ::envoy::config::endpoint::v3::HealthStatus;
127 using ::envoy::config::listener::v3::FilterChainMatch;
128 using ::envoy::config::listener::v3::Listener;
129 using ::envoy::config::route::v3::RouteConfiguration;
130 using ::envoy::extensions::clusters::aggregate::v3::ClusterConfig;
131 using ::envoy::extensions::filters::http::fault::v3::HTTPFault;
132 using ::envoy::extensions::filters::network::http_connection_manager::v3::
133 HttpConnectionManager;
134 using ::envoy::extensions::filters::network::http_connection_manager::v3::
135 HttpFilter;
136 using ::envoy::extensions::transport_sockets::tls::v3::DownstreamTlsContext;
137 using ::envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext;
138 using ::envoy::type::matcher::v3::StringMatcher;
139 using ::envoy::type::v3::FractionalPercent;
140
141 using ClientStats = LrsServiceImpl::ClientStats;
142
143 constexpr char kDefaultLocalityRegion[] = "xds_default_locality_region";
144 constexpr char kDefaultLocalityZone[] = "xds_default_locality_zone";
145 constexpr char kLbDropType[] = "lb";
146 constexpr char kThrottleDropType[] = "throttle";
147 constexpr char kServerName[] = "server.example.com";
148 constexpr char kDefaultRouteConfigurationName[] = "route_config_name";
149 constexpr char kDefaultServerRouteConfigurationName[] =
150 "default_server_route_config_name";
151 constexpr char kDefaultClusterName[] = "cluster_name";
152 constexpr char kDefaultEdsServiceName[] = "eds_service_name";
153 constexpr int kDefaultLocalityWeight = 3;
154 constexpr int kDefaultLocalityPriority = 0;
155
156 constexpr char kRequestMessage[] = "Live long and prosper.";
157 constexpr char kDefaultServiceConfig[] =
158 "{\n"
159 " \"loadBalancingConfig\":[\n"
160 " { \"does_not_exist\":{} },\n"
161 " { \"xds_cluster_resolver_experimental\":{\n"
162 " \"discoveryMechanisms\": [\n"
163 " { \"clusterName\": \"server.example.com\",\n"
164 " \"type\": \"EDS\",\n"
165 " \"lrsLoadReportingServerName\": \"\"\n"
166 " } ]\n"
167 " } }\n"
168 " ]\n"
169 "}";
170 constexpr char kDefaultServiceConfigWithoutLoadReporting[] =
171 "{\n"
172 " \"loadBalancingConfig\":[\n"
173 " { \"does_not_exist\":{} },\n"
174 " { \"xds_cluster_resolver_experimental\":{\n"
175 " \"discoveryMechanisms\": [\n"
176 " { \"clusterName\": \"server.example.com\",\n"
177 " \"type\": \"EDS\"\n"
178 " } ]\n"
179 " } }\n"
180 " ]\n"
181 "}";
182
183 constexpr char kBootstrapFileV3[] =
184 "{\n"
185 " \"xds_servers\": [\n"
186 " {\n"
187 " \"server_uri\": \"fake:///xds_server\",\n"
188 " \"channel_creds\": [\n"
189 " {\n"
190 " \"type\": \"fake\"\n"
191 " }\n"
192 " ],\n"
193 " \"server_features\": [\"xds_v3\"]\n"
194 " }\n"
195 " ],\n"
196 " \"node\": {\n"
197 " \"id\": \"xds_end2end_test\",\n"
198 " \"cluster\": \"test\",\n"
199 " \"metadata\": {\n"
200 " \"foo\": \"bar\"\n"
201 " },\n"
202 " \"locality\": {\n"
203 " \"region\": \"corp\",\n"
204 " \"zone\": \"svl\",\n"
205 " \"sub_zone\": \"mp3\"\n"
206 " }\n"
207 " },\n"
208 " \"server_listener_resource_name_template\": "
209 "\"grpc/server?xds.resource.listening_address=%s\",\n"
210 " \"certificate_providers\": {\n"
211 " \"fake_plugin1\": {\n"
212 " \"plugin_name\": \"fake1\"\n"
213 " },\n"
214 " \"fake_plugin2\": {\n"
215 " \"plugin_name\": \"fake2\"\n"
216 " },\n"
217 " \"file_plugin\": {\n"
218 " \"plugin_name\": \"file_watcher\",\n"
219 " \"config\": {\n"
220 " \"certificate_file\": \"src/core/tsi/test_creds/client.pem\",\n"
221 " \"private_key_file\": \"src/core/tsi/test_creds/client.key\",\n"
222 " \"ca_certificate_file\": \"src/core/tsi/test_creds/ca.pem\"\n"
223 " }"
224 " }\n"
225 " }\n"
226 "}\n";
227
228 constexpr char kBootstrapFileV2[] =
229 "{\n"
230 " \"xds_servers\": [\n"
231 " {\n"
232 " \"server_uri\": \"fake:///xds_server\",\n"
233 " \"channel_creds\": [\n"
234 " {\n"
235 " \"type\": \"fake\"\n"
236 " }\n"
237 " ]\n"
238 " }\n"
239 " ],\n"
240 " \"node\": {\n"
241 " \"id\": \"xds_end2end_test\",\n"
242 " \"cluster\": \"test\",\n"
243 " \"metadata\": {\n"
244 " \"foo\": \"bar\"\n"
245 " },\n"
246 " \"locality\": {\n"
247 " \"region\": \"corp\",\n"
248 " \"zone\": \"svl\",\n"
249 " \"sub_zone\": \"mp3\"\n"
250 " }\n"
251 " }\n"
252 "}\n";
253 constexpr char kCaCertPath[] = "src/core/tsi/test_creds/ca.pem";
254 constexpr char kServerCertPath[] = "src/core/tsi/test_creds/server1.pem";
255 constexpr char kServerKeyPath[] = "src/core/tsi/test_creds/server1.key";
256 constexpr char kClientCertPath[] = "src/core/tsi/test_creds/client.pem";
257 constexpr char kClientKeyPath[] = "src/core/tsi/test_creds/client.key";
258 constexpr char kBadClientCertPath[] = "src/core/tsi/test_creds/badclient.pem";
259 constexpr char kBadClientKeyPath[] = "src/core/tsi/test_creds/badclient.key";
260
261 char* g_bootstrap_file_v3;
262 char* g_bootstrap_file_v2;
263
WriteBootstrapFiles()264 void WriteBootstrapFiles() {
265 char* bootstrap_file;
266 FILE* out = gpr_tmpfile("xds_bootstrap_v3", &bootstrap_file);
267 fputs(kBootstrapFileV3, out);
268 fclose(out);
269 g_bootstrap_file_v3 = bootstrap_file;
270 out = gpr_tmpfile("xds_bootstrap_v2", &bootstrap_file);
271 fputs(kBootstrapFileV2, out);
272 fclose(out);
273 g_bootstrap_file_v2 = bootstrap_file;
274 }
275
276 template <typename RpcService>
277 class BackendServiceImpl
278 : public CountedService<TestMultipleServiceImpl<RpcService>> {
279 public:
BackendServiceImpl()280 BackendServiceImpl() {}
281
Echo(ServerContext * context,const EchoRequest * request,EchoResponse * response)282 Status Echo(ServerContext* context, const EchoRequest* request,
283 EchoResponse* response) override {
284 auto peer_identity = context->auth_context()->GetPeerIdentity();
285 CountedService<TestMultipleServiceImpl<RpcService>>::IncreaseRequestCount();
286 const auto status =
287 TestMultipleServiceImpl<RpcService>::Echo(context, request, response);
288 CountedService<
289 TestMultipleServiceImpl<RpcService>>::IncreaseResponseCount();
290 {
291 grpc_core::MutexLock lock(&mu_);
292 clients_.insert(context->peer());
293 last_peer_identity_.clear();
294 for (const auto& entry : peer_identity) {
295 last_peer_identity_.emplace_back(entry.data(), entry.size());
296 }
297 }
298 return status;
299 }
300
Echo1(ServerContext * context,const EchoRequest * request,EchoResponse * response)301 Status Echo1(ServerContext* context, const EchoRequest* request,
302 EchoResponse* response) override {
303 return Echo(context, request, response);
304 }
305
Echo2(ServerContext * context,const EchoRequest * request,EchoResponse * response)306 Status Echo2(ServerContext* context, const EchoRequest* request,
307 EchoResponse* response) override {
308 return Echo(context, request, response);
309 }
310
Start()311 void Start() {}
Shutdown()312 void Shutdown() {}
313
clients()314 std::set<std::string> clients() {
315 grpc_core::MutexLock lock(&mu_);
316 return clients_;
317 }
318
last_peer_identity()319 const std::vector<std::string>& last_peer_identity() {
320 grpc_core::MutexLock lock(&mu_);
321 return last_peer_identity_;
322 }
323
324 private:
325 grpc_core::Mutex mu_;
326 std::set<std::string> clients_ ABSL_GUARDED_BY(mu_);
327 std::vector<std::string> last_peer_identity_ ABSL_GUARDED_BY(mu_);
328 };
329
330 class TestType {
331 public:
332 enum FilterConfigSetup {
333 // Set the fault injection filter directly from LDS
334 kHTTPConnectionManagerOriginal,
335 // Enable the fault injection filter in LDS, but override the filter config
336 // in route.
337 kRouteOverride,
338 };
339
340 enum BootstrapSource {
341 kBootstrapFromChannelArg,
342 kBootstrapFromFile,
343 kBootstrapFromEnvVar,
344 };
345
set_use_fake_resolver()346 TestType& set_use_fake_resolver() {
347 use_fake_resolver_ = true;
348 return *this;
349 }
350
set_enable_load_reporting()351 TestType& set_enable_load_reporting() {
352 enable_load_reporting_ = true;
353 return *this;
354 }
355
set_enable_rds_testing()356 TestType& set_enable_rds_testing() {
357 enable_rds_testing_ = true;
358 return *this;
359 }
360
set_use_v2()361 TestType& set_use_v2() {
362 use_v2_ = true;
363 return *this;
364 }
365
set_use_xds_credentials()366 TestType& set_use_xds_credentials() {
367 use_xds_credentials_ = true;
368 return *this;
369 }
370
set_use_csds_streaming()371 TestType& set_use_csds_streaming() {
372 use_csds_streaming_ = true;
373 return *this;
374 }
375
set_filter_config_setup(FilterConfigSetup setup)376 TestType& set_filter_config_setup(FilterConfigSetup setup) {
377 filter_config_setup_ = setup;
378 return *this;
379 }
380
set_bootstrap_source(BootstrapSource bootstrap_source)381 TestType& set_bootstrap_source(BootstrapSource bootstrap_source) {
382 bootstrap_source_ = bootstrap_source;
383 return *this;
384 }
385
use_fake_resolver() const386 bool use_fake_resolver() const { return use_fake_resolver_; }
enable_load_reporting() const387 bool enable_load_reporting() const { return enable_load_reporting_; }
enable_rds_testing() const388 bool enable_rds_testing() const { return enable_rds_testing_; }
use_v2() const389 bool use_v2() const { return use_v2_; }
use_xds_credentials() const390 bool use_xds_credentials() const { return use_xds_credentials_; }
use_csds_streaming() const391 bool use_csds_streaming() const { return use_csds_streaming_; }
filter_config_setup() const392 FilterConfigSetup filter_config_setup() const { return filter_config_setup_; }
bootstrap_source() const393 BootstrapSource bootstrap_source() const { return bootstrap_source_; }
394
AsString() const395 std::string AsString() const {
396 std::string retval = (use_fake_resolver_ ? "FakeResolver" : "XdsResolver");
397 retval += (use_v2_ ? "V2" : "V3");
398 if (enable_load_reporting_) retval += "WithLoadReporting";
399 if (enable_rds_testing_) retval += "Rds";
400 if (use_xds_credentials_) retval += "XdsCreds";
401 if (use_csds_streaming_) retval += "CsdsStreaming";
402 if (filter_config_setup_ == kRouteOverride) {
403 retval += "FilterPerRouteOverride";
404 }
405 if (bootstrap_source_ == kBootstrapFromFile) {
406 retval += "BootstrapFromFile";
407 } else if (bootstrap_source_ == kBootstrapFromEnvVar) {
408 retval += "BootstrapFromEnvVar";
409 }
410 return retval;
411 }
412
413 private:
414 bool use_fake_resolver_ = false;
415 bool enable_load_reporting_ = false;
416 bool enable_rds_testing_ = false;
417 bool use_v2_ = false;
418 bool use_xds_credentials_ = false;
419 bool use_csds_streaming_ = false;
420 FilterConfigSetup filter_config_setup_ = kHTTPConnectionManagerOriginal;
421 BootstrapSource bootstrap_source_ = kBootstrapFromChannelArg;
422 };
423
ReadFile(const char * file_path)424 std::string ReadFile(const char* file_path) {
425 grpc_slice slice;
426 GPR_ASSERT(
427 GRPC_LOG_IF_ERROR("load_file", grpc_load_file(file_path, 0, &slice)));
428 std::string file_contents(grpc_core::StringViewFromSlice(slice));
429 grpc_slice_unref(slice);
430 return file_contents;
431 }
432
ReadTlsIdentityPair(const char * key_path,const char * cert_path)433 grpc_core::PemKeyCertPairList ReadTlsIdentityPair(const char* key_path,
434 const char* cert_path) {
435 return grpc_core::PemKeyCertPairList{
436 grpc_core::PemKeyCertPair(ReadFile(key_path), ReadFile(cert_path))};
437 }
438
439 // Based on StaticDataCertificateProvider, but provides alternate certificates
440 // if the certificate name is not empty.
441 class FakeCertificateProvider final : public grpc_tls_certificate_provider {
442 public:
443 struct CertData {
444 std::string root_certificate;
445 grpc_core::PemKeyCertPairList identity_key_cert_pairs;
446 };
447
448 using CertDataMap = std::map<std::string /*cert_name */, CertData>;
449
FakeCertificateProvider(CertDataMap cert_data_map)450 explicit FakeCertificateProvider(CertDataMap cert_data_map)
451 : distributor_(
452 grpc_core::MakeRefCounted<grpc_tls_certificate_distributor>()),
453 cert_data_map_(std::move(cert_data_map)) {
454 distributor_->SetWatchStatusCallback([this](std::string cert_name,
455 bool root_being_watched,
456 bool identity_being_watched) {
457 if (!root_being_watched && !identity_being_watched) return;
458 auto it = cert_data_map_.find(cert_name);
459 if (it == cert_data_map_.end()) {
460 grpc_error_handle error =
461 GRPC_ERROR_CREATE_FROM_CPP_STRING(absl::StrCat(
462 "No certificates available for cert_name \"", cert_name, "\""));
463 distributor_->SetErrorForCert(cert_name, GRPC_ERROR_REF(error),
464 GRPC_ERROR_REF(error));
465 GRPC_ERROR_UNREF(error);
466 } else {
467 absl::optional<std::string> root_certificate;
468 absl::optional<grpc_core::PemKeyCertPairList> pem_key_cert_pairs;
469 if (root_being_watched) {
470 root_certificate = it->second.root_certificate;
471 }
472 if (identity_being_watched) {
473 pem_key_cert_pairs = it->second.identity_key_cert_pairs;
474 }
475 distributor_->SetKeyMaterials(cert_name, std::move(root_certificate),
476 std::move(pem_key_cert_pairs));
477 }
478 });
479 }
480
~FakeCertificateProvider()481 ~FakeCertificateProvider() override {
482 distributor_->SetWatchStatusCallback(nullptr);
483 }
484
distributor() const485 grpc_core::RefCountedPtr<grpc_tls_certificate_distributor> distributor()
486 const override {
487 return distributor_;
488 }
489
490 private:
491 grpc_core::RefCountedPtr<grpc_tls_certificate_distributor> distributor_;
492 CertDataMap cert_data_map_;
493 };
494
495 class FakeCertificateProviderFactory
496 : public grpc_core::CertificateProviderFactory {
497 public:
498 class Config : public grpc_core::CertificateProviderFactory::Config {
499 public:
Config(const char * name)500 explicit Config(const char* name) : name_(name) {}
501
name() const502 const char* name() const override { return name_; }
503
ToString() const504 std::string ToString() const override { return "{}"; }
505
506 private:
507 const char* name_;
508 };
509
FakeCertificateProviderFactory(const char * name,FakeCertificateProvider::CertDataMap ** cert_data_map)510 FakeCertificateProviderFactory(
511 const char* name, FakeCertificateProvider::CertDataMap** cert_data_map)
512 : name_(name), cert_data_map_(cert_data_map) {
513 GPR_ASSERT(cert_data_map != nullptr);
514 }
515
name() const516 const char* name() const override { return name_; }
517
518 grpc_core::RefCountedPtr<grpc_core::CertificateProviderFactory::Config>
CreateCertificateProviderConfig(const grpc_core::Json &,grpc_error_handle *)519 CreateCertificateProviderConfig(const grpc_core::Json& /*config_json*/,
520 grpc_error_handle* /*error*/) override {
521 return grpc_core::MakeRefCounted<Config>(name_);
522 }
523
524 grpc_core::RefCountedPtr<grpc_tls_certificate_provider>
CreateCertificateProvider(grpc_core::RefCountedPtr<grpc_core::CertificateProviderFactory::Config>)525 CreateCertificateProvider(
526 grpc_core::RefCountedPtr<grpc_core::CertificateProviderFactory::Config>
527 /*config*/) override {
528 if (*cert_data_map_ == nullptr) return nullptr;
529 return grpc_core::MakeRefCounted<FakeCertificateProvider>(**cert_data_map_);
530 }
531
532 private:
533 const char* name_;
534 FakeCertificateProvider::CertDataMap** cert_data_map_;
535 };
536
537 // Global variables for each provider.
538 FakeCertificateProvider::CertDataMap* g_fake1_cert_data_map = nullptr;
539 FakeCertificateProvider::CertDataMap* g_fake2_cert_data_map = nullptr;
540
ServerAuthCheckSchedule(void *,grpc_tls_server_authorization_check_arg * arg)541 int ServerAuthCheckSchedule(void* /* config_user_data */,
542 grpc_tls_server_authorization_check_arg* arg) {
543 arg->success = 1;
544 arg->status = GRPC_STATUS_OK;
545 return 0; /* synchronous check */
546 }
547
CreateTlsFallbackCredentials()548 std::shared_ptr<ChannelCredentials> CreateTlsFallbackCredentials() {
549 // TODO(yashykt): Switch to using C++ API once b/173823806 is fixed.
550 grpc_tls_credentials_options* options = grpc_tls_credentials_options_create();
551 grpc_tls_credentials_options_set_server_verification_option(
552 options, GRPC_TLS_SKIP_HOSTNAME_VERIFICATION);
553 grpc_tls_credentials_options_set_certificate_provider(
554 options,
555 grpc_core::MakeRefCounted<grpc_core::StaticDataCertificateProvider>(
556 ReadFile(kCaCertPath),
557 ReadTlsIdentityPair(kServerKeyPath, kServerCertPath))
558 .get());
559 grpc_tls_credentials_options_watch_root_certs(options);
560 grpc_tls_credentials_options_watch_identity_key_cert_pairs(options);
561 grpc_tls_server_authorization_check_config* check_config =
562 grpc_tls_server_authorization_check_config_create(
563 nullptr, ServerAuthCheckSchedule, nullptr, nullptr);
564 grpc_tls_credentials_options_set_server_authorization_check_config(
565 options, check_config);
566 auto channel_creds = std::make_shared<SecureChannelCredentials>(
567 grpc_tls_credentials_create(options));
568 grpc_tls_server_authorization_check_config_release(check_config);
569 return channel_creds;
570 }
571
572 // A No-op HTTP filter used for verifying parsing logic.
573 class NoOpHttpFilter : public grpc_core::XdsHttpFilterImpl {
574 public:
NoOpHttpFilter(std::string name,bool supported_on_clients,bool supported_on_servers,bool is_terminal_filter)575 NoOpHttpFilter(std::string name, bool supported_on_clients,
576 bool supported_on_servers, bool is_terminal_filter)
577 : name_(std::move(name)),
578 supported_on_clients_(supported_on_clients),
579 supported_on_servers_(supported_on_servers),
580 is_terminal_filter_(is_terminal_filter) {}
581
PopulateSymtab(upb_symtab *) const582 void PopulateSymtab(upb_symtab* /* symtab */) const override {}
583
584 absl::StatusOr<grpc_core::XdsHttpFilterImpl::FilterConfig>
GenerateFilterConfig(upb_strview,upb_arena *) const585 GenerateFilterConfig(upb_strview /* serialized_filter_config */,
586 upb_arena* /* arena */) const override {
587 return grpc_core::XdsHttpFilterImpl::FilterConfig{name_, grpc_core::Json()};
588 }
589
590 absl::StatusOr<grpc_core::XdsHttpFilterImpl::FilterConfig>
GenerateFilterConfigOverride(upb_strview,upb_arena *) const591 GenerateFilterConfigOverride(upb_strview /*serialized_filter_config*/,
592 upb_arena* /*arena*/) const override {
593 return grpc_core::XdsHttpFilterImpl::FilterConfig{name_, grpc_core::Json()};
594 }
595
channel_filter() const596 const grpc_channel_filter* channel_filter() const override { return nullptr; }
597
598 absl::StatusOr<grpc_core::XdsHttpFilterImpl::ServiceConfigJsonEntry>
GenerateServiceConfig(const FilterConfig &,const FilterConfig *) const599 GenerateServiceConfig(
600 const FilterConfig& /*hcm_filter_config*/,
601 const FilterConfig* /*filter_config_override*/) const override {
602 return grpc_core::XdsHttpFilterImpl::ServiceConfigJsonEntry{name_, ""};
603 }
604
IsSupportedOnClients() const605 bool IsSupportedOnClients() const override { return supported_on_clients_; }
606
IsSupportedOnServers() const607 bool IsSupportedOnServers() const override { return supported_on_servers_; }
608
IsTerminalFilter() const609 bool IsTerminalFilter() const override { return is_terminal_filter_; }
610
611 private:
612 const std::string name_;
613 const bool supported_on_clients_;
614 const bool supported_on_servers_;
615 const bool is_terminal_filter_;
616 };
617
618 // There is slight difference between time fetched by GPR and by C++ system
619 // clock API. It's unclear if they are using the same syscall, but we do know
620 // GPR round the number at millisecond-level. This creates a 1ms difference,
621 // which could cause flake.
NowFromCycleCounter()622 grpc_millis NowFromCycleCounter() {
623 return grpc_timespec_to_millis_round_down(gpr_now(GPR_CLOCK_MONOTONIC));
624 }
625
626 // Returns the number of RPCs needed to pass error_tolerance at 99.99994%
627 // chance. Rolling dices in drop/fault-injection generates a binomial
628 // distribution (if our code is not horribly wrong). Let's make "n" the number
629 // of samples, "p" the probability. If we have np>5 & n(1-p)>5, we can
630 // approximately treat the binomial distribution as a normal distribution.
631 //
632 // For normal distribution, we can easily look up how many standard deviation we
633 // need to reach 99.995%. Based on Wiki's table
634 // https://en.wikipedia.org/wiki/68%E2%80%9395%E2%80%9399.7_rule, we need 5.00
635 // sigma (standard deviation) to cover the probability area of 99.99994%. In
636 // another word, for a sample with size "n" probability "p" error-tolerance "k",
637 // we want the error always land within 5.00 sigma. The sigma of binominal
638 // distribution and be computed as sqrt(np(1-p)). Hence, we have the equation:
639 //
640 // kn <= 5.00 * sqrt(np(1-p))
ComputeIdealNumRpcs(double p,double error_tolerance)641 size_t ComputeIdealNumRpcs(double p, double error_tolerance) {
642 GPR_ASSERT(p >= 0 && p <= 1);
643 size_t num_rpcs =
644 ceil(p * (1 - p) * 5.00 * 5.00 / error_tolerance / error_tolerance);
645 gpr_log(GPR_INFO,
646 "Sending %" PRIuPTR " RPCs for percentage=%.3f error_tolerance=%.3f",
647 num_rpcs, p, error_tolerance);
648 return num_rpcs;
649 }
650
651 // Channel arg pointer vtable for storing xDS channel args in the parent
652 // channel's channel args.
ChannelArgsArgCopy(void * p)653 void* ChannelArgsArgCopy(void* p) {
654 auto* args = static_cast<grpc_channel_args*>(p);
655 return grpc_channel_args_copy(args);
656 }
ChannelArgsArgDestroy(void * p)657 void ChannelArgsArgDestroy(void* p) {
658 auto* args = static_cast<grpc_channel_args*>(p);
659 grpc_channel_args_destroy(args);
660 }
ChannelArgsArgCmp(void * a,void * b)661 int ChannelArgsArgCmp(void* a, void* b) {
662 auto* args_a = static_cast<grpc_channel_args*>(a);
663 auto* args_b = static_cast<grpc_channel_args*>(b);
664 return grpc_channel_args_compare(args_a, args_b);
665 }
666 const grpc_arg_pointer_vtable kChannelArgsArgVtable = {
667 ChannelArgsArgCopy, ChannelArgsArgDestroy, ChannelArgsArgCmp};
668
669 class XdsEnd2endTest : public ::testing::TestWithParam<TestType> {
670 protected:
671 // TODO(roth): We currently set the number of backends and number of
672 // balancers on a per-test-suite basis, not a per-test-case basis.
673 // However, not every individual test case in a given test suite uses
674 // the same number of backends or balancers, so we wind up having to
675 // set the numbers for the test suite to the max number needed by any
676 // one test case in that test suite. This results in starting more
677 // servers (and using more ports) than we actually need. When we have
678 // time, change each test to directly start the number of backends and
679 // balancers that it needs, so that we aren't wasting resources.
XdsEnd2endTest(size_t num_backends,size_t num_balancers,int client_load_reporting_interval_seconds=100,bool use_xds_enabled_server=false)680 XdsEnd2endTest(size_t num_backends, size_t num_balancers,
681 int client_load_reporting_interval_seconds = 100,
682 bool use_xds_enabled_server = false)
683 : num_backends_(num_backends),
684 num_balancers_(num_balancers),
685 client_load_reporting_interval_seconds_(
686 client_load_reporting_interval_seconds),
687 use_xds_enabled_server_(use_xds_enabled_server) {}
688
SetUp()689 void SetUp() override {
690 bool localhost_resolves_to_ipv4 = false;
691 bool localhost_resolves_to_ipv6 = false;
692 grpc_core::LocalhostResolves(&localhost_resolves_to_ipv4,
693 &localhost_resolves_to_ipv6);
694 ipv6_only_ = !localhost_resolves_to_ipv4 && localhost_resolves_to_ipv6;
695 // Initialize default xDS resources.
696 // Construct LDS resource.
697 default_listener_.set_name(kServerName);
698 HttpConnectionManager http_connection_manager;
699 if (!GetParam().use_v2()) {
700 auto* filter = http_connection_manager.add_http_filters();
701 filter->set_name("router");
702 filter->mutable_typed_config()->PackFrom(
703 envoy::extensions::filters::http::router::v3::Router());
704 }
705 default_listener_.mutable_api_listener()->mutable_api_listener()->PackFrom(
706 http_connection_manager);
707 // Construct RDS resource.
708 default_route_config_.set_name(kDefaultRouteConfigurationName);
709 auto* virtual_host = default_route_config_.add_virtual_hosts();
710 virtual_host->add_domains("*");
711 auto* route = virtual_host->add_routes();
712 route->mutable_match()->set_prefix("");
713 route->mutable_route()->set_cluster(kDefaultClusterName);
714 // Construct CDS resource.
715 default_cluster_.set_name(kDefaultClusterName);
716 default_cluster_.set_type(Cluster::EDS);
717 auto* eds_config = default_cluster_.mutable_eds_cluster_config();
718 eds_config->mutable_eds_config()->mutable_ads();
719 eds_config->set_service_name(kDefaultEdsServiceName);
720 default_cluster_.set_lb_policy(Cluster::ROUND_ROBIN);
721 if (GetParam().enable_load_reporting()) {
722 default_cluster_.mutable_lrs_server()->mutable_self();
723 }
724 // Construct a default server-side RDS resource for tests to use.
725 default_server_route_config_.set_name(kDefaultServerRouteConfigurationName);
726 virtual_host = default_server_route_config_.add_virtual_hosts();
727 virtual_host->add_domains("*");
728 route = virtual_host->add_routes();
729 route->mutable_match()->set_prefix("");
730 route->mutable_non_forwarding_action();
731 // Construct a default server-side Listener resource
732 default_server_listener_.mutable_address()
733 ->mutable_socket_address()
734 ->set_address(ipv6_only_ ? "::1" : "127.0.0.1");
735 default_server_listener_.mutable_default_filter_chain()
736 ->add_filters()
737 ->mutable_typed_config()
738 ->PackFrom(http_connection_manager);
739 // Create the backends but don't start them yet. We need to create the
740 // backends to allocate the ports, so that we know what resource names to
741 // populate in the xDS servers when we start them. However, we can't start
742 // the backends until after we've started the xDS servers, because in the
743 // tests that use xDS-enabled servers, the backends will try to contact the
744 // xDS servers as soon as they start up.
745 for (size_t i = 0; i < num_backends_; ++i) {
746 backends_.emplace_back(
747 new BackendServerThread(this, use_xds_enabled_server_));
748 }
749 // Start the load balancers.
750 for (size_t i = 0; i < num_balancers_; ++i) {
751 balancers_.emplace_back(new BalancerServerThread(
752 this, GetParam().enable_load_reporting()
753 ? client_load_reporting_interval_seconds_
754 : 0));
755 balancers_.back()->Start();
756 // Initialize resources.
757 SetListenerAndRouteConfiguration(i, default_listener_,
758 default_route_config_);
759 if (use_xds_enabled_server_) {
760 for (const auto& backend : backends_) {
761 SetServerListenerNameAndRouteConfiguration(
762 i, default_server_listener_, backend->port(),
763 default_server_route_config_);
764 }
765 }
766 balancers_.back()->ads_service()->SetCdsResource(default_cluster_);
767 }
768 // Create fake resolver response generators used by client.
769 if (GetParam().use_fake_resolver()) {
770 response_generator_ =
771 grpc_core::MakeRefCounted<grpc_core::FakeResolverResponseGenerator>();
772 }
773 logical_dns_cluster_resolver_response_generator_ =
774 grpc_core::MakeRefCounted<grpc_core::FakeResolverResponseGenerator>();
775 lb_channel_response_generator_ =
776 grpc_core::MakeRefCounted<grpc_core::FakeResolverResponseGenerator>();
777 // Construct channel args for XdsClient.
778 xds_channel_args_to_add_.emplace_back(
779 grpc_core::FakeResolverResponseGenerator::MakeChannelArg(
780 lb_channel_response_generator_.get()));
781 if (xds_resource_does_not_exist_timeout_ms_ > 0) {
782 xds_channel_args_to_add_.emplace_back(grpc_channel_arg_integer_create(
783 const_cast<char*>(GRPC_ARG_XDS_RESOURCE_DOES_NOT_EXIST_TIMEOUT_MS),
784 xds_resource_does_not_exist_timeout_ms_));
785 }
786 xds_channel_args_.num_args = xds_channel_args_to_add_.size();
787 xds_channel_args_.args = xds_channel_args_to_add_.data();
788 // Initialize XdsClient state.
789 // TODO(roth): Consider changing this to dynamically generate the
790 // bootstrap config in each individual test instead of hard-coding
791 // the contents here. That would allow us to use an ipv4: or ipv6:
792 // URI for the xDS server instead of using the fake resolver.
793 if (GetParam().bootstrap_source() == TestType::kBootstrapFromEnvVar) {
794 gpr_setenv("GRPC_XDS_BOOTSTRAP_CONFIG",
795 GetParam().use_v2() ? kBootstrapFileV2 : kBootstrapFileV3);
796 } else if (GetParam().bootstrap_source() == TestType::kBootstrapFromFile) {
797 gpr_setenv("GRPC_XDS_BOOTSTRAP", GetParam().use_v2()
798 ? g_bootstrap_file_v2
799 : g_bootstrap_file_v3);
800 }
801 if (GetParam().bootstrap_source() != TestType::kBootstrapFromChannelArg) {
802 // If getting bootstrap from channel arg, we'll pass these args in
803 // via the parent channel args in CreateChannel() instead.
804 grpc_core::internal::SetXdsChannelArgsForTest(&xds_channel_args_);
805 // Make sure each test creates a new XdsClient instance rather than
806 // reusing the one from the previous test. This avoids spurious failures
807 // caused when a load reporting test runs after a non-load reporting test
808 // and the XdsClient is still talking to the old LRS server, which fails
809 // because it's not expecting the client to connect. It also
810 // ensures that each test can independently set the global channel
811 // args for the xDS channel.
812 grpc_core::internal::UnsetGlobalXdsClientForTest();
813 }
814 // Start the backends
815 for (const auto& backend : backends_) {
816 backend->Start();
817 }
818 // Create channel and stub.
819 ResetStub();
820 }
821
DefaultEdsServiceName() const822 const char* DefaultEdsServiceName() const {
823 return GetParam().use_fake_resolver() ? kServerName
824 : kDefaultEdsServiceName;
825 }
826
TearDown()827 void TearDown() override {
828 ShutdownAllBackends();
829 for (auto& balancer : balancers_) balancer->Shutdown();
830 // Clear global xDS channel args, since they will go out of scope
831 // when this test object is destroyed.
832 grpc_core::internal::SetXdsChannelArgsForTest(nullptr);
833 gpr_unsetenv("GRPC_XDS_BOOTSTRAP");
834 gpr_unsetenv("GRPC_XDS_BOOTSTRAP_CONFIG");
835 }
836
StartAllBackends()837 void StartAllBackends() {
838 for (auto& backend : backends_) backend->Start();
839 }
840
StartBackend(size_t index)841 void StartBackend(size_t index) { backends_[index]->Start(); }
842
ShutdownAllBackends()843 void ShutdownAllBackends() {
844 for (auto& backend : backends_) backend->Shutdown();
845 }
846
ShutdownBackend(size_t index)847 void ShutdownBackend(size_t index) { backends_[index]->Shutdown(); }
848
ResetStub(int failover_timeout=0)849 void ResetStub(int failover_timeout = 0) {
850 channel_ = CreateChannel(failover_timeout);
851 stub_ = grpc::testing::EchoTestService::NewStub(channel_);
852 stub1_ = grpc::testing::EchoTest1Service::NewStub(channel_);
853 stub2_ = grpc::testing::EchoTest2Service::NewStub(channel_);
854 }
855
CreateChannel(int failover_timeout=0,const char * server_name=kServerName,grpc_core::FakeResolverResponseGenerator * response_generator=nullptr,grpc_channel_args * xds_channel_args=nullptr)856 std::shared_ptr<Channel> CreateChannel(
857 int failover_timeout = 0, const char* server_name = kServerName,
858 grpc_core::FakeResolverResponseGenerator* response_generator = nullptr,
859 grpc_channel_args* xds_channel_args = nullptr) {
860 ChannelArguments args;
861 if (failover_timeout > 0) {
862 args.SetInt(GRPC_ARG_PRIORITY_FAILOVER_TIMEOUT_MS, failover_timeout);
863 }
864 // If the parent channel is using the fake resolver, we inject the
865 // response generator here.
866 if (GetParam().use_fake_resolver()) {
867 if (response_generator == nullptr) {
868 response_generator = response_generator_.get();
869 }
870 args.SetPointerWithVtable(
871 GRPC_ARG_FAKE_RESOLVER_RESPONSE_GENERATOR, response_generator,
872 &grpc_core::FakeResolverResponseGenerator::kChannelArgPointerVtable);
873 }
874 if (GetParam().bootstrap_source() == TestType::kBootstrapFromChannelArg) {
875 // We're getting the bootstrap from a channel arg, so we do the
876 // same thing for the response generator to use for the xDS
877 // channel and the xDS resource-does-not-exist timeout value.
878 args.SetString(GRPC_ARG_TEST_ONLY_DO_NOT_USE_IN_PROD_XDS_BOOTSTRAP_CONFIG,
879 GetParam().use_v2() ? kBootstrapFileV2 : kBootstrapFileV3);
880 if (xds_channel_args == nullptr) xds_channel_args = &xds_channel_args_;
881 args.SetPointerWithVtable(
882 GRPC_ARG_TEST_ONLY_DO_NOT_USE_IN_PROD_XDS_CLIENT_CHANNEL_ARGS,
883 xds_channel_args, &kChannelArgsArgVtable);
884 }
885 args.SetPointerWithVtable(
886 GRPC_ARG_XDS_LOGICAL_DNS_CLUSTER_FAKE_RESOLVER_RESPONSE_GENERATOR,
887 logical_dns_cluster_resolver_response_generator_.get(),
888 &grpc_core::FakeResolverResponseGenerator::kChannelArgPointerVtable);
889 std::string uri = absl::StrCat(
890 GetParam().use_fake_resolver() ? "fake" : "xds", ":///", server_name);
891 std::shared_ptr<ChannelCredentials> channel_creds =
892 GetParam().use_xds_credentials()
893 ? XdsCredentials(CreateTlsFallbackCredentials())
894 : std::make_shared<SecureChannelCredentials>(
895 grpc_fake_transport_security_credentials_create());
896 return ::grpc::CreateCustomChannel(uri, channel_creds, args);
897 }
898
899 enum RpcService {
900 SERVICE_ECHO,
901 SERVICE_ECHO1,
902 SERVICE_ECHO2,
903 };
904
905 enum RpcMethod {
906 METHOD_ECHO,
907 METHOD_ECHO1,
908 METHOD_ECHO2,
909 };
910
911 struct RpcOptions {
912 RpcService service = SERVICE_ECHO;
913 RpcMethod method = METHOD_ECHO;
914 int timeout_ms = 1000;
915 bool wait_for_ready = false;
916 bool server_fail = false;
917 std::vector<std::pair<std::string, std::string>> metadata;
918 int server_sleep_us = 0;
919 int client_cancel_after_us = 0;
920 bool skip_cancelled_check = false;
921 StatusCode server_expected_error = StatusCode::OK;
922
RpcOptionsgrpc::testing::__anond1b637be0111::XdsEnd2endTest::RpcOptions923 RpcOptions() {}
924
set_rpc_servicegrpc::testing::__anond1b637be0111::XdsEnd2endTest::RpcOptions925 RpcOptions& set_rpc_service(RpcService rpc_service) {
926 service = rpc_service;
927 return *this;
928 }
929
set_rpc_methodgrpc::testing::__anond1b637be0111::XdsEnd2endTest::RpcOptions930 RpcOptions& set_rpc_method(RpcMethod rpc_method) {
931 method = rpc_method;
932 return *this;
933 }
934
set_timeout_msgrpc::testing::__anond1b637be0111::XdsEnd2endTest::RpcOptions935 RpcOptions& set_timeout_ms(int rpc_timeout_ms) {
936 timeout_ms = rpc_timeout_ms;
937 return *this;
938 }
939
set_wait_for_readygrpc::testing::__anond1b637be0111::XdsEnd2endTest::RpcOptions940 RpcOptions& set_wait_for_ready(bool rpc_wait_for_ready) {
941 wait_for_ready = rpc_wait_for_ready;
942 return *this;
943 }
944
set_server_failgrpc::testing::__anond1b637be0111::XdsEnd2endTest::RpcOptions945 RpcOptions& set_server_fail(bool rpc_server_fail) {
946 server_fail = rpc_server_fail;
947 return *this;
948 }
949
set_skip_cancelled_checkgrpc::testing::__anond1b637be0111::XdsEnd2endTest::RpcOptions950 RpcOptions& set_skip_cancelled_check(bool rpc_skip_cancelled_check) {
951 skip_cancelled_check = rpc_skip_cancelled_check;
952 return *this;
953 }
954
set_metadatagrpc::testing::__anond1b637be0111::XdsEnd2endTest::RpcOptions955 RpcOptions& set_metadata(
956 std::vector<std::pair<std::string, std::string>> rpc_metadata) {
957 metadata = std::move(rpc_metadata);
958 return *this;
959 }
960
set_server_sleep_usgrpc::testing::__anond1b637be0111::XdsEnd2endTest::RpcOptions961 RpcOptions& set_server_sleep_us(int rpc_server_sleep_us) {
962 server_sleep_us = rpc_server_sleep_us;
963 return *this;
964 }
965
set_client_cancel_after_usgrpc::testing::__anond1b637be0111::XdsEnd2endTest::RpcOptions966 RpcOptions& set_client_cancel_after_us(int rpc_client_cancel_after_us) {
967 client_cancel_after_us = rpc_client_cancel_after_us;
968 return *this;
969 }
970
set_server_expected_errorgrpc::testing::__anond1b637be0111::XdsEnd2endTest::RpcOptions971 RpcOptions& set_server_expected_error(StatusCode code) {
972 server_expected_error = code;
973 return *this;
974 }
975
976 // Populates context and request.
SetupRpcgrpc::testing::__anond1b637be0111::XdsEnd2endTest::RpcOptions977 void SetupRpc(ClientContext* context, EchoRequest* request) const {
978 for (const auto& item : metadata) {
979 context->AddMetadata(item.first, item.second);
980 }
981 if (timeout_ms != 0) {
982 context->set_deadline(
983 grpc_timeout_milliseconds_to_deadline(timeout_ms));
984 }
985 if (wait_for_ready) context->set_wait_for_ready(true);
986 request->set_message(kRequestMessage);
987 if (server_fail) {
988 request->mutable_param()->mutable_expected_error()->set_code(
989 GRPC_STATUS_FAILED_PRECONDITION);
990 }
991 if (server_sleep_us != 0) {
992 request->mutable_param()->set_server_sleep_us(server_sleep_us);
993 }
994 if (client_cancel_after_us != 0) {
995 request->mutable_param()->set_client_cancel_after_us(
996 client_cancel_after_us);
997 }
998 if (skip_cancelled_check) {
999 request->mutable_param()->set_skip_cancelled_check(true);
1000 }
1001 }
1002 };
1003
1004 template <typename Stub>
SendRpcMethod(Stub * stub,const RpcOptions & rpc_options,ClientContext * context,EchoRequest & request,EchoResponse * response)1005 Status SendRpcMethod(Stub* stub, const RpcOptions& rpc_options,
1006 ClientContext* context, EchoRequest& request,
1007 EchoResponse* response) {
1008 switch (rpc_options.method) {
1009 case METHOD_ECHO:
1010 return (*stub)->Echo(context, request, response);
1011 case METHOD_ECHO1:
1012 return (*stub)->Echo1(context, request, response);
1013 case METHOD_ECHO2:
1014 return (*stub)->Echo2(context, request, response);
1015 }
1016 GPR_UNREACHABLE_CODE();
1017 }
1018
ResetBackendCounters(size_t start_index=0,size_t stop_index=0)1019 void ResetBackendCounters(size_t start_index = 0, size_t stop_index = 0) {
1020 if (stop_index == 0) stop_index = backends_.size();
1021 for (size_t i = start_index; i < stop_index; ++i) {
1022 backends_[i]->backend_service()->ResetCounters();
1023 backends_[i]->backend_service1()->ResetCounters();
1024 backends_[i]->backend_service2()->ResetCounters();
1025 }
1026 }
1027
SeenBackend(size_t backend_idx,const RpcService rpc_service=SERVICE_ECHO)1028 bool SeenBackend(size_t backend_idx,
1029 const RpcService rpc_service = SERVICE_ECHO) {
1030 switch (rpc_service) {
1031 case SERVICE_ECHO:
1032 if (backends_[backend_idx]->backend_service()->request_count() == 0) {
1033 return false;
1034 }
1035 break;
1036 case SERVICE_ECHO1:
1037 if (backends_[backend_idx]->backend_service1()->request_count() == 0) {
1038 return false;
1039 }
1040 break;
1041 case SERVICE_ECHO2:
1042 if (backends_[backend_idx]->backend_service2()->request_count() == 0) {
1043 return false;
1044 }
1045 break;
1046 }
1047 return true;
1048 }
1049
SeenAllBackends(size_t start_index=0,size_t stop_index=0,const RpcService rpc_service=SERVICE_ECHO)1050 bool SeenAllBackends(size_t start_index = 0, size_t stop_index = 0,
1051 const RpcService rpc_service = SERVICE_ECHO) {
1052 if (stop_index == 0) stop_index = backends_.size();
1053 for (size_t i = start_index; i < stop_index; ++i) {
1054 if (!SeenBackend(i, rpc_service)) {
1055 return false;
1056 }
1057 }
1058 return true;
1059 }
1060
SendRpcAndCount(int * num_total,int * num_ok,int * num_failure,int * num_drops,const RpcOptions & rpc_options=RpcOptions (),const char * drop_error_message_prefix="EDS-configured drop: ")1061 void SendRpcAndCount(
1062 int* num_total, int* num_ok, int* num_failure, int* num_drops,
1063 const RpcOptions& rpc_options = RpcOptions(),
1064 const char* drop_error_message_prefix = "EDS-configured drop: ") {
1065 const Status status = SendRpc(rpc_options);
1066 if (status.ok()) {
1067 ++*num_ok;
1068 } else {
1069 if (absl::StartsWith(status.error_message(), drop_error_message_prefix)) {
1070 ++*num_drops;
1071 } else {
1072 ++*num_failure;
1073 }
1074 }
1075 ++*num_total;
1076 }
1077
1078 struct WaitForBackendOptions {
1079 bool reset_counters = true;
1080 bool allow_failures = false;
1081
WaitForBackendOptionsgrpc::testing::__anond1b637be0111::XdsEnd2endTest::WaitForBackendOptions1082 WaitForBackendOptions() {}
1083
set_reset_countersgrpc::testing::__anond1b637be0111::XdsEnd2endTest::WaitForBackendOptions1084 WaitForBackendOptions& set_reset_counters(bool enable) {
1085 reset_counters = enable;
1086 return *this;
1087 }
1088
set_allow_failuresgrpc::testing::__anond1b637be0111::XdsEnd2endTest::WaitForBackendOptions1089 WaitForBackendOptions& set_allow_failures(bool enable) {
1090 allow_failures = enable;
1091 return *this;
1092 }
1093 };
1094
WaitForAllBackends(size_t start_index=0,size_t stop_index=0,const WaitForBackendOptions & wait_options=WaitForBackendOptions (),const RpcOptions & rpc_options=RpcOptions ())1095 std::tuple<int, int, int> WaitForAllBackends(
1096 size_t start_index = 0, size_t stop_index = 0,
1097 const WaitForBackendOptions& wait_options = WaitForBackendOptions(),
1098 const RpcOptions& rpc_options = RpcOptions()) {
1099 int num_ok = 0;
1100 int num_failure = 0;
1101 int num_drops = 0;
1102 int num_total = 0;
1103 gpr_log(GPR_INFO, "========= WAITING FOR All BACKEND %lu TO %lu ==========",
1104 static_cast<unsigned long>(start_index),
1105 static_cast<unsigned long>(stop_index));
1106 while (!SeenAllBackends(start_index, stop_index, rpc_options.service)) {
1107 SendRpcAndCount(&num_total, &num_ok, &num_failure, &num_drops,
1108 rpc_options);
1109 }
1110 if (wait_options.reset_counters) ResetBackendCounters();
1111 gpr_log(GPR_INFO,
1112 "Performed %d warm up requests against the backends. "
1113 "%d succeeded, %d failed, %d dropped.",
1114 num_total, num_ok, num_failure, num_drops);
1115 if (!wait_options.allow_failures) EXPECT_EQ(num_failure, 0);
1116 return std::make_tuple(num_ok, num_failure, num_drops);
1117 }
1118
WaitForBackend(size_t backend_idx,const WaitForBackendOptions & wait_options=WaitForBackendOptions (),const RpcOptions & rpc_options=RpcOptions ())1119 void WaitForBackend(
1120 size_t backend_idx,
1121 const WaitForBackendOptions& wait_options = WaitForBackendOptions(),
1122 const RpcOptions& rpc_options = RpcOptions()) {
1123 gpr_log(GPR_INFO, "========= WAITING FOR BACKEND %lu ==========",
1124 static_cast<unsigned long>(backend_idx));
1125 do {
1126 Status status = SendRpc(rpc_options);
1127 if (!wait_options.allow_failures) {
1128 EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
1129 << " message=" << status.error_message();
1130 }
1131 } while (!SeenBackend(backend_idx, rpc_options.service));
1132 if (wait_options.reset_counters) ResetBackendCounters();
1133 gpr_log(GPR_INFO, "========= BACKEND %lu READY ==========",
1134 static_cast<unsigned long>(backend_idx));
1135 }
1136
CreateAddressListFromPortList(const std::vector<int> & ports)1137 grpc_core::ServerAddressList CreateAddressListFromPortList(
1138 const std::vector<int>& ports) {
1139 grpc_core::ServerAddressList addresses;
1140 for (int port : ports) {
1141 absl::StatusOr<grpc_core::URI> lb_uri = grpc_core::URI::Parse(
1142 absl::StrCat(ipv6_only_ ? "ipv6:[::1]:" : "ipv4:127.0.0.1:", port));
1143 GPR_ASSERT(lb_uri.ok());
1144 grpc_resolved_address address;
1145 GPR_ASSERT(grpc_parse_uri(*lb_uri, &address));
1146 addresses.emplace_back(address.addr, address.len, nullptr);
1147 }
1148 return addresses;
1149 }
1150
CreateMetadataValueThatHashesToBackendPort(int port)1151 std::string CreateMetadataValueThatHashesToBackendPort(int port) {
1152 return absl::StrCat(ipv6_only_ ? "[::1]" : "127.0.0.1", ":", port, "_0");
1153 }
1154
CreateMetadataValueThatHashesToBackend(int index)1155 std::string CreateMetadataValueThatHashesToBackend(int index) {
1156 return CreateMetadataValueThatHashesToBackendPort(backends_[index]->port());
1157 }
1158
SetNextResolution(const std::vector<int> & ports,grpc_core::FakeResolverResponseGenerator * response_generator=nullptr)1159 void SetNextResolution(
1160 const std::vector<int>& ports,
1161 grpc_core::FakeResolverResponseGenerator* response_generator = nullptr) {
1162 if (!GetParam().use_fake_resolver()) return; // Not used with xds resolver.
1163 grpc_core::ExecCtx exec_ctx;
1164 grpc_core::Resolver::Result result;
1165 result.addresses = CreateAddressListFromPortList(ports);
1166 grpc_error_handle error = GRPC_ERROR_NONE;
1167 const char* service_config_json =
1168 GetParam().enable_load_reporting()
1169 ? kDefaultServiceConfig
1170 : kDefaultServiceConfigWithoutLoadReporting;
1171 result.service_config =
1172 grpc_core::ServiceConfig::Create(nullptr, service_config_json, &error);
1173 ASSERT_EQ(error, GRPC_ERROR_NONE) << grpc_error_std_string(error);
1174 ASSERT_NE(result.service_config.get(), nullptr);
1175 if (response_generator == nullptr) {
1176 response_generator = response_generator_.get();
1177 }
1178 response_generator->SetResponse(std::move(result));
1179 }
1180
SetNextResolutionForLbChannelAllBalancers(const char * service_config_json=nullptr,const char * expected_targets=nullptr,grpc_core::FakeResolverResponseGenerator * response_generator=nullptr)1181 void SetNextResolutionForLbChannelAllBalancers(
1182 const char* service_config_json = nullptr,
1183 const char* expected_targets = nullptr,
1184 grpc_core::FakeResolverResponseGenerator* response_generator = nullptr) {
1185 std::vector<int> ports;
1186 for (size_t i = 0; i < balancers_.size(); ++i) {
1187 ports.emplace_back(balancers_[i]->port());
1188 }
1189 SetNextResolutionForLbChannel(ports, service_config_json, expected_targets,
1190 response_generator);
1191 }
1192
SetNextResolutionForLbChannel(const std::vector<int> & ports,const char * service_config_json=nullptr,const char * expected_targets=nullptr,grpc_core::FakeResolverResponseGenerator * response_generator=nullptr)1193 void SetNextResolutionForLbChannel(
1194 const std::vector<int>& ports, const char* service_config_json = nullptr,
1195 const char* expected_targets = nullptr,
1196 grpc_core::FakeResolverResponseGenerator* response_generator = nullptr) {
1197 grpc_core::ExecCtx exec_ctx;
1198 grpc_core::Resolver::Result result;
1199 result.addresses = CreateAddressListFromPortList(ports);
1200 if (service_config_json != nullptr) {
1201 grpc_error_handle error = GRPC_ERROR_NONE;
1202 result.service_config = grpc_core::ServiceConfig::Create(
1203 nullptr, service_config_json, &error);
1204 ASSERT_NE(result.service_config.get(), nullptr);
1205 ASSERT_EQ(error, GRPC_ERROR_NONE) << grpc_error_std_string(error);
1206 }
1207 if (expected_targets != nullptr) {
1208 grpc_arg expected_targets_arg = grpc_channel_arg_string_create(
1209 const_cast<char*>(GRPC_ARG_FAKE_SECURITY_EXPECTED_TARGETS),
1210 const_cast<char*>(expected_targets));
1211 result.args =
1212 grpc_channel_args_copy_and_add(nullptr, &expected_targets_arg, 1);
1213 }
1214 if (response_generator == nullptr) {
1215 response_generator = lb_channel_response_generator_.get();
1216 }
1217 response_generator->SetResponse(std::move(result));
1218 }
1219
SetNextReresolutionResponse(const std::vector<int> & ports)1220 void SetNextReresolutionResponse(const std::vector<int>& ports) {
1221 grpc_core::ExecCtx exec_ctx;
1222 grpc_core::Resolver::Result result;
1223 result.addresses = CreateAddressListFromPortList(ports);
1224 response_generator_->SetReresolutionResponse(std::move(result));
1225 }
1226
GetBackendPorts(size_t start_index=0,size_t stop_index=0) const1227 std::vector<int> GetBackendPorts(size_t start_index = 0,
1228 size_t stop_index = 0) const {
1229 if (stop_index == 0) stop_index = backends_.size();
1230 std::vector<int> backend_ports;
1231 for (size_t i = start_index; i < stop_index; ++i) {
1232 backend_ports.push_back(backends_[i]->port());
1233 }
1234 return backend_ports;
1235 }
1236
SendRpc(const RpcOptions & rpc_options=RpcOptions (),EchoResponse * response=nullptr)1237 Status SendRpc(const RpcOptions& rpc_options = RpcOptions(),
1238 EchoResponse* response = nullptr) {
1239 const bool local_response = (response == nullptr);
1240 if (local_response) response = new EchoResponse;
1241 ClientContext context;
1242 EchoRequest request;
1243 if (rpc_options.server_expected_error != StatusCode::OK) {
1244 auto* error = request.mutable_param()->mutable_expected_error();
1245 error->set_code(rpc_options.server_expected_error);
1246 }
1247 rpc_options.SetupRpc(&context, &request);
1248 Status status;
1249 switch (rpc_options.service) {
1250 case SERVICE_ECHO:
1251 status =
1252 SendRpcMethod(&stub_, rpc_options, &context, request, response);
1253 break;
1254 case SERVICE_ECHO1:
1255 status =
1256 SendRpcMethod(&stub1_, rpc_options, &context, request, response);
1257 break;
1258 case SERVICE_ECHO2:
1259 status =
1260 SendRpcMethod(&stub2_, rpc_options, &context, request, response);
1261 break;
1262 }
1263 if (local_response) delete response;
1264 return status;
1265 }
1266
CheckRpcSendOk(const size_t times=1,const RpcOptions & rpc_options=RpcOptions ())1267 void CheckRpcSendOk(const size_t times = 1,
1268 const RpcOptions& rpc_options = RpcOptions()) {
1269 for (size_t i = 0; i < times; ++i) {
1270 EchoResponse response;
1271 const Status status = SendRpc(rpc_options, &response);
1272 EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
1273 << " message=" << status.error_message();
1274 EXPECT_EQ(response.message(), kRequestMessage);
1275 }
1276 }
1277
1278 struct CheckRpcSendFailureOptions {
__anond1b637be0302grpc::testing::__anond1b637be0111::XdsEnd2endTest::CheckRpcSendFailureOptions1279 std::function<bool(size_t)> continue_predicate = [](size_t i) {
1280 return i < 1;
1281 };
1282 RpcOptions rpc_options;
1283 StatusCode expected_error_code = StatusCode::OK;
1284
CheckRpcSendFailureOptionsgrpc::testing::__anond1b637be0111::XdsEnd2endTest::CheckRpcSendFailureOptions1285 CheckRpcSendFailureOptions() {}
1286
set_timesgrpc::testing::__anond1b637be0111::XdsEnd2endTest::CheckRpcSendFailureOptions1287 CheckRpcSendFailureOptions& set_times(size_t times) {
1288 continue_predicate = [times](size_t i) { return i < times; };
1289 return *this;
1290 }
1291
set_continue_predicategrpc::testing::__anond1b637be0111::XdsEnd2endTest::CheckRpcSendFailureOptions1292 CheckRpcSendFailureOptions& set_continue_predicate(
1293 std::function<bool(size_t)> pred) {
1294 continue_predicate = std::move(pred);
1295 return *this;
1296 }
1297
set_rpc_optionsgrpc::testing::__anond1b637be0111::XdsEnd2endTest::CheckRpcSendFailureOptions1298 CheckRpcSendFailureOptions& set_rpc_options(const RpcOptions& options) {
1299 rpc_options = options;
1300 return *this;
1301 }
1302
set_expected_error_codegrpc::testing::__anond1b637be0111::XdsEnd2endTest::CheckRpcSendFailureOptions1303 CheckRpcSendFailureOptions& set_expected_error_code(StatusCode code) {
1304 expected_error_code = code;
1305 return *this;
1306 }
1307 };
1308
CheckRpcSendFailure(const CheckRpcSendFailureOptions & options=CheckRpcSendFailureOptions ())1309 void CheckRpcSendFailure(const CheckRpcSendFailureOptions& options =
1310 CheckRpcSendFailureOptions()) {
1311 for (size_t i = 0; options.continue_predicate(i); ++i) {
1312 const Status status = SendRpc(options.rpc_options);
1313 EXPECT_FALSE(status.ok());
1314 if (options.expected_error_code != StatusCode::OK) {
1315 EXPECT_EQ(options.expected_error_code, status.error_code());
1316 }
1317 }
1318 }
1319
WaitForNack(std::function<AdsServiceImpl::ResponseState::State ()> get_state,StatusCode expected_status=StatusCode::UNAVAILABLE)1320 bool WaitForNack(
1321 std::function<AdsServiceImpl::ResponseState::State()> get_state,
1322 StatusCode expected_status = StatusCode::UNAVAILABLE) {
1323 auto deadline = absl::Now() + absl::Seconds(30);
1324 bool success = true;
1325 CheckRpcSendFailure(CheckRpcSendFailureOptions()
1326 .set_continue_predicate([&](size_t) {
1327 if (absl::Now() >= deadline) {
1328 success = false;
1329 return false;
1330 }
1331 return get_state() !=
1332 AdsServiceImpl::ResponseState::NACKED;
1333 })
1334 .set_expected_error_code(expected_status));
1335 return success;
1336 }
1337
WaitForLdsNack(StatusCode expected_status=StatusCode::UNAVAILABLE)1338 bool WaitForLdsNack(StatusCode expected_status = StatusCode::UNAVAILABLE) {
1339 return WaitForNack(
1340 [&]() {
1341 return balancers_[0]->ads_service()->lds_response_state().state;
1342 },
1343 expected_status);
1344 }
1345
WaitForRdsNack()1346 bool WaitForRdsNack() {
1347 return WaitForNack(
1348 [&]() { return RouteConfigurationResponseState(0).state; });
1349 }
1350
WaitForCdsNack()1351 bool WaitForCdsNack() {
1352 return WaitForNack([&]() {
1353 return balancers_[0]->ads_service()->cds_response_state().state;
1354 });
1355 }
1356
WaitForEdsNack()1357 bool WaitForEdsNack() {
1358 return WaitForNack([&]() {
1359 return balancers_[0]->ads_service()->eds_response_state().state;
1360 });
1361 }
1362
RouteConfigurationResponseState(int idx) const1363 AdsServiceImpl::ResponseState RouteConfigurationResponseState(int idx) const {
1364 AdsServiceImpl* ads_service = balancers_[idx]->ads_service();
1365 if (GetParam().enable_rds_testing()) {
1366 return ads_service->rds_response_state();
1367 }
1368 return ads_service->lds_response_state();
1369 }
1370
PopulateServerListenerNameAndPort(const Listener & listener_template,int port)1371 Listener PopulateServerListenerNameAndPort(const Listener& listener_template,
1372 int port) {
1373 Listener listener = listener_template;
1374 listener.set_name(
1375 absl::StrCat("grpc/server?xds.resource.listening_address=",
1376 ipv6_only_ ? "[::1]:" : "127.0.0.1:", port));
1377 listener.mutable_address()->mutable_socket_address()->set_port_value(port);
1378 return listener;
1379 }
1380
1381 // Interface for accessing HttpConnectionManager config in Listener.
1382 class HcmAccessor {
1383 public:
1384 virtual ~HcmAccessor() = default;
1385 virtual HttpConnectionManager Unpack(const Listener& listener) const = 0;
1386 virtual void Pack(const HttpConnectionManager& hcm,
1387 Listener* listener) const = 0;
1388 };
1389
1390 // Client-side impl.
1391 class ClientHcmAccessor : public HcmAccessor {
1392 public:
Unpack(const Listener & listener) const1393 HttpConnectionManager Unpack(const Listener& listener) const override {
1394 HttpConnectionManager http_connection_manager;
1395 listener.api_listener().api_listener().UnpackTo(&http_connection_manager);
1396 return http_connection_manager;
1397 }
Pack(const HttpConnectionManager & hcm,Listener * listener) const1398 void Pack(const HttpConnectionManager& hcm,
1399 Listener* listener) const override {
1400 auto* api_listener =
1401 listener->mutable_api_listener()->mutable_api_listener();
1402 api_listener->PackFrom(hcm);
1403 }
1404 };
1405
1406 // Server-side impl.
1407 class ServerHcmAccessor : public HcmAccessor {
1408 public:
Unpack(const Listener & listener) const1409 HttpConnectionManager Unpack(const Listener& listener) const override {
1410 HttpConnectionManager http_connection_manager;
1411 listener.default_filter_chain().filters().at(0).typed_config().UnpackTo(
1412 &http_connection_manager);
1413 return http_connection_manager;
1414 }
Pack(const HttpConnectionManager & hcm,Listener * listener) const1415 void Pack(const HttpConnectionManager& hcm,
1416 Listener* listener) const override {
1417 listener->mutable_default_filter_chain()
1418 ->mutable_filters()
1419 ->at(0)
1420 .mutable_typed_config()
1421 ->PackFrom(hcm);
1422 }
1423 };
1424
SetListenerAndRouteConfiguration(int idx,Listener listener,const RouteConfiguration & route_config,const HcmAccessor & hcm_accessor=ClientHcmAccessor ())1425 void SetListenerAndRouteConfiguration(
1426 int idx, Listener listener, const RouteConfiguration& route_config,
1427 const HcmAccessor& hcm_accessor = ClientHcmAccessor()) {
1428 HttpConnectionManager http_connection_manager =
1429 hcm_accessor.Unpack(listener);
1430 if (GetParam().enable_rds_testing()) {
1431 auto* rds = http_connection_manager.mutable_rds();
1432 rds->set_route_config_name(kDefaultRouteConfigurationName);
1433 rds->mutable_config_source()->mutable_ads();
1434 balancers_[idx]->ads_service()->SetRdsResource(route_config);
1435 } else {
1436 *http_connection_manager.mutable_route_config() = route_config;
1437 }
1438 hcm_accessor.Pack(http_connection_manager, &listener);
1439 balancers_[idx]->ads_service()->SetLdsResource(listener);
1440 }
1441
SetServerListenerNameAndRouteConfiguration(int idx,Listener listener,int port,const RouteConfiguration & route_config)1442 void SetServerListenerNameAndRouteConfiguration(
1443 int idx, Listener listener, int port,
1444 const RouteConfiguration& route_config) {
1445 SetListenerAndRouteConfiguration(
1446 idx, PopulateServerListenerNameAndPort(listener, port), route_config,
1447 ServerHcmAccessor());
1448 }
1449
SetRouteConfiguration(int idx,const RouteConfiguration & route_config,const Listener * listener_to_copy=nullptr)1450 void SetRouteConfiguration(int idx, const RouteConfiguration& route_config,
1451 const Listener* listener_to_copy = nullptr) {
1452 if (GetParam().enable_rds_testing()) {
1453 balancers_[idx]->ads_service()->SetRdsResource(route_config);
1454 } else {
1455 Listener listener(listener_to_copy == nullptr ? default_listener_
1456 : *listener_to_copy);
1457 HttpConnectionManager http_connection_manager;
1458 listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
1459 &http_connection_manager);
1460 *(http_connection_manager.mutable_route_config()) = route_config;
1461 listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
1462 http_connection_manager);
1463 balancers_[idx]->ads_service()->SetLdsResource(listener);
1464 }
1465 }
1466
1467 struct EdsResourceArgs {
1468 struct Endpoint {
Endpointgrpc::testing::__anond1b637be0111::XdsEnd2endTest::EdsResourceArgs::Endpoint1469 explicit Endpoint(int port,
1470 HealthStatus health_status = HealthStatus::UNKNOWN,
1471 int lb_weight = 1)
1472 : port(port), health_status(health_status), lb_weight(lb_weight) {}
1473
1474 int port;
1475 HealthStatus health_status;
1476 int lb_weight;
1477 };
1478
1479 struct Locality {
Localitygrpc::testing::__anond1b637be0111::XdsEnd2endTest::EdsResourceArgs::Locality1480 Locality(std::string sub_zone, std::vector<Endpoint> endpoints,
1481 int lb_weight = kDefaultLocalityWeight,
1482 int priority = kDefaultLocalityPriority)
1483 : sub_zone(std::move(sub_zone)),
1484 endpoints(std::move(endpoints)),
1485 lb_weight(lb_weight),
1486 priority(priority) {}
1487
1488 const std::string sub_zone;
1489 std::vector<Endpoint> endpoints;
1490 int lb_weight;
1491 int priority;
1492 };
1493
1494 EdsResourceArgs() = default;
EdsResourceArgsgrpc::testing::__anond1b637be0111::XdsEnd2endTest::EdsResourceArgs1495 explicit EdsResourceArgs(std::vector<Locality> locality_list)
1496 : locality_list(std::move(locality_list)) {}
1497
1498 std::vector<Locality> locality_list;
1499 std::map<std::string, uint32_t> drop_categories;
1500 FractionalPercent::DenominatorType drop_denominator =
1501 FractionalPercent::MILLION;
1502 };
1503
CreateEndpoint(size_t backend_idx,HealthStatus health_status=HealthStatus::UNKNOWN,int lb_weight=1)1504 EdsResourceArgs::Endpoint CreateEndpoint(
1505 size_t backend_idx, HealthStatus health_status = HealthStatus::UNKNOWN,
1506 int lb_weight = 1) {
1507 return EdsResourceArgs::Endpoint(backends_[backend_idx]->port(),
1508 health_status, lb_weight);
1509 }
1510
CreateEndpointsForBackends(size_t start_index=0,size_t stop_index=0,HealthStatus health_status=HealthStatus::UNKNOWN,int lb_weight=1)1511 std::vector<EdsResourceArgs::Endpoint> CreateEndpointsForBackends(
1512 size_t start_index = 0, size_t stop_index = 0,
1513 HealthStatus health_status = HealthStatus::UNKNOWN, int lb_weight = 1) {
1514 if (stop_index == 0) stop_index = backends_.size();
1515 std::vector<EdsResourceArgs::Endpoint> endpoints;
1516 for (size_t i = start_index; i < stop_index; ++i) {
1517 endpoints.emplace_back(CreateEndpoint(i, health_status, lb_weight));
1518 }
1519 return endpoints;
1520 }
1521
MakeNonExistantEndpoint()1522 EdsResourceArgs::Endpoint MakeNonExistantEndpoint() {
1523 return EdsResourceArgs::Endpoint(grpc_pick_unused_port_or_die());
1524 }
1525
BuildEdsResource(const EdsResourceArgs & args,const char * eds_service_name=kDefaultEdsServiceName)1526 ClusterLoadAssignment BuildEdsResource(
1527 const EdsResourceArgs& args,
1528 const char* eds_service_name = kDefaultEdsServiceName) {
1529 ClusterLoadAssignment assignment;
1530 assignment.set_cluster_name(eds_service_name);
1531 for (const auto& locality : args.locality_list) {
1532 auto* endpoints = assignment.add_endpoints();
1533 endpoints->mutable_load_balancing_weight()->set_value(locality.lb_weight);
1534 endpoints->set_priority(locality.priority);
1535 endpoints->mutable_locality()->set_region(kDefaultLocalityRegion);
1536 endpoints->mutable_locality()->set_zone(kDefaultLocalityZone);
1537 endpoints->mutable_locality()->set_sub_zone(locality.sub_zone);
1538 for (size_t i = 0; i < locality.endpoints.size(); ++i) {
1539 const int& port = locality.endpoints[i].port;
1540 auto* lb_endpoints = endpoints->add_lb_endpoints();
1541 if (locality.endpoints.size() > i &&
1542 locality.endpoints[i].health_status != HealthStatus::UNKNOWN) {
1543 lb_endpoints->set_health_status(locality.endpoints[i].health_status);
1544 }
1545 if (locality.endpoints.size() > i &&
1546 locality.endpoints[i].lb_weight >= 1) {
1547 lb_endpoints->mutable_load_balancing_weight()->set_value(
1548 locality.endpoints[i].lb_weight);
1549 }
1550 auto* endpoint = lb_endpoints->mutable_endpoint();
1551 auto* address = endpoint->mutable_address();
1552 auto* socket_address = address->mutable_socket_address();
1553 socket_address->set_address(ipv6_only_ ? "::1" : "127.0.0.1");
1554 socket_address->set_port_value(port);
1555 }
1556 }
1557 if (!args.drop_categories.empty()) {
1558 auto* policy = assignment.mutable_policy();
1559 for (const auto& p : args.drop_categories) {
1560 const std::string& name = p.first;
1561 const uint32_t parts_per_million = p.second;
1562 auto* drop_overload = policy->add_drop_overloads();
1563 drop_overload->set_category(name);
1564 auto* drop_percentage = drop_overload->mutable_drop_percentage();
1565 drop_percentage->set_numerator(parts_per_million);
1566 drop_percentage->set_denominator(args.drop_denominator);
1567 }
1568 }
1569 return assignment;
1570 }
1571
1572 public:
1573 // This method could benefit test subclasses; to make it accessible
1574 // via bind with a qualified name, it needs to be public.
SetEdsResourceWithDelay(size_t i,const ClusterLoadAssignment & assignment,int delay_ms)1575 void SetEdsResourceWithDelay(size_t i,
1576 const ClusterLoadAssignment& assignment,
1577 int delay_ms) {
1578 GPR_ASSERT(delay_ms > 0);
1579 gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(delay_ms));
1580 balancers_[i]->ads_service()->SetEdsResource(assignment);
1581 }
1582
1583 protected:
1584 class XdsServingStatusNotifier
1585 : public grpc::experimental::XdsServerServingStatusNotifierInterface {
1586 public:
OnServingStatusUpdate(std::string uri,ServingStatusUpdate update)1587 void OnServingStatusUpdate(std::string uri,
1588 ServingStatusUpdate update) override {
1589 grpc_core::MutexLock lock(&mu_);
1590 status_map[uri] = update.status;
1591 cond_.Signal();
1592 }
1593
WaitOnServingStatusChange(std::string uri,grpc::StatusCode expected_status)1594 void WaitOnServingStatusChange(std::string uri,
1595 grpc::StatusCode expected_status) {
1596 grpc_core::MutexLock lock(&mu_);
1597 std::map<std::string, grpc::Status>::iterator it;
1598 while ((it = status_map.find(uri)) == status_map.end() ||
1599 it->second.error_code() != expected_status) {
1600 cond_.Wait(&mu_);
1601 }
1602 }
1603
1604 private:
1605 grpc_core::Mutex mu_;
1606 grpc_core::CondVar cond_;
1607 std::map<std::string, grpc::Status> status_map ABSL_GUARDED_BY(mu_);
1608 };
1609
1610 class ServerThread {
1611 public:
ServerThread(XdsEnd2endTest * test_obj,bool use_xds_enabled_server=false)1612 explicit ServerThread(XdsEnd2endTest* test_obj,
1613 bool use_xds_enabled_server = false)
1614 : test_obj_(test_obj),
1615 port_(grpc_pick_unused_port_or_die()),
1616 use_xds_enabled_server_(use_xds_enabled_server) {}
~ServerThread()1617 virtual ~ServerThread(){};
1618
Start()1619 void Start() {
1620 gpr_log(GPR_INFO, "starting %s server on port %d", Type(), port_);
1621 GPR_ASSERT(!running_);
1622 running_ = true;
1623 StartAllServices();
1624 grpc_core::Mutex mu;
1625 // We need to acquire the lock here in order to prevent the notify_one
1626 // by ServerThread::Serve from firing before the wait below is hit.
1627 grpc_core::MutexLock lock(&mu);
1628 grpc_core::CondVar cond;
1629 thread_ = absl::make_unique<std::thread>(
1630 std::bind(&ServerThread::Serve, this, &mu, &cond));
1631 cond.Wait(&mu);
1632 gpr_log(GPR_INFO, "%s server startup complete", Type());
1633 }
1634
Serve(grpc_core::Mutex * mu,grpc_core::CondVar * cond)1635 void Serve(grpc_core::Mutex* mu, grpc_core::CondVar* cond) {
1636 // We need to acquire the lock here in order to prevent the notify_one
1637 // below from firing before its corresponding wait is executed.
1638 grpc_core::MutexLock lock(mu);
1639 std::ostringstream server_address;
1640 server_address << "localhost:" << port_;
1641 if (use_xds_enabled_server_) {
1642 XdsServerBuilder builder;
1643 if (GetParam().bootstrap_source() ==
1644 TestType::kBootstrapFromChannelArg) {
1645 builder.SetOption(
1646 absl::make_unique<XdsChannelArgsServerBuilderOption>(test_obj_));
1647 }
1648 builder.set_status_notifier(¬ifier_);
1649 builder.AddListeningPort(server_address.str(), Credentials());
1650 RegisterAllServices(&builder);
1651 server_ = builder.BuildAndStart();
1652 } else {
1653 ServerBuilder builder;
1654 builder.AddListeningPort(server_address.str(), Credentials());
1655 RegisterAllServices(&builder);
1656 server_ = builder.BuildAndStart();
1657 }
1658 cond->Signal();
1659 }
1660
Shutdown()1661 void Shutdown() {
1662 if (!running_) return;
1663 gpr_log(GPR_INFO, "%s about to shutdown", Type());
1664 ShutdownAllServices();
1665 server_->Shutdown(grpc_timeout_milliseconds_to_deadline(0));
1666 thread_->join();
1667 gpr_log(GPR_INFO, "%s shutdown completed", Type());
1668 running_ = false;
1669 }
1670
Credentials()1671 virtual std::shared_ptr<ServerCredentials> Credentials() {
1672 return std::make_shared<SecureServerCredentials>(
1673 grpc_fake_transport_security_server_credentials_create());
1674 }
1675
port() const1676 int port() const { return port_; }
1677
use_xds_enabled_server() const1678 bool use_xds_enabled_server() const { return use_xds_enabled_server_; }
1679
notifier()1680 XdsServingStatusNotifier* notifier() { return ¬ifier_; }
1681
1682 private:
1683 class XdsChannelArgsServerBuilderOption
1684 : public ::grpc::ServerBuilderOption {
1685 public:
XdsChannelArgsServerBuilderOption(XdsEnd2endTest * test_obj)1686 explicit XdsChannelArgsServerBuilderOption(XdsEnd2endTest* test_obj)
1687 : test_obj_(test_obj) {}
1688
UpdateArguments(grpc::ChannelArguments * args)1689 void UpdateArguments(grpc::ChannelArguments* args) override {
1690 args->SetString(
1691 GRPC_ARG_TEST_ONLY_DO_NOT_USE_IN_PROD_XDS_BOOTSTRAP_CONFIG,
1692 GetParam().use_v2() ? kBootstrapFileV2 : kBootstrapFileV3);
1693 args->SetPointerWithVtable(
1694 GRPC_ARG_TEST_ONLY_DO_NOT_USE_IN_PROD_XDS_CLIENT_CHANNEL_ARGS,
1695 &test_obj_->xds_channel_args_, &kChannelArgsArgVtable);
1696 }
1697
UpdatePlugins(std::vector<std::unique_ptr<grpc::ServerBuilderPlugin>> *)1698 void UpdatePlugins(
1699 std::vector<std::unique_ptr<grpc::ServerBuilderPlugin>>* /*plugins*/)
1700 override {}
1701
1702 private:
1703 XdsEnd2endTest* test_obj_;
1704 };
1705
1706 virtual void RegisterAllServices(ServerBuilder* builder) = 0;
1707 virtual void StartAllServices() = 0;
1708 virtual void ShutdownAllServices() = 0;
1709
1710 virtual const char* Type() = 0;
1711
1712 XdsEnd2endTest* test_obj_;
1713 const int port_;
1714 std::unique_ptr<Server> server_;
1715 XdsServingStatusNotifier notifier_;
1716 std::unique_ptr<std::thread> thread_;
1717 bool running_ = false;
1718 const bool use_xds_enabled_server_;
1719 };
1720
1721 class BackendServerThread : public ServerThread {
1722 public:
BackendServerThread(XdsEnd2endTest * test_obj,bool use_xds_enabled_server)1723 explicit BackendServerThread(XdsEnd2endTest* test_obj,
1724 bool use_xds_enabled_server)
1725 : ServerThread(test_obj, use_xds_enabled_server) {}
1726
1727 BackendServiceImpl<::grpc::testing::EchoTestService::Service>*
backend_service()1728 backend_service() {
1729 return &backend_service_;
1730 }
1731 BackendServiceImpl<::grpc::testing::EchoTest1Service::Service>*
backend_service1()1732 backend_service1() {
1733 return &backend_service1_;
1734 }
1735 BackendServiceImpl<::grpc::testing::EchoTest2Service::Service>*
backend_service2()1736 backend_service2() {
1737 return &backend_service2_;
1738 }
1739
Credentials()1740 std::shared_ptr<ServerCredentials> Credentials() override {
1741 if (GetParam().use_xds_credentials()) {
1742 if (use_xds_enabled_server()) {
1743 // We are testing server's use of XdsServerCredentials
1744 return XdsServerCredentials(InsecureServerCredentials());
1745 } else {
1746 // We are testing client's use of XdsCredentials
1747 std::string root_cert = ReadFile(kCaCertPath);
1748 std::string identity_cert = ReadFile(kServerCertPath);
1749 std::string private_key = ReadFile(kServerKeyPath);
1750 std::vector<experimental::IdentityKeyCertPair>
1751 identity_key_cert_pairs = {{private_key, identity_cert}};
1752 auto certificate_provider = std::make_shared<
1753 grpc::experimental::StaticDataCertificateProvider>(
1754 root_cert, identity_key_cert_pairs);
1755 grpc::experimental::TlsServerCredentialsOptions options(
1756 certificate_provider);
1757 options.watch_root_certs();
1758 options.watch_identity_key_cert_pairs();
1759 options.set_cert_request_type(
1760 GRPC_SSL_REQUEST_CLIENT_CERTIFICATE_AND_VERIFY);
1761 return grpc::experimental::TlsServerCredentials(options);
1762 }
1763 }
1764 return ServerThread::Credentials();
1765 }
1766
1767 private:
RegisterAllServices(ServerBuilder * builder)1768 void RegisterAllServices(ServerBuilder* builder) override {
1769 builder->RegisterService(&backend_service_);
1770 builder->RegisterService(&backend_service1_);
1771 builder->RegisterService(&backend_service2_);
1772 }
1773
StartAllServices()1774 void StartAllServices() override {
1775 backend_service_.Start();
1776 backend_service1_.Start();
1777 backend_service2_.Start();
1778 }
1779
ShutdownAllServices()1780 void ShutdownAllServices() override {
1781 backend_service_.Shutdown();
1782 backend_service1_.Shutdown();
1783 backend_service2_.Shutdown();
1784 }
1785
Type()1786 const char* Type() override { return "Backend"; }
1787
1788 BackendServiceImpl<::grpc::testing::EchoTestService::Service>
1789 backend_service_;
1790 BackendServiceImpl<::grpc::testing::EchoTest1Service::Service>
1791 backend_service1_;
1792 BackendServiceImpl<::grpc::testing::EchoTest2Service::Service>
1793 backend_service2_;
1794 };
1795
1796 class BalancerServerThread : public ServerThread {
1797 public:
BalancerServerThread(XdsEnd2endTest * test_obj,int client_load_reporting_interval=0)1798 explicit BalancerServerThread(XdsEnd2endTest* test_obj,
1799 int client_load_reporting_interval = 0)
1800 : ServerThread(test_obj),
1801 ads_service_(new AdsServiceImpl()),
1802 lrs_service_(new LrsServiceImpl(client_load_reporting_interval,
1803 {kDefaultClusterName})) {}
1804
ads_service()1805 AdsServiceImpl* ads_service() { return ads_service_.get(); }
lrs_service()1806 LrsServiceImpl* lrs_service() { return lrs_service_.get(); }
1807
1808 private:
RegisterAllServices(ServerBuilder * builder)1809 void RegisterAllServices(ServerBuilder* builder) override {
1810 builder->RegisterService(ads_service_->v2_rpc_service());
1811 builder->RegisterService(ads_service_->v3_rpc_service());
1812 builder->RegisterService(lrs_service_->v2_rpc_service());
1813 builder->RegisterService(lrs_service_->v3_rpc_service());
1814 }
1815
StartAllServices()1816 void StartAllServices() override {
1817 ads_service_->Start();
1818 lrs_service_->Start();
1819 }
1820
ShutdownAllServices()1821 void ShutdownAllServices() override {
1822 ads_service_->Shutdown();
1823 lrs_service_->Shutdown();
1824 }
1825
Type()1826 const char* Type() override { return "Balancer"; }
1827
1828 std::shared_ptr<AdsServiceImpl> ads_service_;
1829 std::shared_ptr<LrsServiceImpl> lrs_service_;
1830 };
1831
1832 #ifndef DISABLED_XDS_PROTO_IN_CC
1833 class AdminServerThread : public ServerThread {
1834 public:
AdminServerThread(XdsEnd2endTest * test_obj)1835 explicit AdminServerThread(XdsEnd2endTest* test_obj)
1836 : ServerThread(test_obj) {}
1837
1838 private:
RegisterAllServices(ServerBuilder * builder)1839 void RegisterAllServices(ServerBuilder* builder) override {
1840 builder->RegisterService(&csds_service_);
1841 }
StartAllServices()1842 void StartAllServices() override {}
ShutdownAllServices()1843 void ShutdownAllServices() override {}
1844
Type()1845 const char* Type() override { return "Admin"; }
1846
1847 grpc::xds::experimental::ClientStatusDiscoveryService csds_service_;
1848 };
1849 #endif // DISABLED_XDS_PROTO_IN_CC
1850
1851 class LongRunningRpc {
1852 public:
StartRpc(grpc::testing::EchoTestService::Stub * stub,const RpcOptions & rpc_options=RpcOptions ().set_timeout_ms (0).set_client_cancel_after_us (1* 1000* 1000))1853 void StartRpc(grpc::testing::EchoTestService::Stub* stub,
1854 const RpcOptions& rpc_options =
1855 RpcOptions().set_timeout_ms(0).set_client_cancel_after_us(
1856 1 * 1000 * 1000)) {
1857 sender_thread_ = std::thread([this, stub, rpc_options]() {
1858 EchoRequest request;
1859 EchoResponse response;
1860 rpc_options.SetupRpc(&context_, &request);
1861 status_ = stub->Echo(&context_, request, &response);
1862 });
1863 }
1864
CancelRpc()1865 void CancelRpc() {
1866 context_.TryCancel();
1867 if (sender_thread_.joinable()) sender_thread_.join();
1868 }
1869
GetStatus()1870 Status GetStatus() {
1871 if (sender_thread_.joinable()) sender_thread_.join();
1872 return status_;
1873 }
1874
1875 private:
1876 std::thread sender_thread_;
1877 ClientContext context_;
1878 Status status_;
1879 };
1880
1881 struct ConcurrentRpc {
1882 ClientContext context;
1883 Status status;
1884 grpc_millis elapsed_time;
1885 EchoResponse response;
1886 };
1887
SendConcurrentRpcs(grpc::testing::EchoTestService::Stub * stub,size_t num_rpcs,const RpcOptions & rpc_options)1888 std::vector<ConcurrentRpc> SendConcurrentRpcs(
1889 grpc::testing::EchoTestService::Stub* stub, size_t num_rpcs,
1890 const RpcOptions& rpc_options) {
1891 // Variables for RPCs.
1892 std::vector<ConcurrentRpc> rpcs(num_rpcs);
1893 EchoRequest request;
1894 // Variables for synchronization
1895 absl::Mutex mu;
1896 absl::CondVar cv;
1897 size_t completed = 0;
1898 // Set-off callback RPCs
1899 for (size_t i = 0; i < num_rpcs; i++) {
1900 ConcurrentRpc* rpc = &rpcs[i];
1901 rpc_options.SetupRpc(&rpc->context, &request);
1902 grpc_millis t0 = NowFromCycleCounter();
1903 stub->async()->Echo(&rpc->context, &request, &rpc->response,
1904 [rpc, &mu, &completed, &cv, num_rpcs, t0](Status s) {
1905 rpc->status = s;
1906 rpc->elapsed_time = NowFromCycleCounter() - t0;
1907 bool done;
1908 {
1909 absl::MutexLock lock(&mu);
1910 done = (++completed) == num_rpcs;
1911 }
1912 if (done) cv.Signal();
1913 });
1914 }
1915 {
1916 absl::MutexLock lock(&mu);
1917 cv.Wait(&mu);
1918 }
1919 EXPECT_EQ(completed, num_rpcs);
1920 return rpcs;
1921 }
1922
1923 const size_t num_backends_;
1924 const size_t num_balancers_;
1925 const int client_load_reporting_interval_seconds_;
1926 bool ipv6_only_ = false;
1927 std::shared_ptr<Channel> channel_;
1928 std::unique_ptr<grpc::testing::EchoTestService::Stub> stub_;
1929 std::unique_ptr<grpc::testing::EchoTest1Service::Stub> stub1_;
1930 std::unique_ptr<grpc::testing::EchoTest2Service::Stub> stub2_;
1931 std::vector<std::unique_ptr<BackendServerThread>> backends_;
1932 std::vector<std::unique_ptr<BalancerServerThread>> balancers_;
1933 grpc_core::RefCountedPtr<grpc_core::FakeResolverResponseGenerator>
1934 response_generator_;
1935 grpc_core::RefCountedPtr<grpc_core::FakeResolverResponseGenerator>
1936 lb_channel_response_generator_;
1937 grpc_core::RefCountedPtr<grpc_core::FakeResolverResponseGenerator>
1938 logical_dns_cluster_resolver_response_generator_;
1939 int xds_resource_does_not_exist_timeout_ms_ = 0;
1940 absl::InlinedVector<grpc_arg, 2> xds_channel_args_to_add_;
1941 grpc_channel_args xds_channel_args_;
1942
1943 Listener default_listener_;
1944 RouteConfiguration default_route_config_;
1945 Listener default_server_listener_;
1946 RouteConfiguration default_server_route_config_;
1947 Cluster default_cluster_;
1948 bool use_xds_enabled_server_;
1949 bool bootstrap_contents_from_env_var_;
1950 };
1951
1952 class BasicTest : public XdsEnd2endTest {
1953 public:
BasicTest()1954 BasicTest() : XdsEnd2endTest(4, 1) {}
1955 };
1956
1957 // Tests that the balancer sends the correct response to the client, and the
1958 // client sends RPCs to the backends using the default child policy.
TEST_P(BasicTest,Vanilla)1959 TEST_P(BasicTest, Vanilla) {
1960 SetNextResolution({});
1961 SetNextResolutionForLbChannelAllBalancers();
1962 const size_t kNumRpcsPerAddress = 100;
1963 EdsResourceArgs args({
1964 {"locality0", CreateEndpointsForBackends()},
1965 });
1966 balancers_[0]->ads_service()->SetEdsResource(
1967 BuildEdsResource(args, DefaultEdsServiceName()));
1968 // Make sure that trying to connect works without a call.
1969 channel_->GetState(true /* try_to_connect */);
1970 // We need to wait for all backends to come online.
1971 WaitForAllBackends();
1972 // Send kNumRpcsPerAddress RPCs per server.
1973 CheckRpcSendOk(kNumRpcsPerAddress * num_backends_);
1974 // Each backend should have gotten 100 requests.
1975 for (size_t i = 0; i < backends_.size(); ++i) {
1976 EXPECT_EQ(kNumRpcsPerAddress,
1977 backends_[i]->backend_service()->request_count());
1978 }
1979 // Check LB policy name for the channel.
1980 EXPECT_EQ(
1981 (GetParam().use_fake_resolver() ? "xds_cluster_resolver_experimental"
1982 : "xds_cluster_manager_experimental"),
1983 channel_->GetLoadBalancingPolicyName());
1984 }
1985
TEST_P(BasicTest,IgnoresUnhealthyEndpoints)1986 TEST_P(BasicTest, IgnoresUnhealthyEndpoints) {
1987 SetNextResolution({});
1988 SetNextResolutionForLbChannelAllBalancers();
1989 const size_t kNumRpcsPerAddress = 100;
1990 auto endpoints = CreateEndpointsForBackends();
1991 endpoints[0].health_status = HealthStatus::DRAINING;
1992 EdsResourceArgs args({
1993 {"locality0", std::move(endpoints), kDefaultLocalityWeight,
1994 kDefaultLocalityPriority},
1995 });
1996 balancers_[0]->ads_service()->SetEdsResource(
1997 BuildEdsResource(args, DefaultEdsServiceName()));
1998 // Make sure that trying to connect works without a call.
1999 channel_->GetState(true /* try_to_connect */);
2000 // We need to wait for all backends to come online.
2001 WaitForAllBackends(/*start_index=*/1);
2002 // Send kNumRpcsPerAddress RPCs per server.
2003 CheckRpcSendOk(kNumRpcsPerAddress * (num_backends_ - 1));
2004 // Each backend should have gotten 100 requests.
2005 for (size_t i = 1; i < backends_.size(); ++i) {
2006 EXPECT_EQ(kNumRpcsPerAddress,
2007 backends_[i]->backend_service()->request_count());
2008 }
2009 }
2010
2011 // Tests that subchannel sharing works when the same backend is listed
2012 // multiple times.
TEST_P(BasicTest,SameBackendListedMultipleTimes)2013 TEST_P(BasicTest, SameBackendListedMultipleTimes) {
2014 SetNextResolution({});
2015 SetNextResolutionForLbChannelAllBalancers();
2016 // Same backend listed twice.
2017 auto endpoints = CreateEndpointsForBackends(0, 1);
2018 endpoints.push_back(endpoints.front());
2019 EdsResourceArgs args({
2020 {"locality0", endpoints},
2021 });
2022 const size_t kNumRpcsPerAddress = 10;
2023 balancers_[0]->ads_service()->SetEdsResource(
2024 BuildEdsResource(args, DefaultEdsServiceName()));
2025 // We need to wait for the backend to come online.
2026 WaitForBackend(0);
2027 // Send kNumRpcsPerAddress RPCs per server.
2028 CheckRpcSendOk(kNumRpcsPerAddress * endpoints.size());
2029 // Backend should have gotten 20 requests.
2030 EXPECT_EQ(kNumRpcsPerAddress * endpoints.size(),
2031 backends_[0]->backend_service()->request_count());
2032 // And they should have come from a single client port, because of
2033 // subchannel sharing.
2034 EXPECT_EQ(1UL, backends_[0]->backend_service()->clients().size());
2035 }
2036
2037 // Tests that RPCs will be blocked until a non-empty serverlist is received.
TEST_P(BasicTest,InitiallyEmptyServerlist)2038 TEST_P(BasicTest, InitiallyEmptyServerlist) {
2039 SetNextResolution({});
2040 SetNextResolutionForLbChannelAllBalancers();
2041 const int kServerlistDelayMs = 500 * grpc_test_slowdown_factor();
2042 const int kCallDeadlineMs = kServerlistDelayMs * 2;
2043 // First response is an empty serverlist, sent right away.
2044 EdsResourceArgs::Locality empty_locality("locality0", {});
2045 EdsResourceArgs args({
2046 empty_locality,
2047 });
2048 balancers_[0]->ads_service()->SetEdsResource(
2049 BuildEdsResource(args, DefaultEdsServiceName()));
2050 // Send non-empty serverlist only after kServerlistDelayMs.
2051 args = EdsResourceArgs({
2052 {"locality0", CreateEndpointsForBackends()},
2053 });
2054 std::thread delayed_resource_setter(std::bind(
2055 &BasicTest::SetEdsResourceWithDelay, this, 0,
2056 BuildEdsResource(args, DefaultEdsServiceName()), kServerlistDelayMs));
2057 const auto t0 = system_clock::now();
2058 // Client will block: LB will initially send empty serverlist.
2059 CheckRpcSendOk(
2060 1, RpcOptions().set_timeout_ms(kCallDeadlineMs).set_wait_for_ready(true));
2061 const auto ellapsed_ms =
2062 std::chrono::duration_cast<std::chrono::milliseconds>(
2063 system_clock::now() - t0);
2064 // but eventually, the LB sends a serverlist update that allows the call to
2065 // proceed. The call delay must be larger than the delay in sending the
2066 // populated serverlist but under the call's deadline (which is enforced by
2067 // the call's deadline).
2068 EXPECT_GT(ellapsed_ms.count(), kServerlistDelayMs);
2069 delayed_resource_setter.join();
2070 }
2071
2072 // Tests that RPCs will fail with UNAVAILABLE instead of DEADLINE_EXCEEDED if
2073 // all the servers are unreachable.
TEST_P(BasicTest,AllServersUnreachableFailFast)2074 TEST_P(BasicTest, AllServersUnreachableFailFast) {
2075 // Set Rpc timeout to 5 seconds to ensure there is enough time
2076 // for communication with the xDS server to take place upon test start up.
2077 const uint32_t kRpcTimeoutMs = 5000;
2078 SetNextResolution({});
2079 SetNextResolutionForLbChannelAllBalancers();
2080 const size_t kNumUnreachableServers = 5;
2081 std::vector<EdsResourceArgs::Endpoint> endpoints;
2082 for (size_t i = 0; i < kNumUnreachableServers; ++i) {
2083 endpoints.emplace_back(grpc_pick_unused_port_or_die());
2084 }
2085 EdsResourceArgs args({
2086 {"locality0", endpoints},
2087 });
2088 balancers_[0]->ads_service()->SetEdsResource(
2089 BuildEdsResource(args, DefaultEdsServiceName()));
2090 const Status status = SendRpc(RpcOptions().set_timeout_ms(kRpcTimeoutMs));
2091 // The error shouldn't be DEADLINE_EXCEEDED because timeout is set to 5
2092 // seconds, and we should disocver in that time that the target backend is
2093 // down.
2094 EXPECT_EQ(StatusCode::UNAVAILABLE, status.error_code());
2095 }
2096
2097 // Tests that RPCs fail when the backends are down, and will succeed again
2098 // after the backends are restarted.
TEST_P(BasicTest,BackendsRestart)2099 TEST_P(BasicTest, BackendsRestart) {
2100 SetNextResolution({});
2101 SetNextResolutionForLbChannelAllBalancers();
2102 EdsResourceArgs args({
2103 {"locality0", CreateEndpointsForBackends()},
2104 });
2105 balancers_[0]->ads_service()->SetEdsResource(
2106 BuildEdsResource(args, DefaultEdsServiceName()));
2107 WaitForAllBackends();
2108 // Stop backends. RPCs should fail.
2109 ShutdownAllBackends();
2110 // Sending multiple failed requests instead of just one to ensure that the
2111 // client notices that all backends are down before we restart them. If we
2112 // didn't do this, then a single RPC could fail here due to the race
2113 // condition between the LB pick and the GOAWAY from the chosen backend
2114 // being shut down, which would not actually prove that the client noticed
2115 // that all of the backends are down. Then, when we send another request
2116 // below (which we expect to succeed), if the callbacks happen in the wrong
2117 // order, the same race condition could happen again due to the client not
2118 // yet having noticed that the backends were all down.
2119 CheckRpcSendFailure(CheckRpcSendFailureOptions().set_times(num_backends_));
2120 // Restart all backends. RPCs should start succeeding again.
2121 StartAllBackends();
2122 CheckRpcSendOk(1, RpcOptions().set_timeout_ms(2000).set_wait_for_ready(true));
2123 }
2124
TEST_P(BasicTest,IgnoresDuplicateUpdates)2125 TEST_P(BasicTest, IgnoresDuplicateUpdates) {
2126 const size_t kNumRpcsPerAddress = 100;
2127 SetNextResolution({});
2128 SetNextResolutionForLbChannelAllBalancers();
2129 EdsResourceArgs args({
2130 {"locality0", CreateEndpointsForBackends()},
2131 });
2132 balancers_[0]->ads_service()->SetEdsResource(
2133 BuildEdsResource(args, DefaultEdsServiceName()));
2134 // Wait for all backends to come online.
2135 WaitForAllBackends();
2136 // Send kNumRpcsPerAddress RPCs per server, but send an EDS update in
2137 // between. If the update is not ignored, this will cause the
2138 // round_robin policy to see an update, which will randomly reset its
2139 // position in the address list.
2140 for (size_t i = 0; i < kNumRpcsPerAddress; ++i) {
2141 CheckRpcSendOk(2);
2142 balancers_[0]->ads_service()->SetEdsResource(
2143 BuildEdsResource(args, DefaultEdsServiceName()));
2144 CheckRpcSendOk(2);
2145 }
2146 // Each backend should have gotten the right number of requests.
2147 for (size_t i = 1; i < backends_.size(); ++i) {
2148 EXPECT_EQ(kNumRpcsPerAddress,
2149 backends_[i]->backend_service()->request_count());
2150 }
2151 }
2152
2153 using XdsResolverOnlyTest = BasicTest;
2154
TEST_P(XdsResolverOnlyTest,ResourceTypeVersionPersistsAcrossStreamRestarts)2155 TEST_P(XdsResolverOnlyTest, ResourceTypeVersionPersistsAcrossStreamRestarts) {
2156 SetNextResolution({});
2157 SetNextResolutionForLbChannelAllBalancers();
2158 EdsResourceArgs args({
2159 {"locality0", CreateEndpointsForBackends(0, 1)},
2160 });
2161 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
2162 // Wait for backends to come online.
2163 WaitForAllBackends(0, 1);
2164 // Stop balancer.
2165 balancers_[0]->Shutdown();
2166 // Tell balancer to require minimum version 1 for all resource types.
2167 balancers_[0]->ads_service()->SetResourceMinVersion(kLdsTypeUrl, 1);
2168 balancers_[0]->ads_service()->SetResourceMinVersion(kRdsTypeUrl, 1);
2169 balancers_[0]->ads_service()->SetResourceMinVersion(kCdsTypeUrl, 1);
2170 balancers_[0]->ads_service()->SetResourceMinVersion(kEdsTypeUrl, 1);
2171 // Update backend, just so we can be sure that the client has
2172 // reconnected to the balancer.
2173 EdsResourceArgs args2({
2174 {"locality0", CreateEndpointsForBackends(1, 2)},
2175 });
2176 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args2));
2177 // Restart balancer.
2178 balancers_[0]->Start();
2179 // Make sure client has reconnected.
2180 WaitForAllBackends(1, 2);
2181 }
2182
2183 // Tests switching over from one cluster to another.
TEST_P(XdsResolverOnlyTest,ChangeClusters)2184 TEST_P(XdsResolverOnlyTest, ChangeClusters) {
2185 const char* kNewClusterName = "new_cluster_name";
2186 const char* kNewEdsServiceName = "new_eds_service_name";
2187 SetNextResolution({});
2188 SetNextResolutionForLbChannelAllBalancers();
2189 EdsResourceArgs args({
2190 {"locality0", CreateEndpointsForBackends(0, 2)},
2191 });
2192 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
2193 // We need to wait for all backends to come online.
2194 WaitForAllBackends(0, 2);
2195 // Populate new EDS resource.
2196 EdsResourceArgs args2({
2197 {"locality0", CreateEndpointsForBackends(2, 4)},
2198 });
2199 balancers_[0]->ads_service()->SetEdsResource(
2200 BuildEdsResource(args2, kNewEdsServiceName));
2201 // Populate new CDS resource.
2202 Cluster new_cluster = default_cluster_;
2203 new_cluster.set_name(kNewClusterName);
2204 new_cluster.mutable_eds_cluster_config()->set_service_name(
2205 kNewEdsServiceName);
2206 balancers_[0]->ads_service()->SetCdsResource(new_cluster);
2207 // Change RDS resource to point to new cluster.
2208 RouteConfiguration new_route_config = default_route_config_;
2209 new_route_config.mutable_virtual_hosts(0)
2210 ->mutable_routes(0)
2211 ->mutable_route()
2212 ->set_cluster(kNewClusterName);
2213 SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
2214 // Wait for all new backends to be used.
2215 std::tuple<int, int, int> counts = WaitForAllBackends(2, 4);
2216 // Make sure no RPCs failed in the transition.
2217 EXPECT_EQ(0, std::get<1>(counts));
2218 }
2219
2220 // Tests that we go into TRANSIENT_FAILURE if the Cluster disappears.
TEST_P(XdsResolverOnlyTest,ClusterRemoved)2221 TEST_P(XdsResolverOnlyTest, ClusterRemoved) {
2222 SetNextResolution({});
2223 SetNextResolutionForLbChannelAllBalancers();
2224 EdsResourceArgs args({
2225 {"locality0", CreateEndpointsForBackends()},
2226 });
2227 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
2228 // We need to wait for all backends to come online.
2229 WaitForAllBackends();
2230 // Unset CDS resource.
2231 balancers_[0]->ads_service()->UnsetResource(kCdsTypeUrl, kDefaultClusterName);
2232 // Wait for RPCs to start failing.
2233 do {
2234 } while (SendRpc(RpcOptions(), nullptr).ok());
2235 // Make sure RPCs are still failing.
2236 CheckRpcSendFailure(CheckRpcSendFailureOptions().set_times(1000));
2237 // Make sure we ACK'ed the update.
2238 EXPECT_EQ(balancers_[0]->ads_service()->cds_response_state().state,
2239 AdsServiceImpl::ResponseState::ACKED);
2240 }
2241
2242 // Tests that we restart all xDS requests when we reestablish the ADS call.
TEST_P(XdsResolverOnlyTest,RestartsRequestsUponReconnection)2243 TEST_P(XdsResolverOnlyTest, RestartsRequestsUponReconnection) {
2244 // Manually configure use of RDS.
2245 auto listener = default_listener_;
2246 HttpConnectionManager http_connection_manager;
2247 listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
2248 &http_connection_manager);
2249 auto* rds = http_connection_manager.mutable_rds();
2250 rds->set_route_config_name(kDefaultRouteConfigurationName);
2251 rds->mutable_config_source()->mutable_ads();
2252 listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
2253 http_connection_manager);
2254 balancers_[0]->ads_service()->SetLdsResource(listener);
2255 balancers_[0]->ads_service()->SetRdsResource(default_route_config_);
2256 const char* kNewClusterName = "new_cluster_name";
2257 const char* kNewEdsServiceName = "new_eds_service_name";
2258 SetNextResolution({});
2259 SetNextResolutionForLbChannelAllBalancers();
2260 EdsResourceArgs args({
2261 {"locality0", CreateEndpointsForBackends(0, 2)},
2262 });
2263 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
2264 // We need to wait for all backends to come online.
2265 WaitForAllBackends(0, 2);
2266 // Now shut down and restart the balancer. When the client
2267 // reconnects, it should automatically restart the requests for all
2268 // resource types.
2269 balancers_[0]->Shutdown();
2270 balancers_[0]->Start();
2271 // Make sure things are still working.
2272 CheckRpcSendOk(100);
2273 // Populate new EDS resource.
2274 EdsResourceArgs args2({
2275 {"locality0", CreateEndpointsForBackends(2, 4)},
2276 });
2277 balancers_[0]->ads_service()->SetEdsResource(
2278 BuildEdsResource(args2, kNewEdsServiceName));
2279 // Populate new CDS resource.
2280 Cluster new_cluster = default_cluster_;
2281 new_cluster.set_name(kNewClusterName);
2282 new_cluster.mutable_eds_cluster_config()->set_service_name(
2283 kNewEdsServiceName);
2284 balancers_[0]->ads_service()->SetCdsResource(new_cluster);
2285 // Change RDS resource to point to new cluster.
2286 RouteConfiguration new_route_config = default_route_config_;
2287 new_route_config.mutable_virtual_hosts(0)
2288 ->mutable_routes(0)
2289 ->mutable_route()
2290 ->set_cluster(kNewClusterName);
2291 balancers_[0]->ads_service()->SetRdsResource(new_route_config);
2292 // Wait for all new backends to be used.
2293 std::tuple<int, int, int> counts = WaitForAllBackends(2, 4);
2294 // Make sure no RPCs failed in the transition.
2295 EXPECT_EQ(0, std::get<1>(counts));
2296 }
2297
TEST_P(XdsResolverOnlyTest,DefaultRouteSpecifiesSlashPrefix)2298 TEST_P(XdsResolverOnlyTest, DefaultRouteSpecifiesSlashPrefix) {
2299 RouteConfiguration route_config = default_route_config_;
2300 route_config.mutable_virtual_hosts(0)
2301 ->mutable_routes(0)
2302 ->mutable_match()
2303 ->set_prefix("/");
2304 SetListenerAndRouteConfiguration(0, default_listener_, route_config);
2305 SetNextResolution({});
2306 SetNextResolutionForLbChannelAllBalancers();
2307 EdsResourceArgs args({
2308 {"locality0", CreateEndpointsForBackends()},
2309 });
2310 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
2311 // We need to wait for all backends to come online.
2312 WaitForAllBackends();
2313 }
2314
TEST_P(XdsResolverOnlyTest,CircuitBreaking)2315 TEST_P(XdsResolverOnlyTest, CircuitBreaking) {
2316 constexpr size_t kMaxConcurrentRequests = 10;
2317 SetNextResolution({});
2318 SetNextResolutionForLbChannelAllBalancers();
2319 // Populate new EDS resources.
2320 EdsResourceArgs args({
2321 {"locality0", CreateEndpointsForBackends(0, 1)},
2322 });
2323 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
2324 // Update CDS resource to set max concurrent request.
2325 CircuitBreakers circuit_breaks;
2326 Cluster cluster = default_cluster_;
2327 auto* threshold = cluster.mutable_circuit_breakers()->add_thresholds();
2328 threshold->set_priority(RoutingPriority::DEFAULT);
2329 threshold->mutable_max_requests()->set_value(kMaxConcurrentRequests);
2330 balancers_[0]->ads_service()->SetCdsResource(cluster);
2331 // Send exactly max_concurrent_requests long RPCs.
2332 LongRunningRpc rpcs[kMaxConcurrentRequests];
2333 for (size_t i = 0; i < kMaxConcurrentRequests; ++i) {
2334 rpcs[i].StartRpc(stub_.get());
2335 }
2336 // Wait for all RPCs to be in flight.
2337 while (backends_[0]->backend_service()->RpcsWaitingForClientCancel() <
2338 kMaxConcurrentRequests) {
2339 gpr_sleep_until(gpr_time_add(gpr_now(GPR_CLOCK_REALTIME),
2340 gpr_time_from_micros(1 * 1000, GPR_TIMESPAN)));
2341 }
2342 // Sending a RPC now should fail, the error message should tell us
2343 // we hit the max concurrent requests limit and got dropped.
2344 Status status = SendRpc();
2345 EXPECT_FALSE(status.ok());
2346 EXPECT_EQ(status.error_message(), "circuit breaker drop");
2347 // Cancel one RPC to allow another one through
2348 rpcs[0].CancelRpc();
2349 status = SendRpc();
2350 EXPECT_TRUE(status.ok());
2351 for (size_t i = 1; i < kMaxConcurrentRequests; ++i) {
2352 rpcs[i].CancelRpc();
2353 }
2354 // Make sure RPCs go to the correct backend:
2355 EXPECT_EQ(kMaxConcurrentRequests + 1,
2356 backends_[0]->backend_service()->request_count());
2357 }
2358
TEST_P(XdsResolverOnlyTest,CircuitBreakingMultipleChannelsShareCallCounter)2359 TEST_P(XdsResolverOnlyTest, CircuitBreakingMultipleChannelsShareCallCounter) {
2360 constexpr size_t kMaxConcurrentRequests = 10;
2361 // Populate new EDS resources.
2362 EdsResourceArgs args({
2363 {"locality0", CreateEndpointsForBackends(0, 1)},
2364 });
2365 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
2366 // Update CDS resource to set max concurrent request.
2367 CircuitBreakers circuit_breaks;
2368 Cluster cluster = default_cluster_;
2369 auto* threshold = cluster.mutable_circuit_breakers()->add_thresholds();
2370 threshold->set_priority(RoutingPriority::DEFAULT);
2371 threshold->mutable_max_requests()->set_value(kMaxConcurrentRequests);
2372 balancers_[0]->ads_service()->SetCdsResource(cluster);
2373 // Create second channel.
2374 auto response_generator2 =
2375 grpc_core::MakeRefCounted<grpc_core::FakeResolverResponseGenerator>();
2376 auto lb_response_generator2 =
2377 grpc_core::MakeRefCounted<grpc_core::FakeResolverResponseGenerator>();
2378 grpc_arg xds_arg = grpc_core::FakeResolverResponseGenerator::MakeChannelArg(
2379 lb_response_generator2.get());
2380 grpc_channel_args xds_channel_args2 = {1, &xds_arg};
2381 auto channel2 = CreateChannel(
2382 /*failover_timeout=*/0, /*server_name=*/kServerName,
2383 response_generator2.get(), &xds_channel_args2);
2384 auto stub2 = grpc::testing::EchoTestService::NewStub(channel2);
2385 // Set resolution results for both channels and for the xDS channel.
2386 SetNextResolution({});
2387 SetNextResolution({}, response_generator2.get());
2388 SetNextResolutionForLbChannelAllBalancers();
2389 SetNextResolutionForLbChannelAllBalancers(nullptr, nullptr,
2390 lb_response_generator2.get());
2391 // Send exactly max_concurrent_requests long RPCs, alternating between
2392 // the two channels.
2393 LongRunningRpc rpcs[kMaxConcurrentRequests];
2394 for (size_t i = 0; i < kMaxConcurrentRequests; ++i) {
2395 rpcs[i].StartRpc(i % 2 == 0 ? stub_.get() : stub2.get());
2396 }
2397 // Wait for all RPCs to be in flight.
2398 while (backends_[0]->backend_service()->RpcsWaitingForClientCancel() <
2399 kMaxConcurrentRequests) {
2400 gpr_sleep_until(gpr_time_add(gpr_now(GPR_CLOCK_REALTIME),
2401 gpr_time_from_micros(1 * 1000, GPR_TIMESPAN)));
2402 }
2403 // Sending a RPC now should fail, the error message should tell us
2404 // we hit the max concurrent requests limit and got dropped.
2405 Status status = SendRpc();
2406 EXPECT_FALSE(status.ok());
2407 EXPECT_EQ(status.error_message(), "circuit breaker drop");
2408 // Cancel one RPC to allow another one through
2409 rpcs[0].CancelRpc();
2410 status = SendRpc();
2411 EXPECT_TRUE(status.ok());
2412 for (size_t i = 1; i < kMaxConcurrentRequests; ++i) {
2413 rpcs[i].CancelRpc();
2414 }
2415 // Make sure RPCs go to the correct backend:
2416 EXPECT_EQ(kMaxConcurrentRequests + 1,
2417 backends_[0]->backend_service()->request_count());
2418 }
2419
TEST_P(XdsResolverOnlyTest,ClusterChangeAfterAdsCallFails)2420 TEST_P(XdsResolverOnlyTest, ClusterChangeAfterAdsCallFails) {
2421 const char* kNewEdsResourceName = "new_eds_resource_name";
2422 // Populate EDS resources.
2423 EdsResourceArgs args({
2424 {"locality0", CreateEndpointsForBackends(0, 1)},
2425 });
2426 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
2427 SetNextResolutionForLbChannelAllBalancers();
2428 // Check that the channel is working.
2429 CheckRpcSendOk();
2430 // Stop and restart the balancer.
2431 balancers_[0]->Shutdown();
2432 balancers_[0]->Start();
2433 // Create new EDS resource.
2434 EdsResourceArgs args2({
2435 {"locality0", CreateEndpointsForBackends(1, 2)},
2436 });
2437 balancers_[0]->ads_service()->SetEdsResource(
2438 BuildEdsResource(args2, kNewEdsResourceName));
2439 // Change CDS resource to point to new EDS resource.
2440 auto cluster = default_cluster_;
2441 cluster.mutable_eds_cluster_config()->set_service_name(kNewEdsResourceName);
2442 balancers_[0]->ads_service()->SetCdsResource(cluster);
2443 // Make sure client sees the change.
2444 // TODO(roth): This should not be allowing errors. The errors are
2445 // being caused by a bug that triggers in the following situation:
2446 //
2447 // 1. xDS call fails.
2448 // 2. When xDS call is restarted, the server sends the updated CDS
2449 // resource that points to the new EDS resource name.
2450 // 3. When the client receives the CDS update, it does two things:
2451 // - Sends the update to the CDS LB policy, which creates a new
2452 // xds_cluster_resolver policy using the new EDS service name.
2453 // - Notices that the CDS update no longer refers to the old EDS
2454 // service name, so removes that resource, notifying the old
2455 // xds_cluster_resolver policy that the resource no longer exists.
2456 //
2457 // Need to figure out a way to fix this bug, and then change this to
2458 // not allow failures.
2459 WaitForBackend(1, WaitForBackendOptions().set_allow_failures(true));
2460 }
2461
2462 using GlobalXdsClientTest = BasicTest;
2463
TEST_P(GlobalXdsClientTest,MultipleChannelsShareXdsClient)2464 TEST_P(GlobalXdsClientTest, MultipleChannelsShareXdsClient) {
2465 const char* kNewServerName = "new-server.example.com";
2466 Listener listener = default_listener_;
2467 listener.set_name(kNewServerName);
2468 SetListenerAndRouteConfiguration(0, listener, default_route_config_);
2469 SetNextResolution({});
2470 SetNextResolutionForLbChannelAllBalancers();
2471 EdsResourceArgs args({
2472 {"locality0", CreateEndpointsForBackends()},
2473 });
2474 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
2475 WaitForAllBackends();
2476 // Create second channel and tell it to connect to kNewServerName.
2477 auto channel2 = CreateChannel(/*failover_timeout=*/0, kNewServerName);
2478 channel2->GetState(/*try_to_connect=*/true);
2479 ASSERT_TRUE(
2480 channel2->WaitForConnected(grpc_timeout_milliseconds_to_deadline(100)));
2481 // Make sure there's only one client connected.
2482 EXPECT_EQ(1UL, balancers_[0]->ads_service()->clients().size());
2483 }
2484
2485 // Tests that the NACK for multiple bad LDS resources includes both errors.
TEST_P(GlobalXdsClientTest,MultipleBadResources)2486 TEST_P(GlobalXdsClientTest, MultipleBadResources) {
2487 constexpr char kServerName2[] = "server.other.com";
2488 constexpr char kServerName3[] = "server.another.com";
2489 auto listener = default_listener_;
2490 listener.clear_api_listener();
2491 balancers_[0]->ads_service()->SetLdsResource(listener);
2492 listener.set_name(kServerName2);
2493 balancers_[0]->ads_service()->SetLdsResource(listener);
2494 listener = default_listener_;
2495 listener.set_name(kServerName3);
2496 SetListenerAndRouteConfiguration(0, listener, default_route_config_);
2497 EdsResourceArgs args({
2498 {"locality0", CreateEndpointsForBackends(0, 1)},
2499 });
2500 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
2501 SetNextResolutionForLbChannelAllBalancers();
2502 CheckRpcSendFailure();
2503 // Need to create a second channel to subscribe to a second LDS resource.
2504 auto channel2 = CreateChannel(0, kServerName2);
2505 auto stub2 = grpc::testing::EchoTestService::NewStub(channel2);
2506 {
2507 ClientContext context;
2508 EchoRequest request;
2509 request.set_message(kRequestMessage);
2510 EchoResponse response;
2511 grpc::Status status = stub2->Echo(&context, request, &response);
2512 EXPECT_FALSE(status.ok());
2513 // Wait for second NACK to be reported to xDS server.
2514 auto deadline = absl::Now() + absl::Seconds(30);
2515 bool timed_out = false;
2516 CheckRpcSendFailure(
2517 CheckRpcSendFailureOptions().set_continue_predicate([&](size_t) {
2518 if (absl::Now() >= deadline) {
2519 timed_out = true;
2520 return false;
2521 }
2522 const auto response_state =
2523 balancers_[0]->ads_service()->lds_response_state();
2524 return response_state.state !=
2525 AdsServiceImpl::ResponseState::NACKED ||
2526 ::testing::Matches(::testing::ContainsRegex(absl::StrCat(
2527 kServerName,
2528 ": validation error.*"
2529 "Listener has neither address nor ApiListener.*",
2530 kServerName2,
2531 ": validation error.*"
2532 "Listener has neither address nor ApiListener")))(
2533 response_state.error_message);
2534 }));
2535 ASSERT_FALSE(timed_out);
2536 }
2537 // Now start a new channel with a third server name, this one with a
2538 // valid resource.
2539 auto channel3 = CreateChannel(0, kServerName3);
2540 auto stub3 = grpc::testing::EchoTestService::NewStub(channel3);
2541 {
2542 ClientContext context;
2543 EchoRequest request;
2544 request.set_message(kRequestMessage);
2545 EchoResponse response;
2546 grpc::Status status = stub3->Echo(&context, request, &response);
2547 EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
2548 << " message=" << status.error_message();
2549 }
2550 }
2551
2552 // Tests that we don't trigger does-not-exist callbacks for a resource
2553 // that was previously valid but is updated to be invalid.
TEST_P(GlobalXdsClientTest,InvalidListenerStillExistsIfPreviouslyCached)2554 TEST_P(GlobalXdsClientTest, InvalidListenerStillExistsIfPreviouslyCached) {
2555 // Set up valid resources and check that the channel works.
2556 EdsResourceArgs args({
2557 {"locality0", CreateEndpointsForBackends(0, 1)},
2558 });
2559 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
2560 SetNextResolutionForLbChannelAllBalancers();
2561 CheckRpcSendOk();
2562 // Now send an update changing the Listener to be invalid.
2563 auto listener = default_listener_;
2564 listener.clear_api_listener();
2565 balancers_[0]->ads_service()->SetLdsResource(listener);
2566 // Wait for xDS server to see NACK.
2567 auto deadline = absl::Now() + absl::Seconds(30);
2568 do {
2569 CheckRpcSendOk();
2570 ASSERT_LT(absl::Now(), deadline);
2571 } while (balancers_[0]->ads_service()->lds_response_state().state !=
2572 AdsServiceImpl::ResponseState::NACKED);
2573 EXPECT_THAT(balancers_[0]->ads_service()->lds_response_state().error_message,
2574 ::testing::ContainsRegex(absl::StrCat(
2575 kServerName,
2576 ": validation error.*"
2577 "Listener has neither address nor ApiListener")));
2578 // Check one more time, just to make sure it still works after NACK.
2579 CheckRpcSendOk();
2580 }
2581
2582 class XdsResolverLoadReportingOnlyTest : public XdsEnd2endTest {
2583 public:
XdsResolverLoadReportingOnlyTest()2584 XdsResolverLoadReportingOnlyTest() : XdsEnd2endTest(4, 1, 3) {}
2585 };
2586
2587 // Tests load reporting when switching over from one cluster to another.
TEST_P(XdsResolverLoadReportingOnlyTest,ChangeClusters)2588 TEST_P(XdsResolverLoadReportingOnlyTest, ChangeClusters) {
2589 const char* kNewClusterName = "new_cluster_name";
2590 const char* kNewEdsServiceName = "new_eds_service_name";
2591 balancers_[0]->lrs_service()->set_cluster_names(
2592 {kDefaultClusterName, kNewClusterName});
2593 SetNextResolution({});
2594 SetNextResolutionForLbChannelAllBalancers();
2595 // cluster kDefaultClusterName -> locality0 -> backends 0 and 1
2596 EdsResourceArgs args({
2597 {"locality0", CreateEndpointsForBackends(0, 2)},
2598 });
2599 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
2600 // cluster kNewClusterName -> locality1 -> backends 2 and 3
2601 EdsResourceArgs args2({
2602 {"locality1", CreateEndpointsForBackends(2, 4)},
2603 });
2604 balancers_[0]->ads_service()->SetEdsResource(
2605 BuildEdsResource(args2, kNewEdsServiceName));
2606 // CDS resource for kNewClusterName.
2607 Cluster new_cluster = default_cluster_;
2608 new_cluster.set_name(kNewClusterName);
2609 new_cluster.mutable_eds_cluster_config()->set_service_name(
2610 kNewEdsServiceName);
2611 balancers_[0]->ads_service()->SetCdsResource(new_cluster);
2612 // Wait for all backends to come online.
2613 int num_ok = 0;
2614 int num_failure = 0;
2615 int num_drops = 0;
2616 std::tie(num_ok, num_failure, num_drops) = WaitForAllBackends(0, 2);
2617 // The load report received at the balancer should be correct.
2618 std::vector<ClientStats> load_report =
2619 balancers_[0]->lrs_service()->WaitForLoadReport();
2620 EXPECT_THAT(
2621 load_report,
2622 ::testing::ElementsAre(::testing::AllOf(
2623 ::testing::Property(&ClientStats::cluster_name, kDefaultClusterName),
2624 ::testing::Property(
2625 &ClientStats::locality_stats,
2626 ::testing::ElementsAre(::testing::Pair(
2627 "locality0",
2628 ::testing::AllOf(
2629 ::testing::Field(&ClientStats::LocalityStats::
2630 total_successful_requests,
2631 num_ok),
2632 ::testing::Field(&ClientStats::LocalityStats::
2633 total_requests_in_progress,
2634 0UL),
2635 ::testing::Field(
2636 &ClientStats::LocalityStats::total_error_requests,
2637 num_failure),
2638 ::testing::Field(
2639 &ClientStats::LocalityStats::total_issued_requests,
2640 num_failure + num_ok))))),
2641 ::testing::Property(&ClientStats::total_dropped_requests,
2642 num_drops))));
2643 // Change RDS resource to point to new cluster.
2644 RouteConfiguration new_route_config = default_route_config_;
2645 new_route_config.mutable_virtual_hosts(0)
2646 ->mutable_routes(0)
2647 ->mutable_route()
2648 ->set_cluster(kNewClusterName);
2649 SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
2650 // Wait for all new backends to be used.
2651 std::tie(num_ok, num_failure, num_drops) = WaitForAllBackends(2, 4);
2652 // The load report received at the balancer should be correct.
2653 load_report = balancers_[0]->lrs_service()->WaitForLoadReport();
2654 EXPECT_THAT(
2655 load_report,
2656 ::testing::ElementsAre(
2657 ::testing::AllOf(
2658 ::testing::Property(&ClientStats::cluster_name,
2659 kDefaultClusterName),
2660 ::testing::Property(
2661 &ClientStats::locality_stats,
2662 ::testing::ElementsAre(::testing::Pair(
2663 "locality0",
2664 ::testing::AllOf(
2665 ::testing::Field(&ClientStats::LocalityStats::
2666 total_successful_requests,
2667 ::testing::Lt(num_ok)),
2668 ::testing::Field(&ClientStats::LocalityStats::
2669 total_requests_in_progress,
2670 0UL),
2671 ::testing::Field(
2672 &ClientStats::LocalityStats::total_error_requests,
2673 ::testing::Le(num_failure)),
2674 ::testing::Field(
2675 &ClientStats::LocalityStats::
2676 total_issued_requests,
2677 ::testing::Le(num_failure + num_ok)))))),
2678 ::testing::Property(&ClientStats::total_dropped_requests,
2679 num_drops)),
2680 ::testing::AllOf(
2681 ::testing::Property(&ClientStats::cluster_name, kNewClusterName),
2682 ::testing::Property(
2683 &ClientStats::locality_stats,
2684 ::testing::ElementsAre(::testing::Pair(
2685 "locality1",
2686 ::testing::AllOf(
2687 ::testing::Field(&ClientStats::LocalityStats::
2688 total_successful_requests,
2689 ::testing::Le(num_ok)),
2690 ::testing::Field(&ClientStats::LocalityStats::
2691 total_requests_in_progress,
2692 0UL),
2693 ::testing::Field(
2694 &ClientStats::LocalityStats::total_error_requests,
2695 ::testing::Le(num_failure)),
2696 ::testing::Field(
2697 &ClientStats::LocalityStats::
2698 total_issued_requests,
2699 ::testing::Le(num_failure + num_ok)))))),
2700 ::testing::Property(&ClientStats::total_dropped_requests,
2701 num_drops))));
2702 int total_ok = 0;
2703 int total_failure = 0;
2704 for (const ClientStats& client_stats : load_report) {
2705 total_ok += client_stats.total_successful_requests();
2706 total_failure += client_stats.total_error_requests();
2707 }
2708 EXPECT_EQ(total_ok, num_ok);
2709 EXPECT_EQ(total_failure, num_failure);
2710 // The LRS service got a single request, and sent a single response.
2711 EXPECT_EQ(1U, balancers_[0]->lrs_service()->request_count());
2712 EXPECT_EQ(1U, balancers_[0]->lrs_service()->response_count());
2713 }
2714
2715 using SecureNamingTest = BasicTest;
2716
2717 // Tests that secure naming check passes if target name is expected.
TEST_P(SecureNamingTest,TargetNameIsExpected)2718 TEST_P(SecureNamingTest, TargetNameIsExpected) {
2719 SetNextResolution({});
2720 SetNextResolutionForLbChannel({balancers_[0]->port()}, nullptr, "xds_server");
2721 EdsResourceArgs args({
2722 {"locality0", CreateEndpointsForBackends()},
2723 });
2724 balancers_[0]->ads_service()->SetEdsResource(
2725 BuildEdsResource(args, DefaultEdsServiceName()));
2726 CheckRpcSendOk();
2727 }
2728
2729 // Tests that secure naming check fails if target name is unexpected.
TEST_P(SecureNamingTest,TargetNameIsUnexpected)2730 TEST_P(SecureNamingTest, TargetNameIsUnexpected) {
2731 GRPC_GTEST_FLAG_SET_DEATH_TEST_STYLE("threadsafe");
2732 SetNextResolution({});
2733 SetNextResolutionForLbChannel({balancers_[0]->port()}, nullptr,
2734 "incorrect_server_name");
2735 EdsResourceArgs args({
2736 {"locality0", CreateEndpointsForBackends()},
2737 });
2738 balancers_[0]->ads_service()->SetEdsResource(
2739 BuildEdsResource(args, DefaultEdsServiceName()));
2740 // Make sure that we blow up (via abort() from the security connector) when
2741 // the name from the balancer doesn't match expectations.
2742 ASSERT_DEATH_IF_SUPPORTED({ CheckRpcSendOk(); }, "");
2743 }
2744
2745 using LdsTest = BasicTest;
2746
2747 // Tests that LDS client should send a NACK if there is no API listener in the
2748 // Listener in the LDS response.
TEST_P(LdsTest,NoApiListener)2749 TEST_P(LdsTest, NoApiListener) {
2750 auto listener = default_listener_;
2751 listener.clear_api_listener();
2752 balancers_[0]->ads_service()->SetLdsResource(listener);
2753 SetNextResolutionForLbChannelAllBalancers();
2754 ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK";
2755 const auto response_state =
2756 balancers_[0]->ads_service()->lds_response_state();
2757 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
2758 EXPECT_THAT(
2759 response_state.error_message,
2760 ::testing::HasSubstr("Listener has neither address nor ApiListener"));
2761 }
2762
2763 // Tests that LDS client should send a NACK if the route_specifier in the
2764 // http_connection_manager is neither inlined route_config nor RDS.
TEST_P(LdsTest,WrongRouteSpecifier)2765 TEST_P(LdsTest, WrongRouteSpecifier) {
2766 auto listener = default_listener_;
2767 HttpConnectionManager http_connection_manager;
2768 listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
2769 &http_connection_manager);
2770 http_connection_manager.mutable_scoped_routes();
2771 listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
2772 http_connection_manager);
2773 balancers_[0]->ads_service()->SetLdsResource(listener);
2774 SetNextResolutionForLbChannelAllBalancers();
2775 ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK";
2776 const auto response_state =
2777 balancers_[0]->ads_service()->lds_response_state();
2778 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
2779 EXPECT_THAT(
2780 response_state.error_message,
2781 ::testing::HasSubstr(
2782 "HttpConnectionManager neither has inlined route_config nor RDS."));
2783 }
2784
2785 // Tests that LDS client should send a NACK if the rds message in the
2786 // http_connection_manager is missing the config_source field.
TEST_P(LdsTest,RdsMissingConfigSource)2787 TEST_P(LdsTest, RdsMissingConfigSource) {
2788 auto listener = default_listener_;
2789 HttpConnectionManager http_connection_manager;
2790 listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
2791 &http_connection_manager);
2792 http_connection_manager.mutable_rds()->set_route_config_name(
2793 kDefaultRouteConfigurationName);
2794 listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
2795 http_connection_manager);
2796 balancers_[0]->ads_service()->SetLdsResource(listener);
2797 SetNextResolutionForLbChannelAllBalancers();
2798 ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK";
2799 const auto response_state =
2800 balancers_[0]->ads_service()->lds_response_state();
2801 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
2802 EXPECT_THAT(response_state.error_message,
2803 ::testing::HasSubstr(
2804 "HttpConnectionManager missing config_source for RDS."));
2805 }
2806
2807 // Tests that LDS client should send a NACK if the rds message in the
2808 // http_connection_manager has a config_source field that does not specify
2809 // ADS.
TEST_P(LdsTest,RdsConfigSourceDoesNotSpecifyAds)2810 TEST_P(LdsTest, RdsConfigSourceDoesNotSpecifyAds) {
2811 auto listener = default_listener_;
2812 HttpConnectionManager http_connection_manager;
2813 listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
2814 &http_connection_manager);
2815 auto* rds = http_connection_manager.mutable_rds();
2816 rds->set_route_config_name(kDefaultRouteConfigurationName);
2817 rds->mutable_config_source()->mutable_self();
2818 listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
2819 http_connection_manager);
2820 balancers_[0]->ads_service()->SetLdsResource(listener);
2821 SetNextResolutionForLbChannelAllBalancers();
2822 ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK";
2823 const auto response_state =
2824 balancers_[0]->ads_service()->lds_response_state();
2825 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
2826 EXPECT_THAT(response_state.error_message,
2827 ::testing::HasSubstr("HttpConnectionManager ConfigSource for "
2828 "RDS does not specify ADS."));
2829 }
2830
2831 // Tests that we NACK non-terminal filters at the end of the list.
TEST_P(LdsTest,NacksNonTerminalHttpFilterAtEndOfList)2832 TEST_P(LdsTest, NacksNonTerminalHttpFilterAtEndOfList) {
2833 SetNextResolutionForLbChannelAllBalancers();
2834 auto listener = default_listener_;
2835 HttpConnectionManager http_connection_manager;
2836 listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
2837 &http_connection_manager);
2838 auto* filter = http_connection_manager.mutable_http_filters(0);
2839 filter->set_name("unknown");
2840 filter->mutable_typed_config()->set_type_url(
2841 "grpc.testing.client_only_http_filter");
2842 listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
2843 http_connection_manager);
2844 SetListenerAndRouteConfiguration(0, listener, default_route_config_);
2845 SetNextResolutionForLbChannelAllBalancers();
2846 ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK";
2847 const auto response_state =
2848 balancers_[0]->ads_service()->lds_response_state();
2849 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
2850 EXPECT_THAT(response_state.error_message,
2851 ::testing::HasSubstr(
2852 "non-terminal filter for config type grpc.testing"
2853 ".client_only_http_filter is the last filter in the chain"));
2854 }
2855
2856 // Test that we NACK terminal filters that are not at the end of the list.
TEST_P(LdsTest,NacksTerminalFilterBeforeEndOfList)2857 TEST_P(LdsTest, NacksTerminalFilterBeforeEndOfList) {
2858 SetNextResolutionForLbChannelAllBalancers();
2859 auto listener = default_listener_;
2860 HttpConnectionManager http_connection_manager;
2861 listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
2862 &http_connection_manager);
2863 // The default_listener_ has a terminal router filter by default. Add an
2864 // additional filter.
2865 auto* filter = http_connection_manager.add_http_filters();
2866 filter->set_name("grpc.testing.terminal_http_filter");
2867 filter->mutable_typed_config()->set_type_url(
2868 "grpc.testing.terminal_http_filter");
2869 listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
2870 http_connection_manager);
2871 SetListenerAndRouteConfiguration(0, listener, default_route_config_);
2872 SetNextResolutionForLbChannelAllBalancers();
2873 ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK";
2874 const auto response_state =
2875 balancers_[0]->ads_service()->lds_response_state();
2876 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
2877 EXPECT_THAT(
2878 response_state.error_message,
2879 ::testing::HasSubstr(
2880 "terminal filter for config type envoy.extensions.filters.http"
2881 ".router.v3.Router must be the last filter in the chain"));
2882 }
2883
2884 // Test that we NACK empty filter names.
TEST_P(LdsTest,RejectsEmptyHttpFilterName)2885 TEST_P(LdsTest, RejectsEmptyHttpFilterName) {
2886 auto listener = default_listener_;
2887 HttpConnectionManager http_connection_manager;
2888 listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
2889 &http_connection_manager);
2890 *http_connection_manager.add_http_filters() =
2891 http_connection_manager.http_filters(0);
2892 auto* filter = http_connection_manager.mutable_http_filters(0);
2893 filter->Clear();
2894 filter->mutable_typed_config()->PackFrom(Listener());
2895 listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
2896 http_connection_manager);
2897 SetListenerAndRouteConfiguration(0, listener, default_route_config_);
2898 SetNextResolutionForLbChannelAllBalancers();
2899 ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK";
2900 const auto response_state =
2901 balancers_[0]->ads_service()->lds_response_state();
2902 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
2903 EXPECT_THAT(response_state.error_message,
2904 ::testing::HasSubstr("empty filter name at index 0"));
2905 }
2906
2907 // Test that we NACK duplicate HTTP filter names.
TEST_P(LdsTest,RejectsDuplicateHttpFilterName)2908 TEST_P(LdsTest, RejectsDuplicateHttpFilterName) {
2909 auto listener = default_listener_;
2910 HttpConnectionManager http_connection_manager;
2911 listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
2912 &http_connection_manager);
2913 *http_connection_manager.add_http_filters() =
2914 http_connection_manager.http_filters(0);
2915 http_connection_manager.mutable_http_filters(0)
2916 ->mutable_typed_config()
2917 ->PackFrom(HTTPFault());
2918 listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
2919 http_connection_manager);
2920 SetListenerAndRouteConfiguration(0, listener, default_route_config_);
2921 SetNextResolutionForLbChannelAllBalancers();
2922 ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK";
2923 const auto response_state =
2924 balancers_[0]->ads_service()->lds_response_state();
2925 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
2926 EXPECT_THAT(response_state.error_message,
2927 ::testing::HasSubstr("duplicate HTTP filter name: router"));
2928 }
2929
2930 // Test that we NACK unknown filter types.
TEST_P(LdsTest,RejectsUnknownHttpFilterType)2931 TEST_P(LdsTest, RejectsUnknownHttpFilterType) {
2932 auto listener = default_listener_;
2933 HttpConnectionManager http_connection_manager;
2934 listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
2935 &http_connection_manager);
2936 *http_connection_manager.add_http_filters() =
2937 http_connection_manager.http_filters(0);
2938 auto* filter = http_connection_manager.mutable_http_filters(0);
2939 filter->set_name("unknown");
2940 filter->mutable_typed_config()->PackFrom(Listener());
2941 listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
2942 http_connection_manager);
2943 SetListenerAndRouteConfiguration(0, listener, default_route_config_);
2944 SetNextResolutionForLbChannelAllBalancers();
2945 ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK";
2946 const auto response_state =
2947 balancers_[0]->ads_service()->lds_response_state();
2948 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
2949 EXPECT_THAT(response_state.error_message,
2950 ::testing::HasSubstr("no filter registered for config type "
2951 "envoy.config.listener.v3.Listener"));
2952 }
2953
2954 // Test that we ignore optional unknown filter types.
TEST_P(LdsTest,IgnoresOptionalUnknownHttpFilterType)2955 TEST_P(LdsTest, IgnoresOptionalUnknownHttpFilterType) {
2956 auto listener = default_listener_;
2957 HttpConnectionManager http_connection_manager;
2958 listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
2959 &http_connection_manager);
2960 *http_connection_manager.add_http_filters() =
2961 http_connection_manager.http_filters(0);
2962 auto* filter = http_connection_manager.mutable_http_filters(0);
2963 filter->set_name("unknown");
2964 filter->mutable_typed_config()->PackFrom(Listener());
2965 filter->set_is_optional(true);
2966 listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
2967 http_connection_manager);
2968 SetListenerAndRouteConfiguration(0, listener, default_route_config_);
2969 EdsResourceArgs args({
2970 {"locality0", CreateEndpointsForBackends()},
2971 });
2972 balancers_[0]->ads_service()->SetEdsResource(
2973 BuildEdsResource(args, DefaultEdsServiceName()));
2974 SetNextResolutionForLbChannelAllBalancers();
2975 WaitForAllBackends();
2976 EXPECT_EQ(balancers_[0]->ads_service()->lds_response_state().state,
2977 AdsServiceImpl::ResponseState::ACKED);
2978 }
2979
2980 // Test that we NACK filters without configs.
TEST_P(LdsTest,RejectsHttpFilterWithoutConfig)2981 TEST_P(LdsTest, RejectsHttpFilterWithoutConfig) {
2982 auto listener = default_listener_;
2983 HttpConnectionManager http_connection_manager;
2984 listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
2985 &http_connection_manager);
2986 *http_connection_manager.add_http_filters() =
2987 http_connection_manager.http_filters(0);
2988 auto* filter = http_connection_manager.mutable_http_filters(0);
2989 filter->Clear();
2990 filter->set_name("unknown");
2991 listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
2992 http_connection_manager);
2993 SetListenerAndRouteConfiguration(0, listener, default_route_config_);
2994 SetNextResolutionForLbChannelAllBalancers();
2995 ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK";
2996 const auto response_state =
2997 balancers_[0]->ads_service()->lds_response_state();
2998 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
2999 EXPECT_THAT(response_state.error_message,
3000 ::testing::HasSubstr(
3001 "no filter config specified for filter name unknown"));
3002 }
3003
3004 // Test that we ignore optional filters without configs.
TEST_P(LdsTest,IgnoresOptionalHttpFilterWithoutConfig)3005 TEST_P(LdsTest, IgnoresOptionalHttpFilterWithoutConfig) {
3006 auto listener = default_listener_;
3007 HttpConnectionManager http_connection_manager;
3008 listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
3009 &http_connection_manager);
3010 *http_connection_manager.add_http_filters() =
3011 http_connection_manager.http_filters(0);
3012 auto* filter = http_connection_manager.mutable_http_filters(0);
3013 filter->Clear();
3014 filter->set_name("unknown");
3015 filter->set_is_optional(true);
3016 listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
3017 http_connection_manager);
3018 SetListenerAndRouteConfiguration(0, listener, default_route_config_);
3019 EdsResourceArgs args({
3020 {"locality0", CreateEndpointsForBackends()},
3021 });
3022 balancers_[0]->ads_service()->SetEdsResource(
3023 BuildEdsResource(args, DefaultEdsServiceName()));
3024 SetNextResolutionForLbChannelAllBalancers();
3025 WaitForAllBackends();
3026 EXPECT_EQ(balancers_[0]->ads_service()->lds_response_state().state,
3027 AdsServiceImpl::ResponseState::ACKED);
3028 }
3029
3030 // Test that we NACK unparseable filter configs.
TEST_P(LdsTest,RejectsUnparseableHttpFilterType)3031 TEST_P(LdsTest, RejectsUnparseableHttpFilterType) {
3032 auto listener = default_listener_;
3033 HttpConnectionManager http_connection_manager;
3034 listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
3035 &http_connection_manager);
3036 *http_connection_manager.add_http_filters() =
3037 http_connection_manager.http_filters(0);
3038 auto* filter = http_connection_manager.mutable_http_filters(0);
3039 filter->set_name("unknown");
3040 filter->mutable_typed_config()->PackFrom(listener);
3041 filter->mutable_typed_config()->set_type_url(
3042 "type.googleapis.com/envoy.extensions.filters.http.fault.v3.HTTPFault");
3043 listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
3044 http_connection_manager);
3045 SetListenerAndRouteConfiguration(0, listener, default_route_config_);
3046 SetNextResolutionForLbChannelAllBalancers();
3047 ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK";
3048 const auto response_state =
3049 balancers_[0]->ads_service()->lds_response_state();
3050 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3051 EXPECT_THAT(
3052 response_state.error_message,
3053 ::testing::HasSubstr(
3054 "filter config for type "
3055 "envoy.extensions.filters.http.fault.v3.HTTPFault failed to parse"));
3056 }
3057
3058 // Test that we NACK HTTP filters unsupported on client-side.
TEST_P(LdsTest,RejectsHttpFiltersNotSupportedOnClients)3059 TEST_P(LdsTest, RejectsHttpFiltersNotSupportedOnClients) {
3060 auto listener = default_listener_;
3061 HttpConnectionManager http_connection_manager;
3062 listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
3063 &http_connection_manager);
3064 *http_connection_manager.add_http_filters() =
3065 http_connection_manager.http_filters(0);
3066 auto* filter = http_connection_manager.mutable_http_filters(0);
3067 filter->set_name("grpc.testing.server_only_http_filter");
3068 filter->mutable_typed_config()->set_type_url(
3069 "grpc.testing.server_only_http_filter");
3070 listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
3071 http_connection_manager);
3072 SetListenerAndRouteConfiguration(0, listener, default_route_config_);
3073 SetNextResolutionForLbChannelAllBalancers();
3074 ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK";
3075 const auto response_state =
3076 balancers_[0]->ads_service()->lds_response_state();
3077 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3078 EXPECT_THAT(
3079 response_state.error_message,
3080 ::testing::HasSubstr("Filter grpc.testing.server_only_http_filter is not "
3081 "supported on clients"));
3082 }
3083
3084 // Test that we ignore optional HTTP filters unsupported on client-side.
TEST_P(LdsTest,IgnoresOptionalHttpFiltersNotSupportedOnClients)3085 TEST_P(LdsTest, IgnoresOptionalHttpFiltersNotSupportedOnClients) {
3086 auto listener = default_listener_;
3087 HttpConnectionManager http_connection_manager;
3088 listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
3089 &http_connection_manager);
3090 *http_connection_manager.add_http_filters() =
3091 http_connection_manager.http_filters(0);
3092 auto* filter = http_connection_manager.mutable_http_filters(0);
3093 filter->set_name("grpc.testing.server_only_http_filter");
3094 filter->mutable_typed_config()->set_type_url(
3095 "grpc.testing.server_only_http_filter");
3096 filter->set_is_optional(true);
3097 listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
3098 http_connection_manager);
3099 SetListenerAndRouteConfiguration(0, listener, default_route_config_);
3100 EdsResourceArgs args({
3101 {"locality0", CreateEndpointsForBackends(0, 1)},
3102 });
3103 balancers_[0]->ads_service()->SetEdsResource(
3104 BuildEdsResource(args, DefaultEdsServiceName()));
3105 SetNextResolutionForLbChannelAllBalancers();
3106 WaitForBackend(0);
3107 EXPECT_EQ(balancers_[0]->ads_service()->lds_response_state().state,
3108 AdsServiceImpl::ResponseState::ACKED);
3109 }
3110
3111 using LdsV2Test = LdsTest;
3112
3113 // Tests that we ignore the HTTP filter list in v2.
3114 // TODO(roth): The test framework is not set up to allow us to test
3115 // the server sending v2 resources when the client requests v3, so this
3116 // just tests a pure v2 setup. When we have time, fix this.
TEST_P(LdsV2Test,IgnoresHttpFilters)3117 TEST_P(LdsV2Test, IgnoresHttpFilters) {
3118 auto listener = default_listener_;
3119 HttpConnectionManager http_connection_manager;
3120 listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
3121 &http_connection_manager);
3122 auto* filter = http_connection_manager.add_http_filters();
3123 filter->set_name("unknown");
3124 filter->mutable_typed_config()->PackFrom(Listener());
3125 listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
3126 http_connection_manager);
3127 SetListenerAndRouteConfiguration(0, listener, default_route_config_);
3128 EdsResourceArgs args({
3129 {"locality0", CreateEndpointsForBackends(0, 1)},
3130 });
3131 balancers_[0]->ads_service()->SetEdsResource(
3132 BuildEdsResource(args, DefaultEdsServiceName()));
3133 SetNextResolutionForLbChannelAllBalancers();
3134 CheckRpcSendOk();
3135 }
3136
3137 using LdsRdsTest = BasicTest;
3138
3139 // Tests that LDS client should send an ACK upon correct LDS response (with
3140 // inlined RDS result).
TEST_P(LdsRdsTest,Vanilla)3141 TEST_P(LdsRdsTest, Vanilla) {
3142 SetNextResolution({});
3143 SetNextResolutionForLbChannelAllBalancers();
3144 (void)SendRpc();
3145 EXPECT_EQ(RouteConfigurationResponseState(0).state,
3146 AdsServiceImpl::ResponseState::ACKED);
3147 // Make sure we actually used the RPC service for the right version of xDS.
3148 EXPECT_EQ(balancers_[0]->ads_service()->seen_v2_client(),
3149 GetParam().use_v2());
3150 EXPECT_NE(balancers_[0]->ads_service()->seen_v3_client(),
3151 GetParam().use_v2());
3152 }
3153
3154 // Tests that we go into TRANSIENT_FAILURE if the Listener is removed.
TEST_P(LdsRdsTest,ListenerRemoved)3155 TEST_P(LdsRdsTest, ListenerRemoved) {
3156 SetNextResolution({});
3157 SetNextResolutionForLbChannelAllBalancers();
3158 EdsResourceArgs args({
3159 {"locality0", CreateEndpointsForBackends()},
3160 });
3161 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
3162 // We need to wait for all backends to come online.
3163 WaitForAllBackends();
3164 // Unset LDS resource.
3165 balancers_[0]->ads_service()->UnsetResource(kLdsTypeUrl, kServerName);
3166 // Wait for RPCs to start failing.
3167 do {
3168 } while (SendRpc(RpcOptions(), nullptr).ok());
3169 // Make sure RPCs are still failing.
3170 CheckRpcSendFailure(CheckRpcSendFailureOptions().set_times(1000));
3171 // Make sure we ACK'ed the update.
3172 EXPECT_EQ(balancers_[0]->ads_service()->lds_response_state().state,
3173 AdsServiceImpl::ResponseState::ACKED);
3174 }
3175
3176 // Tests that LDS client ACKs but fails if matching domain can't be found in
3177 // the LDS response.
TEST_P(LdsRdsTest,NoMatchedDomain)3178 TEST_P(LdsRdsTest, NoMatchedDomain) {
3179 RouteConfiguration route_config = default_route_config_;
3180 route_config.mutable_virtual_hosts(0)->clear_domains();
3181 route_config.mutable_virtual_hosts(0)->add_domains("unmatched_domain");
3182 SetRouteConfiguration(0, route_config);
3183 SetNextResolution({});
3184 SetNextResolutionForLbChannelAllBalancers();
3185 CheckRpcSendFailure();
3186 // Do a bit of polling, to allow the ACK to get to the ADS server.
3187 channel_->WaitForConnected(grpc_timeout_milliseconds_to_deadline(100));
3188 const auto response_state = RouteConfigurationResponseState(0);
3189 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::ACKED);
3190 }
3191
3192 // Tests that LDS client should choose the virtual host with matching domain
3193 // if multiple virtual hosts exist in the LDS response.
TEST_P(LdsRdsTest,ChooseMatchedDomain)3194 TEST_P(LdsRdsTest, ChooseMatchedDomain) {
3195 RouteConfiguration route_config = default_route_config_;
3196 *(route_config.add_virtual_hosts()) = route_config.virtual_hosts(0);
3197 route_config.mutable_virtual_hosts(0)->clear_domains();
3198 route_config.mutable_virtual_hosts(0)->add_domains("unmatched_domain");
3199 SetRouteConfiguration(0, route_config);
3200 SetNextResolution({});
3201 SetNextResolutionForLbChannelAllBalancers();
3202 (void)SendRpc();
3203 EXPECT_EQ(RouteConfigurationResponseState(0).state,
3204 AdsServiceImpl::ResponseState::ACKED);
3205 }
3206
3207 // Tests that LDS client should choose the last route in the virtual host if
3208 // multiple routes exist in the LDS response.
TEST_P(LdsRdsTest,ChooseLastRoute)3209 TEST_P(LdsRdsTest, ChooseLastRoute) {
3210 RouteConfiguration route_config = default_route_config_;
3211 *(route_config.mutable_virtual_hosts(0)->add_routes()) =
3212 route_config.virtual_hosts(0).routes(0);
3213 route_config.mutable_virtual_hosts(0)
3214 ->mutable_routes(0)
3215 ->mutable_route()
3216 ->mutable_cluster_header();
3217 SetRouteConfiguration(0, route_config);
3218 SetNextResolution({});
3219 SetNextResolutionForLbChannelAllBalancers();
3220 (void)SendRpc();
3221 EXPECT_EQ(RouteConfigurationResponseState(0).state,
3222 AdsServiceImpl::ResponseState::ACKED);
3223 }
3224
3225 // Tests that LDS client should ignore route which has query_parameters.
TEST_P(LdsRdsTest,RouteMatchHasQueryParameters)3226 TEST_P(LdsRdsTest, RouteMatchHasQueryParameters) {
3227 RouteConfiguration route_config = default_route_config_;
3228 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3229 route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
3230 route1->mutable_match()->add_query_parameters();
3231 SetRouteConfiguration(0, route_config);
3232 SetNextResolution({});
3233 SetNextResolutionForLbChannelAllBalancers();
3234 ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
3235 const auto response_state = RouteConfigurationResponseState(0);
3236 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3237 EXPECT_THAT(response_state.error_message,
3238 ::testing::HasSubstr("No valid routes specified."));
3239 }
3240
3241 // Tests that LDS client should send a ACK if route match has a prefix
3242 // that is either empty or a single slash
TEST_P(LdsRdsTest,RouteMatchHasValidPrefixEmptyOrSingleSlash)3243 TEST_P(LdsRdsTest, RouteMatchHasValidPrefixEmptyOrSingleSlash) {
3244 RouteConfiguration route_config = default_route_config_;
3245 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3246 route1->mutable_match()->set_prefix("");
3247 auto* default_route = route_config.mutable_virtual_hosts(0)->add_routes();
3248 default_route->mutable_match()->set_prefix("/");
3249 default_route->mutable_route()->set_cluster(kDefaultClusterName);
3250 SetRouteConfiguration(0, route_config);
3251 SetNextResolution({});
3252 SetNextResolutionForLbChannelAllBalancers();
3253 (void)SendRpc();
3254 const auto response_state = RouteConfigurationResponseState(0);
3255 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::ACKED);
3256 }
3257
3258 // Tests that LDS client should ignore route which has a path
3259 // prefix string does not start with "/".
TEST_P(LdsRdsTest,RouteMatchHasInvalidPrefixNoLeadingSlash)3260 TEST_P(LdsRdsTest, RouteMatchHasInvalidPrefixNoLeadingSlash) {
3261 RouteConfiguration route_config = default_route_config_;
3262 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3263 route1->mutable_match()->set_prefix("grpc.testing.EchoTest1Service/");
3264 SetRouteConfiguration(0, route_config);
3265 SetNextResolution({});
3266 SetNextResolutionForLbChannelAllBalancers();
3267 ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
3268 const auto response_state = RouteConfigurationResponseState(0);
3269 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3270 EXPECT_THAT(response_state.error_message,
3271 ::testing::HasSubstr("No valid routes specified."));
3272 }
3273
3274 // Tests that LDS client should ignore route which has a prefix
3275 // string with more than 2 slashes.
TEST_P(LdsRdsTest,RouteMatchHasInvalidPrefixExtraContent)3276 TEST_P(LdsRdsTest, RouteMatchHasInvalidPrefixExtraContent) {
3277 RouteConfiguration route_config = default_route_config_;
3278 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3279 route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/Echo1/");
3280 SetRouteConfiguration(0, route_config);
3281 SetNextResolution({});
3282 SetNextResolutionForLbChannelAllBalancers();
3283 ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
3284 const auto response_state = RouteConfigurationResponseState(0);
3285 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3286 EXPECT_THAT(response_state.error_message,
3287 ::testing::HasSubstr("No valid routes specified."));
3288 }
3289
3290 // Tests that LDS client should ignore route which has a prefix
3291 // string "//".
TEST_P(LdsRdsTest,RouteMatchHasInvalidPrefixDoubleSlash)3292 TEST_P(LdsRdsTest, RouteMatchHasInvalidPrefixDoubleSlash) {
3293 RouteConfiguration route_config = default_route_config_;
3294 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3295 route1->mutable_match()->set_prefix("//");
3296 SetRouteConfiguration(0, route_config);
3297 SetNextResolution({});
3298 SetNextResolutionForLbChannelAllBalancers();
3299 ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
3300 const auto response_state = RouteConfigurationResponseState(0);
3301 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3302 EXPECT_THAT(response_state.error_message,
3303 ::testing::HasSubstr("No valid routes specified."));
3304 }
3305
3306 // Tests that LDS client should ignore route which has path
3307 // but it's empty.
TEST_P(LdsRdsTest,RouteMatchHasInvalidPathEmptyPath)3308 TEST_P(LdsRdsTest, RouteMatchHasInvalidPathEmptyPath) {
3309 RouteConfiguration route_config = default_route_config_;
3310 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3311 route1->mutable_match()->set_path("");
3312 SetRouteConfiguration(0, route_config);
3313 SetNextResolution({});
3314 SetNextResolutionForLbChannelAllBalancers();
3315 ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
3316 const auto response_state = RouteConfigurationResponseState(0);
3317 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3318 EXPECT_THAT(response_state.error_message,
3319 ::testing::HasSubstr("No valid routes specified."));
3320 }
3321
3322 // Tests that LDS client should ignore route which has path
3323 // string does not start with "/".
TEST_P(LdsRdsTest,RouteMatchHasInvalidPathNoLeadingSlash)3324 TEST_P(LdsRdsTest, RouteMatchHasInvalidPathNoLeadingSlash) {
3325 RouteConfiguration route_config = default_route_config_;
3326 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3327 route1->mutable_match()->set_path("grpc.testing.EchoTest1Service/Echo1");
3328 SetRouteConfiguration(0, route_config);
3329 SetNextResolution({});
3330 SetNextResolutionForLbChannelAllBalancers();
3331 ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
3332 const auto response_state = RouteConfigurationResponseState(0);
3333 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3334 EXPECT_THAT(response_state.error_message,
3335 ::testing::HasSubstr("No valid routes specified."));
3336 }
3337
3338 // Tests that LDS client should ignore route which has path
3339 // string that has too many slashes; for example, ends with "/".
TEST_P(LdsRdsTest,RouteMatchHasInvalidPathTooManySlashes)3340 TEST_P(LdsRdsTest, RouteMatchHasInvalidPathTooManySlashes) {
3341 RouteConfiguration route_config = default_route_config_;
3342 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3343 route1->mutable_match()->set_path("/grpc.testing.EchoTest1Service/Echo1/");
3344 SetRouteConfiguration(0, route_config);
3345 SetNextResolution({});
3346 SetNextResolutionForLbChannelAllBalancers();
3347 ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
3348 const auto response_state = RouteConfigurationResponseState(0);
3349 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3350 EXPECT_THAT(response_state.error_message,
3351 ::testing::HasSubstr("No valid routes specified."));
3352 }
3353
3354 // Tests that LDS client should ignore route which has path
3355 // string that has only 1 slash: missing "/" between service and method.
TEST_P(LdsRdsTest,RouteMatchHasInvalidPathOnlyOneSlash)3356 TEST_P(LdsRdsTest, RouteMatchHasInvalidPathOnlyOneSlash) {
3357 RouteConfiguration route_config = default_route_config_;
3358 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3359 route1->mutable_match()->set_path("/grpc.testing.EchoTest1Service.Echo1");
3360 SetRouteConfiguration(0, route_config);
3361 SetNextResolution({});
3362 SetNextResolutionForLbChannelAllBalancers();
3363 ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
3364 const auto response_state = RouteConfigurationResponseState(0);
3365 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3366 EXPECT_THAT(response_state.error_message,
3367 ::testing::HasSubstr("No valid routes specified."));
3368 }
3369
3370 // Tests that LDS client should ignore route which has path
3371 // string that is missing service.
TEST_P(LdsRdsTest,RouteMatchHasInvalidPathMissingService)3372 TEST_P(LdsRdsTest, RouteMatchHasInvalidPathMissingService) {
3373 RouteConfiguration route_config = default_route_config_;
3374 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3375 route1->mutable_match()->set_path("//Echo1");
3376 SetRouteConfiguration(0, route_config);
3377 SetNextResolution({});
3378 SetNextResolutionForLbChannelAllBalancers();
3379 ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
3380 const auto response_state = RouteConfigurationResponseState(0);
3381 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3382 EXPECT_THAT(response_state.error_message,
3383 ::testing::HasSubstr("No valid routes specified."));
3384 }
3385
3386 // Tests that LDS client should ignore route which has path
3387 // string that is missing method.
TEST_P(LdsRdsTest,RouteMatchHasInvalidPathMissingMethod)3388 TEST_P(LdsRdsTest, RouteMatchHasInvalidPathMissingMethod) {
3389 RouteConfiguration route_config = default_route_config_;
3390 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3391 route1->mutable_match()->set_path("/grpc.testing.EchoTest1Service/");
3392 SetRouteConfiguration(0, route_config);
3393 SetNextResolution({});
3394 SetNextResolutionForLbChannelAllBalancers();
3395 ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
3396 const auto response_state = RouteConfigurationResponseState(0);
3397 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3398 EXPECT_THAT(response_state.error_message,
3399 ::testing::HasSubstr("No valid routes specified."));
3400 }
3401
3402 // Test that LDS client should reject route which has invalid path regex.
TEST_P(LdsRdsTest,RouteMatchHasInvalidPathRegex)3403 TEST_P(LdsRdsTest, RouteMatchHasInvalidPathRegex) {
3404 const char* kNewCluster1Name = "new_cluster_1";
3405 RouteConfiguration route_config = default_route_config_;
3406 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3407 route1->mutable_match()->mutable_safe_regex()->set_regex("a[z-a]");
3408 route1->mutable_route()->set_cluster(kNewCluster1Name);
3409 SetRouteConfiguration(0, route_config);
3410 SetNextResolution({});
3411 SetNextResolutionForLbChannelAllBalancers();
3412 ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
3413 const auto response_state = RouteConfigurationResponseState(0);
3414 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3415 EXPECT_THAT(response_state.error_message,
3416 ::testing::HasSubstr(
3417 "path matcher: Invalid regex string specified in matcher."));
3418 }
3419
3420 // Tests that LDS client should fail RPCs with UNAVAILABLE status code if the
3421 // matching route has an action other than RouteAction.
TEST_P(LdsRdsTest,MatchingRouteHasNoRouteAction)3422 TEST_P(LdsRdsTest, MatchingRouteHasNoRouteAction) {
3423 RouteConfiguration route_config = default_route_config_;
3424 // Set a route with an inappropriate route action
3425 auto* vhost = route_config.mutable_virtual_hosts(0);
3426 vhost->mutable_routes(0)->mutable_redirect();
3427 // Add another route to make sure that the resolver code actually tries to
3428 // match to a route instead of using a shorthand logic to error out.
3429 auto* route = vhost->add_routes();
3430 route->mutable_match()->set_prefix("");
3431 route->mutable_route()->set_cluster(kDefaultClusterName);
3432 SetRouteConfiguration(0, route_config);
3433 SetNextResolution({});
3434 SetNextResolutionForLbChannelAllBalancers();
3435 CheckRpcSendFailure(CheckRpcSendFailureOptions().set_expected_error_code(
3436 StatusCode::UNAVAILABLE));
3437 }
3438
TEST_P(LdsRdsTest,RouteActionClusterHasEmptyClusterName)3439 TEST_P(LdsRdsTest, RouteActionClusterHasEmptyClusterName) {
3440 RouteConfiguration route_config = default_route_config_;
3441 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3442 route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
3443 route1->mutable_route()->set_cluster("");
3444 auto* default_route = route_config.mutable_virtual_hosts(0)->add_routes();
3445 default_route->mutable_match()->set_prefix("");
3446 default_route->mutable_route()->set_cluster(kDefaultClusterName);
3447 SetRouteConfiguration(0, route_config);
3448 SetNextResolution({});
3449 SetNextResolutionForLbChannelAllBalancers();
3450 ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
3451 const auto response_state = RouteConfigurationResponseState(0);
3452 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3453 EXPECT_THAT(
3454 response_state.error_message,
3455 ::testing::HasSubstr("RouteAction cluster contains empty cluster name."));
3456 }
3457
TEST_P(LdsRdsTest,RouteActionWeightedTargetHasIncorrectTotalWeightSet)3458 TEST_P(LdsRdsTest, RouteActionWeightedTargetHasIncorrectTotalWeightSet) {
3459 const size_t kWeight75 = 75;
3460 const char* kNewCluster1Name = "new_cluster_1";
3461 RouteConfiguration route_config = default_route_config_;
3462 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3463 route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
3464 auto* weighted_cluster1 =
3465 route1->mutable_route()->mutable_weighted_clusters()->add_clusters();
3466 weighted_cluster1->set_name(kNewCluster1Name);
3467 weighted_cluster1->mutable_weight()->set_value(kWeight75);
3468 route1->mutable_route()
3469 ->mutable_weighted_clusters()
3470 ->mutable_total_weight()
3471 ->set_value(kWeight75 + 1);
3472 auto* default_route = route_config.mutable_virtual_hosts(0)->add_routes();
3473 default_route->mutable_match()->set_prefix("");
3474 default_route->mutable_route()->set_cluster(kDefaultClusterName);
3475 SetRouteConfiguration(0, route_config);
3476 SetNextResolution({});
3477 SetNextResolutionForLbChannelAllBalancers();
3478 ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
3479 const auto response_state = RouteConfigurationResponseState(0);
3480 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3481 EXPECT_THAT(response_state.error_message,
3482 ::testing::HasSubstr(
3483 "RouteAction weighted_cluster has incorrect total weight"));
3484 }
3485
TEST_P(LdsRdsTest,RouteActionWeightedClusterHasZeroTotalWeight)3486 TEST_P(LdsRdsTest, RouteActionWeightedClusterHasZeroTotalWeight) {
3487 const char* kNewCluster1Name = "new_cluster_1";
3488 RouteConfiguration route_config = default_route_config_;
3489 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3490 route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
3491 auto* weighted_cluster1 =
3492 route1->mutable_route()->mutable_weighted_clusters()->add_clusters();
3493 weighted_cluster1->set_name(kNewCluster1Name);
3494 weighted_cluster1->mutable_weight()->set_value(0);
3495 route1->mutable_route()
3496 ->mutable_weighted_clusters()
3497 ->mutable_total_weight()
3498 ->set_value(0);
3499 auto* default_route = route_config.mutable_virtual_hosts(0)->add_routes();
3500 default_route->mutable_match()->set_prefix("");
3501 default_route->mutable_route()->set_cluster(kDefaultClusterName);
3502 SetRouteConfiguration(0, route_config);
3503 SetNextResolution({});
3504 SetNextResolutionForLbChannelAllBalancers();
3505 ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
3506 const auto response_state = RouteConfigurationResponseState(0);
3507 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3508 EXPECT_THAT(
3509 response_state.error_message,
3510 ::testing::HasSubstr(
3511 "RouteAction weighted_cluster has no valid clusters specified."));
3512 }
3513
TEST_P(LdsRdsTest,RouteActionWeightedTargetClusterHasEmptyClusterName)3514 TEST_P(LdsRdsTest, RouteActionWeightedTargetClusterHasEmptyClusterName) {
3515 const size_t kWeight75 = 75;
3516 RouteConfiguration route_config = default_route_config_;
3517 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3518 route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
3519 auto* weighted_cluster1 =
3520 route1->mutable_route()->mutable_weighted_clusters()->add_clusters();
3521 weighted_cluster1->set_name("");
3522 weighted_cluster1->mutable_weight()->set_value(kWeight75);
3523 route1->mutable_route()
3524 ->mutable_weighted_clusters()
3525 ->mutable_total_weight()
3526 ->set_value(kWeight75);
3527 auto* default_route = route_config.mutable_virtual_hosts(0)->add_routes();
3528 default_route->mutable_match()->set_prefix("");
3529 default_route->mutable_route()->set_cluster(kDefaultClusterName);
3530 SetRouteConfiguration(0, route_config);
3531 SetNextResolution({});
3532 SetNextResolutionForLbChannelAllBalancers();
3533 ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
3534 const auto response_state = RouteConfigurationResponseState(0);
3535 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3536 EXPECT_THAT(response_state.error_message,
3537 ::testing::HasSubstr("RouteAction weighted_cluster cluster "
3538 "contains empty cluster name."));
3539 }
3540
TEST_P(LdsRdsTest,RouteActionWeightedTargetClusterHasNoWeight)3541 TEST_P(LdsRdsTest, RouteActionWeightedTargetClusterHasNoWeight) {
3542 const size_t kWeight75 = 75;
3543 const char* kNewCluster1Name = "new_cluster_1";
3544 RouteConfiguration route_config = default_route_config_;
3545 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3546 route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
3547 auto* weighted_cluster1 =
3548 route1->mutable_route()->mutable_weighted_clusters()->add_clusters();
3549 weighted_cluster1->set_name(kNewCluster1Name);
3550 route1->mutable_route()
3551 ->mutable_weighted_clusters()
3552 ->mutable_total_weight()
3553 ->set_value(kWeight75);
3554 auto* default_route = route_config.mutable_virtual_hosts(0)->add_routes();
3555 default_route->mutable_match()->set_prefix("");
3556 default_route->mutable_route()->set_cluster(kDefaultClusterName);
3557 SetRouteConfiguration(0, route_config);
3558 SetNextResolution({});
3559 SetNextResolutionForLbChannelAllBalancers();
3560 ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
3561 const auto response_state = RouteConfigurationResponseState(0);
3562 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3563 EXPECT_THAT(response_state.error_message,
3564 ::testing::HasSubstr(
3565 "RouteAction weighted_cluster cluster missing weight"));
3566 }
3567
TEST_P(LdsRdsTest,RouteHeaderMatchInvalidRegex)3568 TEST_P(LdsRdsTest, RouteHeaderMatchInvalidRegex) {
3569 const char* kNewCluster1Name = "new_cluster_1";
3570 RouteConfiguration route_config = default_route_config_;
3571 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3572 route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
3573 auto* header_matcher1 = route1->mutable_match()->add_headers();
3574 header_matcher1->set_name("header1");
3575 header_matcher1->mutable_safe_regex_match()->set_regex("a[z-a]");
3576 route1->mutable_route()->set_cluster(kNewCluster1Name);
3577 SetRouteConfiguration(0, route_config);
3578 SetNextResolution({});
3579 SetNextResolutionForLbChannelAllBalancers();
3580 ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
3581 const auto response_state = RouteConfigurationResponseState(0);
3582 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3583 EXPECT_THAT(
3584 response_state.error_message,
3585 ::testing::HasSubstr(
3586 "header matcher: Invalid regex string specified in matcher."));
3587 }
3588
TEST_P(LdsRdsTest,RouteHeaderMatchInvalidRange)3589 TEST_P(LdsRdsTest, RouteHeaderMatchInvalidRange) {
3590 const char* kNewCluster1Name = "new_cluster_1";
3591 RouteConfiguration route_config = default_route_config_;
3592 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3593 route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
3594 auto* header_matcher1 = route1->mutable_match()->add_headers();
3595 header_matcher1->set_name("header1");
3596 header_matcher1->mutable_range_match()->set_start(1001);
3597 header_matcher1->mutable_range_match()->set_end(1000);
3598 route1->mutable_route()->set_cluster(kNewCluster1Name);
3599 SetRouteConfiguration(0, route_config);
3600 SetNextResolution({});
3601 SetNextResolutionForLbChannelAllBalancers();
3602 ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
3603 const auto response_state = RouteConfigurationResponseState(0);
3604 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3605 EXPECT_THAT(
3606 response_state.error_message,
3607 ::testing::HasSubstr(
3608 "header matcher: Invalid range specifier specified: end cannot be "
3609 "smaller than start."));
3610 }
3611
3612 // Tests that LDS client should choose the default route (with no matching
3613 // specified) after unable to find a match with previous routes.
TEST_P(LdsRdsTest,XdsRoutingPathMatching)3614 TEST_P(LdsRdsTest, XdsRoutingPathMatching) {
3615 const char* kNewCluster1Name = "new_cluster_1";
3616 const char* kNewEdsService1Name = "new_eds_service_name_1";
3617 const char* kNewCluster2Name = "new_cluster_2";
3618 const char* kNewEdsService2Name = "new_eds_service_name_2";
3619 const size_t kNumEcho1Rpcs = 10;
3620 const size_t kNumEcho2Rpcs = 20;
3621 const size_t kNumEchoRpcs = 30;
3622 SetNextResolution({});
3623 SetNextResolutionForLbChannelAllBalancers();
3624 // Populate new EDS resources.
3625 EdsResourceArgs args({
3626 {"locality0", CreateEndpointsForBackends(0, 2)},
3627 });
3628 EdsResourceArgs args1({
3629 {"locality0", CreateEndpointsForBackends(2, 3)},
3630 });
3631 EdsResourceArgs args2({
3632 {"locality0", CreateEndpointsForBackends(3, 4)},
3633 });
3634 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
3635 balancers_[0]->ads_service()->SetEdsResource(
3636 BuildEdsResource(args1, kNewEdsService1Name));
3637 balancers_[0]->ads_service()->SetEdsResource(
3638 BuildEdsResource(args2, kNewEdsService2Name));
3639 // Populate new CDS resources.
3640 Cluster new_cluster1 = default_cluster_;
3641 new_cluster1.set_name(kNewCluster1Name);
3642 new_cluster1.mutable_eds_cluster_config()->set_service_name(
3643 kNewEdsService1Name);
3644 balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
3645 Cluster new_cluster2 = default_cluster_;
3646 new_cluster2.set_name(kNewCluster2Name);
3647 new_cluster2.mutable_eds_cluster_config()->set_service_name(
3648 kNewEdsService2Name);
3649 balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
3650 // Populating Route Configurations for LDS.
3651 RouteConfiguration new_route_config = default_route_config_;
3652 auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3653 route1->mutable_match()->set_path("/grpc.testing.EchoTest1Service/Echo1");
3654 route1->mutable_route()->set_cluster(kNewCluster1Name);
3655 auto* route2 = new_route_config.mutable_virtual_hosts(0)->add_routes();
3656 route2->mutable_match()->set_path("/grpc.testing.EchoTest2Service/Echo2");
3657 route2->mutable_route()->set_cluster(kNewCluster2Name);
3658 auto* route3 = new_route_config.mutable_virtual_hosts(0)->add_routes();
3659 route3->mutable_match()->set_path("/grpc.testing.EchoTest3Service/Echo3");
3660 route3->mutable_route()->set_cluster(kDefaultClusterName);
3661 auto* default_route = new_route_config.mutable_virtual_hosts(0)->add_routes();
3662 default_route->mutable_match()->set_prefix("");
3663 default_route->mutable_route()->set_cluster(kDefaultClusterName);
3664 SetRouteConfiguration(0, new_route_config);
3665 WaitForAllBackends(0, 2);
3666 CheckRpcSendOk(kNumEchoRpcs, RpcOptions().set_wait_for_ready(true));
3667 CheckRpcSendOk(kNumEcho1Rpcs, RpcOptions()
3668 .set_rpc_service(SERVICE_ECHO1)
3669 .set_rpc_method(METHOD_ECHO1)
3670 .set_wait_for_ready(true));
3671 CheckRpcSendOk(kNumEcho2Rpcs, RpcOptions()
3672 .set_rpc_service(SERVICE_ECHO2)
3673 .set_rpc_method(METHOD_ECHO2)
3674 .set_wait_for_ready(true));
3675 // Make sure RPCs all go to the correct backend.
3676 for (size_t i = 0; i < 2; ++i) {
3677 EXPECT_EQ(kNumEchoRpcs / 2,
3678 backends_[i]->backend_service()->request_count());
3679 EXPECT_EQ(0, backends_[i]->backend_service1()->request_count());
3680 EXPECT_EQ(0, backends_[i]->backend_service2()->request_count());
3681 }
3682 EXPECT_EQ(0, backends_[2]->backend_service()->request_count());
3683 EXPECT_EQ(kNumEcho1Rpcs, backends_[2]->backend_service1()->request_count());
3684 EXPECT_EQ(0, backends_[2]->backend_service2()->request_count());
3685 EXPECT_EQ(0, backends_[3]->backend_service()->request_count());
3686 EXPECT_EQ(0, backends_[3]->backend_service1()->request_count());
3687 EXPECT_EQ(kNumEcho2Rpcs, backends_[3]->backend_service2()->request_count());
3688 }
3689
TEST_P(LdsRdsTest,XdsRoutingPathMatchingCaseInsensitive)3690 TEST_P(LdsRdsTest, XdsRoutingPathMatchingCaseInsensitive) {
3691 const char* kNewCluster1Name = "new_cluster_1";
3692 const char* kNewEdsService1Name = "new_eds_service_name_1";
3693 const char* kNewCluster2Name = "new_cluster_2";
3694 const char* kNewEdsService2Name = "new_eds_service_name_2";
3695 const size_t kNumEcho1Rpcs = 10;
3696 const size_t kNumEchoRpcs = 30;
3697 SetNextResolution({});
3698 SetNextResolutionForLbChannelAllBalancers();
3699 // Populate new EDS resources.
3700 EdsResourceArgs args({
3701 {"locality0", CreateEndpointsForBackends(0, 1)},
3702 });
3703 EdsResourceArgs args1({
3704 {"locality0", CreateEndpointsForBackends(1, 2)},
3705 });
3706 EdsResourceArgs args2({
3707 {"locality0", CreateEndpointsForBackends(2, 3)},
3708 });
3709 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
3710 balancers_[0]->ads_service()->SetEdsResource(
3711 BuildEdsResource(args1, kNewEdsService1Name));
3712 balancers_[0]->ads_service()->SetEdsResource(
3713 BuildEdsResource(args2, kNewEdsService2Name));
3714 // Populate new CDS resources.
3715 Cluster new_cluster1 = default_cluster_;
3716 new_cluster1.set_name(kNewCluster1Name);
3717 new_cluster1.mutable_eds_cluster_config()->set_service_name(
3718 kNewEdsService1Name);
3719 balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
3720 Cluster new_cluster2 = default_cluster_;
3721 new_cluster2.set_name(kNewCluster2Name);
3722 new_cluster2.mutable_eds_cluster_config()->set_service_name(
3723 kNewEdsService2Name);
3724 balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
3725 // Populating Route Configurations for LDS.
3726 RouteConfiguration new_route_config = default_route_config_;
3727 // First route will not match, since it's case-sensitive.
3728 // Second route will match with same path.
3729 auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3730 route1->mutable_match()->set_path("/GrPc.TeStInG.EcHoTeSt1SErViCe/EcHo1");
3731 route1->mutable_route()->set_cluster(kNewCluster1Name);
3732 auto* route2 = new_route_config.mutable_virtual_hosts(0)->add_routes();
3733 route2->mutable_match()->set_path("/GrPc.TeStInG.EcHoTeSt1SErViCe/EcHo1");
3734 route2->mutable_match()->mutable_case_sensitive()->set_value(false);
3735 route2->mutable_route()->set_cluster(kNewCluster2Name);
3736 auto* default_route = new_route_config.mutable_virtual_hosts(0)->add_routes();
3737 default_route->mutable_match()->set_prefix("");
3738 default_route->mutable_route()->set_cluster(kDefaultClusterName);
3739 SetRouteConfiguration(0, new_route_config);
3740 CheckRpcSendOk(kNumEchoRpcs, RpcOptions().set_wait_for_ready(true));
3741 CheckRpcSendOk(kNumEcho1Rpcs, RpcOptions()
3742 .set_rpc_service(SERVICE_ECHO1)
3743 .set_rpc_method(METHOD_ECHO1)
3744 .set_wait_for_ready(true));
3745 // Make sure RPCs all go to the correct backend.
3746 EXPECT_EQ(kNumEchoRpcs, backends_[0]->backend_service()->request_count());
3747 EXPECT_EQ(0, backends_[0]->backend_service1()->request_count());
3748 EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
3749 EXPECT_EQ(0, backends_[1]->backend_service1()->request_count());
3750 EXPECT_EQ(0, backends_[2]->backend_service()->request_count());
3751 EXPECT_EQ(kNumEcho1Rpcs, backends_[2]->backend_service1()->request_count());
3752 }
3753
TEST_P(LdsRdsTest,XdsRoutingPrefixMatching)3754 TEST_P(LdsRdsTest, XdsRoutingPrefixMatching) {
3755 const char* kNewCluster1Name = "new_cluster_1";
3756 const char* kNewEdsService1Name = "new_eds_service_name_1";
3757 const char* kNewCluster2Name = "new_cluster_2";
3758 const char* kNewEdsService2Name = "new_eds_service_name_2";
3759 const size_t kNumEcho1Rpcs = 10;
3760 const size_t kNumEcho2Rpcs = 20;
3761 const size_t kNumEchoRpcs = 30;
3762 SetNextResolution({});
3763 SetNextResolutionForLbChannelAllBalancers();
3764 // Populate new EDS resources.
3765 EdsResourceArgs args({
3766 {"locality0", CreateEndpointsForBackends(0, 2)},
3767 });
3768 EdsResourceArgs args1({
3769 {"locality0", CreateEndpointsForBackends(2, 3)},
3770 });
3771 EdsResourceArgs args2({
3772 {"locality0", CreateEndpointsForBackends(3, 4)},
3773 });
3774 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
3775 balancers_[0]->ads_service()->SetEdsResource(
3776 BuildEdsResource(args1, kNewEdsService1Name));
3777 balancers_[0]->ads_service()->SetEdsResource(
3778 BuildEdsResource(args2, kNewEdsService2Name));
3779 // Populate new CDS resources.
3780 Cluster new_cluster1 = default_cluster_;
3781 new_cluster1.set_name(kNewCluster1Name);
3782 new_cluster1.mutable_eds_cluster_config()->set_service_name(
3783 kNewEdsService1Name);
3784 balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
3785 Cluster new_cluster2 = default_cluster_;
3786 new_cluster2.set_name(kNewCluster2Name);
3787 new_cluster2.mutable_eds_cluster_config()->set_service_name(
3788 kNewEdsService2Name);
3789 balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
3790 // Populating Route Configurations for LDS.
3791 RouteConfiguration new_route_config = default_route_config_;
3792 auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3793 route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
3794 route1->mutable_route()->set_cluster(kNewCluster1Name);
3795 auto* route2 = new_route_config.mutable_virtual_hosts(0)->add_routes();
3796 route2->mutable_match()->set_prefix("/grpc.testing.EchoTest2Service/");
3797 route2->mutable_route()->set_cluster(kNewCluster2Name);
3798 auto* default_route = new_route_config.mutable_virtual_hosts(0)->add_routes();
3799 default_route->mutable_match()->set_prefix("");
3800 default_route->mutable_route()->set_cluster(kDefaultClusterName);
3801 SetRouteConfiguration(0, new_route_config);
3802 WaitForAllBackends(0, 2);
3803 CheckRpcSendOk(kNumEchoRpcs, RpcOptions().set_wait_for_ready(true));
3804 CheckRpcSendOk(
3805 kNumEcho1Rpcs,
3806 RpcOptions().set_rpc_service(SERVICE_ECHO1).set_wait_for_ready(true));
3807 CheckRpcSendOk(
3808 kNumEcho2Rpcs,
3809 RpcOptions().set_rpc_service(SERVICE_ECHO2).set_wait_for_ready(true));
3810 // Make sure RPCs all go to the correct backend.
3811 for (size_t i = 0; i < 2; ++i) {
3812 EXPECT_EQ(kNumEchoRpcs / 2,
3813 backends_[i]->backend_service()->request_count());
3814 EXPECT_EQ(0, backends_[i]->backend_service1()->request_count());
3815 EXPECT_EQ(0, backends_[i]->backend_service2()->request_count());
3816 }
3817 EXPECT_EQ(0, backends_[2]->backend_service()->request_count());
3818 EXPECT_EQ(kNumEcho1Rpcs, backends_[2]->backend_service1()->request_count());
3819 EXPECT_EQ(0, backends_[2]->backend_service2()->request_count());
3820 EXPECT_EQ(0, backends_[3]->backend_service()->request_count());
3821 EXPECT_EQ(0, backends_[3]->backend_service1()->request_count());
3822 EXPECT_EQ(kNumEcho2Rpcs, backends_[3]->backend_service2()->request_count());
3823 }
3824
TEST_P(LdsRdsTest,XdsRoutingPrefixMatchingCaseInsensitive)3825 TEST_P(LdsRdsTest, XdsRoutingPrefixMatchingCaseInsensitive) {
3826 const char* kNewCluster1Name = "new_cluster_1";
3827 const char* kNewEdsService1Name = "new_eds_service_name_1";
3828 const char* kNewCluster2Name = "new_cluster_2";
3829 const char* kNewEdsService2Name = "new_eds_service_name_2";
3830 const size_t kNumEcho1Rpcs = 10;
3831 const size_t kNumEchoRpcs = 30;
3832 SetNextResolution({});
3833 SetNextResolutionForLbChannelAllBalancers();
3834 // Populate new EDS resources.
3835 EdsResourceArgs args({
3836 {"locality0", CreateEndpointsForBackends(0, 1)},
3837 });
3838 EdsResourceArgs args1({
3839 {"locality0", CreateEndpointsForBackends(1, 2)},
3840 });
3841 EdsResourceArgs args2({
3842 {"locality0", CreateEndpointsForBackends(2, 3)},
3843 });
3844 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
3845 balancers_[0]->ads_service()->SetEdsResource(
3846 BuildEdsResource(args1, kNewEdsService1Name));
3847 balancers_[0]->ads_service()->SetEdsResource(
3848 BuildEdsResource(args2, kNewEdsService2Name));
3849 // Populate new CDS resources.
3850 Cluster new_cluster1 = default_cluster_;
3851 new_cluster1.set_name(kNewCluster1Name);
3852 new_cluster1.mutable_eds_cluster_config()->set_service_name(
3853 kNewEdsService1Name);
3854 balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
3855 Cluster new_cluster2 = default_cluster_;
3856 new_cluster2.set_name(kNewCluster2Name);
3857 new_cluster2.mutable_eds_cluster_config()->set_service_name(
3858 kNewEdsService2Name);
3859 balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
3860 // Populating Route Configurations for LDS.
3861 RouteConfiguration new_route_config = default_route_config_;
3862 // First route will not match, since it's case-sensitive.
3863 // Second route will match with same path.
3864 auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3865 route1->mutable_match()->set_prefix("/GrPc.TeStInG.EcHoTeSt1SErViCe");
3866 route1->mutable_route()->set_cluster(kNewCluster1Name);
3867 auto* route2 = new_route_config.mutable_virtual_hosts(0)->add_routes();
3868 route2->mutable_match()->set_prefix("/GrPc.TeStInG.EcHoTeSt1SErViCe");
3869 route2->mutable_match()->mutable_case_sensitive()->set_value(false);
3870 route2->mutable_route()->set_cluster(kNewCluster2Name);
3871 auto* default_route = new_route_config.mutable_virtual_hosts(0)->add_routes();
3872 default_route->mutable_match()->set_prefix("");
3873 default_route->mutable_route()->set_cluster(kDefaultClusterName);
3874 SetRouteConfiguration(0, new_route_config);
3875 CheckRpcSendOk(kNumEchoRpcs, RpcOptions().set_wait_for_ready(true));
3876 CheckRpcSendOk(kNumEcho1Rpcs, RpcOptions()
3877 .set_rpc_service(SERVICE_ECHO1)
3878 .set_rpc_method(METHOD_ECHO1)
3879 .set_wait_for_ready(true));
3880 // Make sure RPCs all go to the correct backend.
3881 EXPECT_EQ(kNumEchoRpcs, backends_[0]->backend_service()->request_count());
3882 EXPECT_EQ(0, backends_[0]->backend_service1()->request_count());
3883 EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
3884 EXPECT_EQ(0, backends_[1]->backend_service1()->request_count());
3885 EXPECT_EQ(0, backends_[2]->backend_service()->request_count());
3886 EXPECT_EQ(kNumEcho1Rpcs, backends_[2]->backend_service1()->request_count());
3887 }
3888
TEST_P(LdsRdsTest,XdsRoutingPathRegexMatching)3889 TEST_P(LdsRdsTest, XdsRoutingPathRegexMatching) {
3890 const char* kNewCluster1Name = "new_cluster_1";
3891 const char* kNewEdsService1Name = "new_eds_service_name_1";
3892 const char* kNewCluster2Name = "new_cluster_2";
3893 const char* kNewEdsService2Name = "new_eds_service_name_2";
3894 const size_t kNumEcho1Rpcs = 10;
3895 const size_t kNumEcho2Rpcs = 20;
3896 const size_t kNumEchoRpcs = 30;
3897 SetNextResolution({});
3898 SetNextResolutionForLbChannelAllBalancers();
3899 // Populate new EDS resources.
3900 EdsResourceArgs args({
3901 {"locality0", CreateEndpointsForBackends(0, 2)},
3902 });
3903 EdsResourceArgs args1({
3904 {"locality0", CreateEndpointsForBackends(2, 3)},
3905 });
3906 EdsResourceArgs args2({
3907 {"locality0", CreateEndpointsForBackends(3, 4)},
3908 });
3909 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
3910 balancers_[0]->ads_service()->SetEdsResource(
3911 BuildEdsResource(args1, kNewEdsService1Name));
3912 balancers_[0]->ads_service()->SetEdsResource(
3913 BuildEdsResource(args2, kNewEdsService2Name));
3914 // Populate new CDS resources.
3915 Cluster new_cluster1 = default_cluster_;
3916 new_cluster1.set_name(kNewCluster1Name);
3917 new_cluster1.mutable_eds_cluster_config()->set_service_name(
3918 kNewEdsService1Name);
3919 balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
3920 Cluster new_cluster2 = default_cluster_;
3921 new_cluster2.set_name(kNewCluster2Name);
3922 new_cluster2.mutable_eds_cluster_config()->set_service_name(
3923 kNewEdsService2Name);
3924 balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
3925 // Populating Route Configurations for LDS.
3926 RouteConfiguration new_route_config = default_route_config_;
3927 auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
3928 // Will match "/grpc.testing.EchoTest1Service/"
3929 route1->mutable_match()->mutable_safe_regex()->set_regex(".*1.*");
3930 route1->mutable_route()->set_cluster(kNewCluster1Name);
3931 auto* route2 = new_route_config.mutable_virtual_hosts(0)->add_routes();
3932 // Will match "/grpc.testing.EchoTest2Service/"
3933 route2->mutable_match()->mutable_safe_regex()->set_regex(".*2.*");
3934 route2->mutable_route()->set_cluster(kNewCluster2Name);
3935 auto* default_route = new_route_config.mutable_virtual_hosts(0)->add_routes();
3936 default_route->mutable_match()->set_prefix("");
3937 default_route->mutable_route()->set_cluster(kDefaultClusterName);
3938 SetRouteConfiguration(0, new_route_config);
3939 WaitForAllBackends(0, 2);
3940 CheckRpcSendOk(kNumEchoRpcs, RpcOptions().set_wait_for_ready(true));
3941 CheckRpcSendOk(
3942 kNumEcho1Rpcs,
3943 RpcOptions().set_rpc_service(SERVICE_ECHO1).set_wait_for_ready(true));
3944 CheckRpcSendOk(
3945 kNumEcho2Rpcs,
3946 RpcOptions().set_rpc_service(SERVICE_ECHO2).set_wait_for_ready(true));
3947 // Make sure RPCs all go to the correct backend.
3948 for (size_t i = 0; i < 2; ++i) {
3949 EXPECT_EQ(kNumEchoRpcs / 2,
3950 backends_[i]->backend_service()->request_count());
3951 EXPECT_EQ(0, backends_[i]->backend_service1()->request_count());
3952 EXPECT_EQ(0, backends_[i]->backend_service2()->request_count());
3953 }
3954 EXPECT_EQ(0, backends_[2]->backend_service()->request_count());
3955 EXPECT_EQ(kNumEcho1Rpcs, backends_[2]->backend_service1()->request_count());
3956 EXPECT_EQ(0, backends_[2]->backend_service2()->request_count());
3957 EXPECT_EQ(0, backends_[3]->backend_service()->request_count());
3958 EXPECT_EQ(0, backends_[3]->backend_service1()->request_count());
3959 EXPECT_EQ(kNumEcho2Rpcs, backends_[3]->backend_service2()->request_count());
3960 }
3961
TEST_P(LdsRdsTest,XdsRoutingWeightedCluster)3962 TEST_P(LdsRdsTest, XdsRoutingWeightedCluster) {
3963 const char* kNewCluster1Name = "new_cluster_1";
3964 const char* kNewEdsService1Name = "new_eds_service_name_1";
3965 const char* kNewCluster2Name = "new_cluster_2";
3966 const char* kNewEdsService2Name = "new_eds_service_name_2";
3967 const char* kNotUsedClusterName = "not_used_cluster";
3968 const size_t kNumEchoRpcs = 10; // RPCs that will go to a fixed backend.
3969 const size_t kWeight75 = 75;
3970 const size_t kWeight25 = 25;
3971 const double kErrorTolerance = 0.05;
3972 const double kWeight75Percent = static_cast<double>(kWeight75) / 100;
3973 const double kWeight25Percent = static_cast<double>(kWeight25) / 100;
3974 const size_t kNumEcho1Rpcs =
3975 ComputeIdealNumRpcs(kWeight75Percent, kErrorTolerance);
3976 SetNextResolution({});
3977 SetNextResolutionForLbChannelAllBalancers();
3978 // Populate new EDS resources.
3979 EdsResourceArgs args({
3980 {"locality0", CreateEndpointsForBackends(0, 1)},
3981 });
3982 EdsResourceArgs args1({
3983 {"locality0", CreateEndpointsForBackends(1, 2)},
3984 });
3985 EdsResourceArgs args2({
3986 {"locality0", CreateEndpointsForBackends(2, 3)},
3987 });
3988 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
3989 balancers_[0]->ads_service()->SetEdsResource(
3990 BuildEdsResource(args1, kNewEdsService1Name));
3991 balancers_[0]->ads_service()->SetEdsResource(
3992 BuildEdsResource(args2, kNewEdsService2Name));
3993 // Populate new CDS resources.
3994 Cluster new_cluster1 = default_cluster_;
3995 new_cluster1.set_name(kNewCluster1Name);
3996 new_cluster1.mutable_eds_cluster_config()->set_service_name(
3997 kNewEdsService1Name);
3998 balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
3999 Cluster new_cluster2 = default_cluster_;
4000 new_cluster2.set_name(kNewCluster2Name);
4001 new_cluster2.mutable_eds_cluster_config()->set_service_name(
4002 kNewEdsService2Name);
4003 balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
4004 // Populating Route Configurations for LDS.
4005 RouteConfiguration new_route_config = default_route_config_;
4006 auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4007 route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
4008 auto* weighted_cluster1 =
4009 route1->mutable_route()->mutable_weighted_clusters()->add_clusters();
4010 weighted_cluster1->set_name(kNewCluster1Name);
4011 weighted_cluster1->mutable_weight()->set_value(kWeight75);
4012 auto* weighted_cluster2 =
4013 route1->mutable_route()->mutable_weighted_clusters()->add_clusters();
4014 weighted_cluster2->set_name(kNewCluster2Name);
4015 weighted_cluster2->mutable_weight()->set_value(kWeight25);
4016 // Cluster with weight 0 will not be used.
4017 auto* weighted_cluster3 =
4018 route1->mutable_route()->mutable_weighted_clusters()->add_clusters();
4019 weighted_cluster3->set_name(kNotUsedClusterName);
4020 weighted_cluster3->mutable_weight()->set_value(0);
4021 route1->mutable_route()
4022 ->mutable_weighted_clusters()
4023 ->mutable_total_weight()
4024 ->set_value(kWeight75 + kWeight25);
4025 auto* default_route = new_route_config.mutable_virtual_hosts(0)->add_routes();
4026 default_route->mutable_match()->set_prefix("");
4027 default_route->mutable_route()->set_cluster(kDefaultClusterName);
4028 SetRouteConfiguration(0, new_route_config);
4029 WaitForAllBackends(0, 1);
4030 WaitForAllBackends(1, 3, WaitForBackendOptions(),
4031 RpcOptions().set_rpc_service(SERVICE_ECHO1));
4032 CheckRpcSendOk(kNumEchoRpcs);
4033 CheckRpcSendOk(kNumEcho1Rpcs, RpcOptions().set_rpc_service(SERVICE_ECHO1));
4034 // Make sure RPCs all go to the correct backend.
4035 EXPECT_EQ(kNumEchoRpcs, backends_[0]->backend_service()->request_count());
4036 EXPECT_EQ(0, backends_[0]->backend_service1()->request_count());
4037 EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
4038 const int weight_75_request_count =
4039 backends_[1]->backend_service1()->request_count();
4040 EXPECT_EQ(0, backends_[2]->backend_service()->request_count());
4041 const int weight_25_request_count =
4042 backends_[2]->backend_service1()->request_count();
4043 gpr_log(GPR_INFO, "target_75 received %d rpcs and target_25 received %d rpcs",
4044 weight_75_request_count, weight_25_request_count);
4045 EXPECT_THAT(static_cast<double>(weight_75_request_count) / kNumEcho1Rpcs,
4046 ::testing::DoubleNear(kWeight75Percent, kErrorTolerance));
4047 EXPECT_THAT(static_cast<double>(weight_25_request_count) / kNumEcho1Rpcs,
4048 ::testing::DoubleNear(kWeight25Percent, kErrorTolerance));
4049 }
4050
TEST_P(LdsRdsTest,RouteActionWeightedTargetDefaultRoute)4051 TEST_P(LdsRdsTest, RouteActionWeightedTargetDefaultRoute) {
4052 const char* kNewCluster1Name = "new_cluster_1";
4053 const char* kNewEdsService1Name = "new_eds_service_name_1";
4054 const char* kNewCluster2Name = "new_cluster_2";
4055 const char* kNewEdsService2Name = "new_eds_service_name_2";
4056 const size_t kWeight75 = 75;
4057 const size_t kWeight25 = 25;
4058 const double kErrorTolerance = 0.05;
4059 const double kWeight75Percent = static_cast<double>(kWeight75) / 100;
4060 const double kWeight25Percent = static_cast<double>(kWeight25) / 100;
4061 const size_t kNumEchoRpcs =
4062 ComputeIdealNumRpcs(kWeight75Percent, kErrorTolerance);
4063 SetNextResolution({});
4064 SetNextResolutionForLbChannelAllBalancers();
4065 // Populate new EDS resources.
4066 EdsResourceArgs args({
4067 {"locality0", CreateEndpointsForBackends(0, 1)},
4068 });
4069 EdsResourceArgs args1({
4070 {"locality0", CreateEndpointsForBackends(1, 2)},
4071 });
4072 EdsResourceArgs args2({
4073 {"locality0", CreateEndpointsForBackends(2, 3)},
4074 });
4075 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
4076 balancers_[0]->ads_service()->SetEdsResource(
4077 BuildEdsResource(args1, kNewEdsService1Name));
4078 balancers_[0]->ads_service()->SetEdsResource(
4079 BuildEdsResource(args2, kNewEdsService2Name));
4080 // Populate new CDS resources.
4081 Cluster new_cluster1 = default_cluster_;
4082 new_cluster1.set_name(kNewCluster1Name);
4083 new_cluster1.mutable_eds_cluster_config()->set_service_name(
4084 kNewEdsService1Name);
4085 balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
4086 Cluster new_cluster2 = default_cluster_;
4087 new_cluster2.set_name(kNewCluster2Name);
4088 new_cluster2.mutable_eds_cluster_config()->set_service_name(
4089 kNewEdsService2Name);
4090 balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
4091 // Populating Route Configurations for LDS.
4092 RouteConfiguration new_route_config = default_route_config_;
4093 auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4094 route1->mutable_match()->set_prefix("");
4095 auto* weighted_cluster1 =
4096 route1->mutable_route()->mutable_weighted_clusters()->add_clusters();
4097 weighted_cluster1->set_name(kNewCluster1Name);
4098 weighted_cluster1->mutable_weight()->set_value(kWeight75);
4099 auto* weighted_cluster2 =
4100 route1->mutable_route()->mutable_weighted_clusters()->add_clusters();
4101 weighted_cluster2->set_name(kNewCluster2Name);
4102 weighted_cluster2->mutable_weight()->set_value(kWeight25);
4103 route1->mutable_route()
4104 ->mutable_weighted_clusters()
4105 ->mutable_total_weight()
4106 ->set_value(kWeight75 + kWeight25);
4107 SetRouteConfiguration(0, new_route_config);
4108 WaitForAllBackends(1, 3);
4109 CheckRpcSendOk(kNumEchoRpcs);
4110 // Make sure RPCs all go to the correct backend.
4111 EXPECT_EQ(0, backends_[0]->backend_service()->request_count());
4112 const int weight_75_request_count =
4113 backends_[1]->backend_service()->request_count();
4114 const int weight_25_request_count =
4115 backends_[2]->backend_service()->request_count();
4116 gpr_log(GPR_INFO, "target_75 received %d rpcs and target_25 received %d rpcs",
4117 weight_75_request_count, weight_25_request_count);
4118 EXPECT_THAT(static_cast<double>(weight_75_request_count) / kNumEchoRpcs,
4119 ::testing::DoubleNear(kWeight75Percent, kErrorTolerance));
4120 EXPECT_THAT(static_cast<double>(weight_25_request_count) / kNumEchoRpcs,
4121 ::testing::DoubleNear(kWeight25Percent, kErrorTolerance));
4122 }
4123
TEST_P(LdsRdsTest,XdsRoutingWeightedClusterUpdateWeights)4124 TEST_P(LdsRdsTest, XdsRoutingWeightedClusterUpdateWeights) {
4125 const char* kNewCluster1Name = "new_cluster_1";
4126 const char* kNewEdsService1Name = "new_eds_service_name_1";
4127 const char* kNewCluster2Name = "new_cluster_2";
4128 const char* kNewEdsService2Name = "new_eds_service_name_2";
4129 const char* kNewCluster3Name = "new_cluster_3";
4130 const char* kNewEdsService3Name = "new_eds_service_name_3";
4131 const size_t kNumEchoRpcs = 10;
4132 const size_t kWeight75 = 75;
4133 const size_t kWeight25 = 25;
4134 const size_t kWeight50 = 50;
4135 const double kErrorTolerance = 0.05;
4136 const double kWeight75Percent = static_cast<double>(kWeight75) / 100;
4137 const double kWeight25Percent = static_cast<double>(kWeight25) / 100;
4138 const double kWeight50Percent = static_cast<double>(kWeight50) / 100;
4139 const size_t kNumEcho1Rpcs7525 =
4140 ComputeIdealNumRpcs(kWeight75Percent, kErrorTolerance);
4141 const size_t kNumEcho1Rpcs5050 =
4142 ComputeIdealNumRpcs(kWeight50Percent, kErrorTolerance);
4143 SetNextResolution({});
4144 SetNextResolutionForLbChannelAllBalancers();
4145 // Populate new EDS resources.
4146 EdsResourceArgs args({
4147 {"locality0", CreateEndpointsForBackends(0, 1)},
4148 });
4149 EdsResourceArgs args1({
4150 {"locality0", CreateEndpointsForBackends(1, 2)},
4151 });
4152 EdsResourceArgs args2({
4153 {"locality0", CreateEndpointsForBackends(2, 3)},
4154 });
4155 EdsResourceArgs args3({
4156 {"locality0", CreateEndpointsForBackends(3, 4)},
4157 });
4158 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
4159 balancers_[0]->ads_service()->SetEdsResource(
4160 BuildEdsResource(args1, kNewEdsService1Name));
4161 balancers_[0]->ads_service()->SetEdsResource(
4162 BuildEdsResource(args2, kNewEdsService2Name));
4163 balancers_[0]->ads_service()->SetEdsResource(
4164 BuildEdsResource(args3, kNewEdsService3Name));
4165 // Populate new CDS resources.
4166 Cluster new_cluster1 = default_cluster_;
4167 new_cluster1.set_name(kNewCluster1Name);
4168 new_cluster1.mutable_eds_cluster_config()->set_service_name(
4169 kNewEdsService1Name);
4170 balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
4171 Cluster new_cluster2 = default_cluster_;
4172 new_cluster2.set_name(kNewCluster2Name);
4173 new_cluster2.mutable_eds_cluster_config()->set_service_name(
4174 kNewEdsService2Name);
4175 balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
4176 Cluster new_cluster3 = default_cluster_;
4177 new_cluster3.set_name(kNewCluster3Name);
4178 new_cluster3.mutable_eds_cluster_config()->set_service_name(
4179 kNewEdsService3Name);
4180 balancers_[0]->ads_service()->SetCdsResource(new_cluster3);
4181 // Populating Route Configurations.
4182 RouteConfiguration new_route_config = default_route_config_;
4183 auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4184 route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
4185 auto* weighted_cluster1 =
4186 route1->mutable_route()->mutable_weighted_clusters()->add_clusters();
4187 weighted_cluster1->set_name(kNewCluster1Name);
4188 weighted_cluster1->mutable_weight()->set_value(kWeight75);
4189 auto* weighted_cluster2 =
4190 route1->mutable_route()->mutable_weighted_clusters()->add_clusters();
4191 weighted_cluster2->set_name(kNewCluster2Name);
4192 weighted_cluster2->mutable_weight()->set_value(kWeight25);
4193 route1->mutable_route()
4194 ->mutable_weighted_clusters()
4195 ->mutable_total_weight()
4196 ->set_value(kWeight75 + kWeight25);
4197 auto* default_route = new_route_config.mutable_virtual_hosts(0)->add_routes();
4198 default_route->mutable_match()->set_prefix("");
4199 default_route->mutable_route()->set_cluster(kDefaultClusterName);
4200 SetRouteConfiguration(0, new_route_config);
4201 WaitForAllBackends(0, 1);
4202 WaitForAllBackends(1, 3, WaitForBackendOptions(),
4203 RpcOptions().set_rpc_service(SERVICE_ECHO1));
4204 CheckRpcSendOk(kNumEchoRpcs);
4205 CheckRpcSendOk(kNumEcho1Rpcs7525,
4206 RpcOptions().set_rpc_service(SERVICE_ECHO1));
4207 // Make sure RPCs all go to the correct backend.
4208 EXPECT_EQ(kNumEchoRpcs, backends_[0]->backend_service()->request_count());
4209 EXPECT_EQ(0, backends_[0]->backend_service1()->request_count());
4210 EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
4211 const int weight_75_request_count =
4212 backends_[1]->backend_service1()->request_count();
4213 EXPECT_EQ(0, backends_[1]->backend_service2()->request_count());
4214 EXPECT_EQ(0, backends_[2]->backend_service()->request_count());
4215 const int weight_25_request_count =
4216 backends_[2]->backend_service1()->request_count();
4217 EXPECT_EQ(0, backends_[3]->backend_service()->request_count());
4218 EXPECT_EQ(0, backends_[3]->backend_service1()->request_count());
4219 gpr_log(GPR_INFO, "target_75 received %d rpcs and target_25 received %d rpcs",
4220 weight_75_request_count, weight_25_request_count);
4221 EXPECT_THAT(static_cast<double>(weight_75_request_count) / kNumEcho1Rpcs7525,
4222 ::testing::DoubleNear(kWeight75Percent, kErrorTolerance));
4223 EXPECT_THAT(static_cast<double>(weight_25_request_count) / kNumEcho1Rpcs7525,
4224 ::testing::DoubleNear(kWeight25Percent, kErrorTolerance));
4225 // Change Route Configurations: same clusters different weights.
4226 weighted_cluster1->mutable_weight()->set_value(kWeight50);
4227 weighted_cluster2->mutable_weight()->set_value(kWeight50);
4228 // Change default route to a new cluster to help to identify when new
4229 // polices are seen by the client.
4230 default_route->mutable_route()->set_cluster(kNewCluster3Name);
4231 SetRouteConfiguration(0, new_route_config);
4232 ResetBackendCounters();
4233 WaitForAllBackends(3, 4);
4234 CheckRpcSendOk(kNumEchoRpcs);
4235 CheckRpcSendOk(kNumEcho1Rpcs5050,
4236 RpcOptions().set_rpc_service(SERVICE_ECHO1));
4237 // Make sure RPCs all go to the correct backend.
4238 EXPECT_EQ(0, backends_[0]->backend_service()->request_count());
4239 EXPECT_EQ(0, backends_[0]->backend_service1()->request_count());
4240 EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
4241 const int weight_50_request_count_1 =
4242 backends_[1]->backend_service1()->request_count();
4243 EXPECT_EQ(0, backends_[2]->backend_service()->request_count());
4244 const int weight_50_request_count_2 =
4245 backends_[2]->backend_service1()->request_count();
4246 EXPECT_EQ(kNumEchoRpcs, backends_[3]->backend_service()->request_count());
4247 EXPECT_EQ(0, backends_[3]->backend_service1()->request_count());
4248 EXPECT_THAT(
4249 static_cast<double>(weight_50_request_count_1) / kNumEcho1Rpcs5050,
4250 ::testing::DoubleNear(kWeight50Percent, kErrorTolerance));
4251 EXPECT_THAT(
4252 static_cast<double>(weight_50_request_count_2) / kNumEcho1Rpcs5050,
4253 ::testing::DoubleNear(kWeight50Percent, kErrorTolerance));
4254 }
4255
TEST_P(LdsRdsTest,XdsRoutingWeightedClusterUpdateClusters)4256 TEST_P(LdsRdsTest, XdsRoutingWeightedClusterUpdateClusters) {
4257 const char* kNewCluster1Name = "new_cluster_1";
4258 const char* kNewEdsService1Name = "new_eds_service_name_1";
4259 const char* kNewCluster2Name = "new_cluster_2";
4260 const char* kNewEdsService2Name = "new_eds_service_name_2";
4261 const char* kNewCluster3Name = "new_cluster_3";
4262 const char* kNewEdsService3Name = "new_eds_service_name_3";
4263 const size_t kNumEchoRpcs = 10;
4264 const size_t kWeight75 = 75;
4265 const size_t kWeight25 = 25;
4266 const size_t kWeight50 = 50;
4267 const double kErrorTolerance = 0.05;
4268 const double kWeight75Percent = static_cast<double>(kWeight75) / 100;
4269 const double kWeight25Percent = static_cast<double>(kWeight25) / 100;
4270 const double kWeight50Percent = static_cast<double>(kWeight50) / 100;
4271 const size_t kNumEcho1Rpcs7525 =
4272 ComputeIdealNumRpcs(kWeight75Percent, kErrorTolerance);
4273 const size_t kNumEcho1Rpcs5050 =
4274 ComputeIdealNumRpcs(kWeight50Percent, kErrorTolerance);
4275 SetNextResolution({});
4276 SetNextResolutionForLbChannelAllBalancers();
4277 // Populate new EDS resources.
4278 EdsResourceArgs args({
4279 {"locality0", CreateEndpointsForBackends(0, 1)},
4280 });
4281 EdsResourceArgs args1({
4282 {"locality0", CreateEndpointsForBackends(1, 2)},
4283 });
4284 EdsResourceArgs args2({
4285 {"locality0", CreateEndpointsForBackends(2, 3)},
4286 });
4287 EdsResourceArgs args3({
4288 {"locality0", CreateEndpointsForBackends(3, 4)},
4289 });
4290 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
4291 balancers_[0]->ads_service()->SetEdsResource(
4292 BuildEdsResource(args1, kNewEdsService1Name));
4293 balancers_[0]->ads_service()->SetEdsResource(
4294 BuildEdsResource(args2, kNewEdsService2Name));
4295 balancers_[0]->ads_service()->SetEdsResource(
4296 BuildEdsResource(args3, kNewEdsService3Name));
4297 // Populate new CDS resources.
4298 Cluster new_cluster1 = default_cluster_;
4299 new_cluster1.set_name(kNewCluster1Name);
4300 new_cluster1.mutable_eds_cluster_config()->set_service_name(
4301 kNewEdsService1Name);
4302 balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
4303 Cluster new_cluster2 = default_cluster_;
4304 new_cluster2.set_name(kNewCluster2Name);
4305 new_cluster2.mutable_eds_cluster_config()->set_service_name(
4306 kNewEdsService2Name);
4307 balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
4308 Cluster new_cluster3 = default_cluster_;
4309 new_cluster3.set_name(kNewCluster3Name);
4310 new_cluster3.mutable_eds_cluster_config()->set_service_name(
4311 kNewEdsService3Name);
4312 balancers_[0]->ads_service()->SetCdsResource(new_cluster3);
4313 // Populating Route Configurations.
4314 RouteConfiguration new_route_config = default_route_config_;
4315 auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4316 route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
4317 auto* weighted_cluster1 =
4318 route1->mutable_route()->mutable_weighted_clusters()->add_clusters();
4319 weighted_cluster1->set_name(kNewCluster1Name);
4320 weighted_cluster1->mutable_weight()->set_value(kWeight75);
4321 auto* weighted_cluster2 =
4322 route1->mutable_route()->mutable_weighted_clusters()->add_clusters();
4323 weighted_cluster2->set_name(kDefaultClusterName);
4324 weighted_cluster2->mutable_weight()->set_value(kWeight25);
4325 route1->mutable_route()
4326 ->mutable_weighted_clusters()
4327 ->mutable_total_weight()
4328 ->set_value(kWeight75 + kWeight25);
4329 auto* default_route = new_route_config.mutable_virtual_hosts(0)->add_routes();
4330 default_route->mutable_match()->set_prefix("");
4331 default_route->mutable_route()->set_cluster(kDefaultClusterName);
4332 SetRouteConfiguration(0, new_route_config);
4333 WaitForBackend(0);
4334 WaitForBackend(1, WaitForBackendOptions(),
4335 RpcOptions().set_rpc_service(SERVICE_ECHO1));
4336 CheckRpcSendOk(kNumEchoRpcs);
4337 CheckRpcSendOk(kNumEcho1Rpcs7525,
4338 RpcOptions().set_rpc_service(SERVICE_ECHO1));
4339 // Make sure RPCs all go to the correct backend.
4340 EXPECT_EQ(kNumEchoRpcs, backends_[0]->backend_service()->request_count());
4341 int weight_25_request_count =
4342 backends_[0]->backend_service1()->request_count();
4343 EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
4344 int weight_75_request_count =
4345 backends_[1]->backend_service1()->request_count();
4346 EXPECT_EQ(0, backends_[2]->backend_service()->request_count());
4347 EXPECT_EQ(0, backends_[2]->backend_service1()->request_count());
4348 EXPECT_EQ(0, backends_[3]->backend_service()->request_count());
4349 EXPECT_EQ(0, backends_[3]->backend_service1()->request_count());
4350 gpr_log(GPR_INFO, "target_75 received %d rpcs and target_25 received %d rpcs",
4351 weight_75_request_count, weight_25_request_count);
4352 EXPECT_THAT(static_cast<double>(weight_75_request_count) / kNumEcho1Rpcs7525,
4353 ::testing::DoubleNear(kWeight75Percent, kErrorTolerance));
4354 EXPECT_THAT(static_cast<double>(weight_25_request_count) / kNumEcho1Rpcs7525,
4355 ::testing::DoubleNear(kWeight25Percent, kErrorTolerance));
4356 // Change Route Configurations: new set of clusters with different weights.
4357 weighted_cluster1->mutable_weight()->set_value(kWeight50);
4358 weighted_cluster2->set_name(kNewCluster2Name);
4359 weighted_cluster2->mutable_weight()->set_value(kWeight50);
4360 SetRouteConfiguration(0, new_route_config);
4361 ResetBackendCounters();
4362 WaitForBackend(2, WaitForBackendOptions(),
4363 RpcOptions().set_rpc_service(SERVICE_ECHO1));
4364 CheckRpcSendOk(kNumEchoRpcs);
4365 CheckRpcSendOk(kNumEcho1Rpcs5050,
4366 RpcOptions().set_rpc_service(SERVICE_ECHO1));
4367 // Make sure RPCs all go to the correct backend.
4368 EXPECT_EQ(kNumEchoRpcs, backends_[0]->backend_service()->request_count());
4369 EXPECT_EQ(0, backends_[0]->backend_service1()->request_count());
4370 EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
4371 const int weight_50_request_count_1 =
4372 backends_[1]->backend_service1()->request_count();
4373 EXPECT_EQ(0, backends_[2]->backend_service()->request_count());
4374 const int weight_50_request_count_2 =
4375 backends_[2]->backend_service1()->request_count();
4376 EXPECT_EQ(0, backends_[3]->backend_service()->request_count());
4377 EXPECT_EQ(0, backends_[3]->backend_service1()->request_count());
4378 EXPECT_THAT(
4379 static_cast<double>(weight_50_request_count_1) / kNumEcho1Rpcs5050,
4380 ::testing::DoubleNear(kWeight50Percent, kErrorTolerance));
4381 EXPECT_THAT(
4382 static_cast<double>(weight_50_request_count_2) / kNumEcho1Rpcs5050,
4383 ::testing::DoubleNear(kWeight50Percent, kErrorTolerance));
4384 // Change Route Configurations.
4385 weighted_cluster1->mutable_weight()->set_value(kWeight75);
4386 weighted_cluster2->set_name(kNewCluster3Name);
4387 weighted_cluster2->mutable_weight()->set_value(kWeight25);
4388 SetRouteConfiguration(0, new_route_config);
4389 ResetBackendCounters();
4390 WaitForBackend(3, WaitForBackendOptions(),
4391 RpcOptions().set_rpc_service(SERVICE_ECHO1));
4392 CheckRpcSendOk(kNumEchoRpcs);
4393 CheckRpcSendOk(kNumEcho1Rpcs7525,
4394 RpcOptions().set_rpc_service(SERVICE_ECHO1));
4395 // Make sure RPCs all go to the correct backend.
4396 EXPECT_EQ(kNumEchoRpcs, backends_[0]->backend_service()->request_count());
4397 EXPECT_EQ(0, backends_[0]->backend_service1()->request_count());
4398 EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
4399 weight_75_request_count = backends_[1]->backend_service1()->request_count();
4400 EXPECT_EQ(0, backends_[2]->backend_service()->request_count());
4401 EXPECT_EQ(0, backends_[2]->backend_service1()->request_count());
4402 EXPECT_EQ(0, backends_[3]->backend_service()->request_count());
4403 weight_25_request_count = backends_[3]->backend_service1()->request_count();
4404 gpr_log(GPR_INFO, "target_75 received %d rpcs and target_25 received %d rpcs",
4405 weight_75_request_count, weight_25_request_count);
4406 EXPECT_THAT(static_cast<double>(weight_75_request_count) / kNumEcho1Rpcs7525,
4407 ::testing::DoubleNear(kWeight75Percent, kErrorTolerance));
4408 EXPECT_THAT(static_cast<double>(weight_25_request_count) / kNumEcho1Rpcs7525,
4409 ::testing::DoubleNear(kWeight25Percent, kErrorTolerance));
4410 }
4411
TEST_P(LdsRdsTest,XdsRoutingClusterUpdateClusters)4412 TEST_P(LdsRdsTest, XdsRoutingClusterUpdateClusters) {
4413 const char* kNewClusterName = "new_cluster";
4414 const char* kNewEdsServiceName = "new_eds_service_name";
4415 const size_t kNumEchoRpcs = 5;
4416 SetNextResolution({});
4417 SetNextResolutionForLbChannelAllBalancers();
4418 // Populate new EDS resources.
4419 EdsResourceArgs args({
4420 {"locality0", CreateEndpointsForBackends(0, 1)},
4421 });
4422 EdsResourceArgs args1({
4423 {"locality0", CreateEndpointsForBackends(1, 2)},
4424 });
4425 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
4426 balancers_[0]->ads_service()->SetEdsResource(
4427 BuildEdsResource(args1, kNewEdsServiceName));
4428 // Populate new CDS resources.
4429 Cluster new_cluster = default_cluster_;
4430 new_cluster.set_name(kNewClusterName);
4431 new_cluster.mutable_eds_cluster_config()->set_service_name(
4432 kNewEdsServiceName);
4433 balancers_[0]->ads_service()->SetCdsResource(new_cluster);
4434 // Send Route Configuration.
4435 RouteConfiguration new_route_config = default_route_config_;
4436 SetRouteConfiguration(0, new_route_config);
4437 WaitForAllBackends(0, 1);
4438 CheckRpcSendOk(kNumEchoRpcs);
4439 // Make sure RPCs all go to the correct backend.
4440 EXPECT_EQ(kNumEchoRpcs, backends_[0]->backend_service()->request_count());
4441 // Change Route Configurations: new default cluster.
4442 auto* default_route =
4443 new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4444 default_route->mutable_route()->set_cluster(kNewClusterName);
4445 SetRouteConfiguration(0, new_route_config);
4446 WaitForAllBackends(1, 2);
4447 CheckRpcSendOk(kNumEchoRpcs);
4448 // Make sure RPCs all go to the correct backend.
4449 EXPECT_EQ(kNumEchoRpcs, backends_[1]->backend_service()->request_count());
4450 }
4451
TEST_P(LdsRdsTest,XdsRoutingClusterUpdateClustersWithPickingDelays)4452 TEST_P(LdsRdsTest, XdsRoutingClusterUpdateClustersWithPickingDelays) {
4453 const char* kNewClusterName = "new_cluster";
4454 const char* kNewEdsServiceName = "new_eds_service_name";
4455 SetNextResolution({});
4456 SetNextResolutionForLbChannelAllBalancers();
4457 // Populate new EDS resources.
4458 EdsResourceArgs args({
4459 {"locality0", CreateEndpointsForBackends(0, 1)},
4460 });
4461 EdsResourceArgs args1({
4462 {"locality0", CreateEndpointsForBackends(1, 2)},
4463 });
4464 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
4465 balancers_[0]->ads_service()->SetEdsResource(
4466 BuildEdsResource(args1, kNewEdsServiceName));
4467 // Populate new CDS resources.
4468 Cluster new_cluster = default_cluster_;
4469 new_cluster.set_name(kNewClusterName);
4470 new_cluster.mutable_eds_cluster_config()->set_service_name(
4471 kNewEdsServiceName);
4472 balancers_[0]->ads_service()->SetCdsResource(new_cluster);
4473 // Bring down the current backend: 0, this will delay route picking time,
4474 // resulting in un-committed RPCs.
4475 ShutdownBackend(0);
4476 // Send a RouteConfiguration with a default route that points to
4477 // backend 0.
4478 RouteConfiguration new_route_config = default_route_config_;
4479 SetRouteConfiguration(0, new_route_config);
4480 // Send exactly one RPC with no deadline and with wait_for_ready=true.
4481 // This RPC will not complete until after backend 0 is started.
4482 std::thread sending_rpc([this]() {
4483 CheckRpcSendOk(1, RpcOptions().set_wait_for_ready(true).set_timeout_ms(0));
4484 });
4485 // Send a non-wait_for_ready RPC which should fail, this will tell us
4486 // that the client has received the update and attempted to connect.
4487 const Status status = SendRpc(RpcOptions().set_timeout_ms(0));
4488 EXPECT_FALSE(status.ok());
4489 // Send a update RouteConfiguration to use backend 1.
4490 auto* default_route =
4491 new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4492 default_route->mutable_route()->set_cluster(kNewClusterName);
4493 SetRouteConfiguration(0, new_route_config);
4494 // Wait for RPCs to go to the new backend: 1, this ensures that the client
4495 // has processed the update.
4496 WaitForBackend(
4497 1, WaitForBackendOptions().set_reset_counters(false).set_allow_failures(
4498 true));
4499 // Bring up the previous backend: 0, this will allow the delayed RPC to
4500 // finally call on_call_committed upon completion.
4501 StartBackend(0);
4502 sending_rpc.join();
4503 // Make sure RPCs go to the correct backend:
4504 EXPECT_EQ(1, backends_[0]->backend_service()->request_count());
4505 EXPECT_EQ(1, backends_[1]->backend_service()->request_count());
4506 }
4507
TEST_P(LdsRdsTest,XdsRoutingApplyXdsTimeout)4508 TEST_P(LdsRdsTest, XdsRoutingApplyXdsTimeout) {
4509 const int64_t kTimeoutMillis = 500;
4510 const int64_t kTimeoutNano = kTimeoutMillis * 1000000;
4511 const int64_t kTimeoutGrpcTimeoutHeaderMaxSecond = 1;
4512 const int64_t kTimeoutMaxStreamDurationSecond = 2;
4513 const int64_t kTimeoutHttpMaxStreamDurationSecond = 3;
4514 const int64_t kTimeoutApplicationSecond = 4;
4515 const char* kNewCluster1Name = "new_cluster_1";
4516 const char* kNewEdsService1Name = "new_eds_service_name_1";
4517 const char* kNewCluster2Name = "new_cluster_2";
4518 const char* kNewEdsService2Name = "new_eds_service_name_2";
4519 const char* kNewCluster3Name = "new_cluster_3";
4520 const char* kNewEdsService3Name = "new_eds_service_name_3";
4521 SetNextResolution({});
4522 SetNextResolutionForLbChannelAllBalancers();
4523 // Populate new EDS resources.
4524 EdsResourceArgs args({{"locality0", {MakeNonExistantEndpoint()}}});
4525 EdsResourceArgs args1({{"locality0", {MakeNonExistantEndpoint()}}});
4526 EdsResourceArgs args2({{"locality0", {MakeNonExistantEndpoint()}}});
4527 EdsResourceArgs args3({{"locality0", {MakeNonExistantEndpoint()}}});
4528 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
4529 balancers_[0]->ads_service()->SetEdsResource(
4530 BuildEdsResource(args1, kNewEdsService1Name));
4531 balancers_[0]->ads_service()->SetEdsResource(
4532 BuildEdsResource(args2, kNewEdsService2Name));
4533 balancers_[0]->ads_service()->SetEdsResource(
4534 BuildEdsResource(args3, kNewEdsService3Name));
4535 // Populate new CDS resources.
4536 Cluster new_cluster1 = default_cluster_;
4537 new_cluster1.set_name(kNewCluster1Name);
4538 new_cluster1.mutable_eds_cluster_config()->set_service_name(
4539 kNewEdsService1Name);
4540 balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
4541 Cluster new_cluster2 = default_cluster_;
4542 new_cluster2.set_name(kNewCluster2Name);
4543 new_cluster2.mutable_eds_cluster_config()->set_service_name(
4544 kNewEdsService2Name);
4545 balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
4546 Cluster new_cluster3 = default_cluster_;
4547 new_cluster3.set_name(kNewCluster3Name);
4548 new_cluster3.mutable_eds_cluster_config()->set_service_name(
4549 kNewEdsService3Name);
4550 balancers_[0]->ads_service()->SetCdsResource(new_cluster3);
4551 // Construct listener.
4552 auto listener = default_listener_;
4553 HttpConnectionManager http_connection_manager;
4554 listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
4555 &http_connection_manager);
4556 // Set up HTTP max_stream_duration of 3.5 seconds
4557 auto* duration =
4558 http_connection_manager.mutable_common_http_protocol_options()
4559 ->mutable_max_stream_duration();
4560 duration->set_seconds(kTimeoutHttpMaxStreamDurationSecond);
4561 duration->set_nanos(kTimeoutNano);
4562 listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
4563 http_connection_manager);
4564 // Construct route config.
4565 RouteConfiguration new_route_config = default_route_config_;
4566 // route 1: Set max_stream_duration of 2.5 seconds, Set
4567 // grpc_timeout_header_max of 1.5
4568 auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4569 route1->mutable_match()->set_path("/grpc.testing.EchoTest1Service/Echo1");
4570 route1->mutable_route()->set_cluster(kNewCluster1Name);
4571 auto* max_stream_duration =
4572 route1->mutable_route()->mutable_max_stream_duration();
4573 duration = max_stream_duration->mutable_max_stream_duration();
4574 duration->set_seconds(kTimeoutMaxStreamDurationSecond);
4575 duration->set_nanos(kTimeoutNano);
4576 duration = max_stream_duration->mutable_grpc_timeout_header_max();
4577 duration->set_seconds(kTimeoutGrpcTimeoutHeaderMaxSecond);
4578 duration->set_nanos(kTimeoutNano);
4579 // route 2: Set max_stream_duration of 2.5 seconds
4580 auto* route2 = new_route_config.mutable_virtual_hosts(0)->add_routes();
4581 route2->mutable_match()->set_path("/grpc.testing.EchoTest2Service/Echo2");
4582 route2->mutable_route()->set_cluster(kNewCluster2Name);
4583 max_stream_duration = route2->mutable_route()->mutable_max_stream_duration();
4584 duration = max_stream_duration->mutable_max_stream_duration();
4585 duration->set_seconds(kTimeoutMaxStreamDurationSecond);
4586 duration->set_nanos(kTimeoutNano);
4587 // route 3: No timeout values in route configuration
4588 auto* route3 = new_route_config.mutable_virtual_hosts(0)->add_routes();
4589 route3->mutable_match()->set_path("/grpc.testing.EchoTestService/Echo");
4590 route3->mutable_route()->set_cluster(kNewCluster3Name);
4591 // Set listener and route config.
4592 SetListenerAndRouteConfiguration(0, std::move(listener), new_route_config);
4593 // Test grpc_timeout_header_max of 1.5 seconds applied
4594 grpc_millis t0 = NowFromCycleCounter();
4595 grpc_millis t1 =
4596 t0 + kTimeoutGrpcTimeoutHeaderMaxSecond * 1000 + kTimeoutMillis;
4597 grpc_millis t2 = t0 + kTimeoutMaxStreamDurationSecond * 1000 + kTimeoutMillis;
4598 CheckRpcSendFailure(
4599 CheckRpcSendFailureOptions()
4600 .set_rpc_options(
4601 RpcOptions()
4602 .set_rpc_service(SERVICE_ECHO1)
4603 .set_rpc_method(METHOD_ECHO1)
4604 .set_wait_for_ready(true)
4605 .set_timeout_ms(kTimeoutApplicationSecond * 1000))
4606 .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
4607 t0 = NowFromCycleCounter();
4608 EXPECT_GE(t0, t1);
4609 EXPECT_LT(t0, t2);
4610 // Test max_stream_duration of 2.5 seconds applied
4611 t0 = NowFromCycleCounter();
4612 t1 = t0 + kTimeoutMaxStreamDurationSecond * 1000 + kTimeoutMillis;
4613 t2 = t0 + kTimeoutHttpMaxStreamDurationSecond * 1000 + kTimeoutMillis;
4614 CheckRpcSendFailure(
4615 CheckRpcSendFailureOptions()
4616 .set_rpc_options(
4617 RpcOptions()
4618 .set_rpc_service(SERVICE_ECHO2)
4619 .set_rpc_method(METHOD_ECHO2)
4620 .set_wait_for_ready(true)
4621 .set_timeout_ms(kTimeoutApplicationSecond * 1000))
4622 .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
4623 t0 = NowFromCycleCounter();
4624 EXPECT_GE(t0, t1);
4625 EXPECT_LT(t0, t2);
4626 // Test http_stream_duration of 3.5 seconds applied
4627 t0 = NowFromCycleCounter();
4628 t1 = t0 + kTimeoutHttpMaxStreamDurationSecond * 1000 + kTimeoutMillis;
4629 t2 = t0 + kTimeoutApplicationSecond * 1000 + kTimeoutMillis;
4630 CheckRpcSendFailure(
4631 CheckRpcSendFailureOptions()
4632 .set_rpc_options(RpcOptions().set_wait_for_ready(true).set_timeout_ms(
4633 kTimeoutApplicationSecond * 1000))
4634 .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
4635 t0 = NowFromCycleCounter();
4636 EXPECT_GE(t0, t1);
4637 EXPECT_LT(t0, t2);
4638 }
4639
TEST_P(LdsRdsTest,XdsRoutingApplyApplicationTimeoutWhenXdsTimeoutExplicit0)4640 TEST_P(LdsRdsTest, XdsRoutingApplyApplicationTimeoutWhenXdsTimeoutExplicit0) {
4641 const int64_t kTimeoutNano = 500000000;
4642 const int64_t kTimeoutMaxStreamDurationSecond = 2;
4643 const int64_t kTimeoutHttpMaxStreamDurationSecond = 3;
4644 const int64_t kTimeoutApplicationSecond = 4;
4645 const char* kNewCluster1Name = "new_cluster_1";
4646 const char* kNewEdsService1Name = "new_eds_service_name_1";
4647 const char* kNewCluster2Name = "new_cluster_2";
4648 const char* kNewEdsService2Name = "new_eds_service_name_2";
4649 SetNextResolution({});
4650 SetNextResolutionForLbChannelAllBalancers();
4651 // Populate new EDS resources.
4652 EdsResourceArgs args({{"locality0", {MakeNonExistantEndpoint()}}});
4653 EdsResourceArgs args1({{"locality0", {MakeNonExistantEndpoint()}}});
4654 EdsResourceArgs args2({{"locality0", {MakeNonExistantEndpoint()}}});
4655 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
4656 balancers_[0]->ads_service()->SetEdsResource(
4657 BuildEdsResource(args1, kNewEdsService1Name));
4658 balancers_[0]->ads_service()->SetEdsResource(
4659 BuildEdsResource(args2, kNewEdsService2Name));
4660 // Populate new CDS resources.
4661 Cluster new_cluster1 = default_cluster_;
4662 new_cluster1.set_name(kNewCluster1Name);
4663 new_cluster1.mutable_eds_cluster_config()->set_service_name(
4664 kNewEdsService1Name);
4665 balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
4666 Cluster new_cluster2 = default_cluster_;
4667 new_cluster2.set_name(kNewCluster2Name);
4668 new_cluster2.mutable_eds_cluster_config()->set_service_name(
4669 kNewEdsService2Name);
4670 balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
4671 // Construct listener.
4672 auto listener = default_listener_;
4673 HttpConnectionManager http_connection_manager;
4674 listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
4675 &http_connection_manager);
4676 // Set up HTTP max_stream_duration of 3.5 seconds
4677 auto* duration =
4678 http_connection_manager.mutable_common_http_protocol_options()
4679 ->mutable_max_stream_duration();
4680 duration->set_seconds(kTimeoutHttpMaxStreamDurationSecond);
4681 duration->set_nanos(kTimeoutNano);
4682 listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
4683 http_connection_manager);
4684 // Construct route config.
4685 RouteConfiguration new_route_config = default_route_config_;
4686 // route 1: Set max_stream_duration of 2.5 seconds, Set
4687 // grpc_timeout_header_max of 0
4688 auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4689 route1->mutable_match()->set_path("/grpc.testing.EchoTest1Service/Echo1");
4690 route1->mutable_route()->set_cluster(kNewCluster1Name);
4691 auto* max_stream_duration =
4692 route1->mutable_route()->mutable_max_stream_duration();
4693 duration = max_stream_duration->mutable_max_stream_duration();
4694 duration->set_seconds(kTimeoutMaxStreamDurationSecond);
4695 duration->set_nanos(kTimeoutNano);
4696 duration = max_stream_duration->mutable_grpc_timeout_header_max();
4697 duration->set_seconds(0);
4698 duration->set_nanos(0);
4699 // route 2: Set max_stream_duration to 0
4700 auto* route2 = new_route_config.mutable_virtual_hosts(0)->add_routes();
4701 route2->mutable_match()->set_path("/grpc.testing.EchoTest2Service/Echo2");
4702 route2->mutable_route()->set_cluster(kNewCluster2Name);
4703 max_stream_duration = route2->mutable_route()->mutable_max_stream_duration();
4704 duration = max_stream_duration->mutable_max_stream_duration();
4705 duration->set_seconds(0);
4706 duration->set_nanos(0);
4707 // Set listener and route config.
4708 SetListenerAndRouteConfiguration(0, std::move(listener), new_route_config);
4709 // Test application timeout is applied for route 1
4710 auto t0 = system_clock::now();
4711 CheckRpcSendFailure(
4712 CheckRpcSendFailureOptions()
4713 .set_rpc_options(
4714 RpcOptions()
4715 .set_rpc_service(SERVICE_ECHO1)
4716 .set_rpc_method(METHOD_ECHO1)
4717 .set_wait_for_ready(true)
4718 .set_timeout_ms(kTimeoutApplicationSecond * 1000))
4719 .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
4720 auto ellapsed_nano_seconds =
4721 std::chrono::duration_cast<std::chrono::nanoseconds>(system_clock::now() -
4722 t0);
4723 EXPECT_GT(ellapsed_nano_seconds.count(),
4724 kTimeoutApplicationSecond * 1000000000);
4725 // Test application timeout is applied for route 2
4726 t0 = system_clock::now();
4727 CheckRpcSendFailure(
4728 CheckRpcSendFailureOptions()
4729 .set_rpc_options(
4730 RpcOptions()
4731 .set_rpc_service(SERVICE_ECHO2)
4732 .set_rpc_method(METHOD_ECHO2)
4733 .set_wait_for_ready(true)
4734 .set_timeout_ms(kTimeoutApplicationSecond * 1000))
4735 .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
4736 ellapsed_nano_seconds = std::chrono::duration_cast<std::chrono::nanoseconds>(
4737 system_clock::now() - t0);
4738 EXPECT_GT(ellapsed_nano_seconds.count(),
4739 kTimeoutApplicationSecond * 1000000000);
4740 }
4741
TEST_P(LdsRdsTest,XdsRoutingApplyApplicationTimeoutWhenHttpTimeoutExplicit0)4742 TEST_P(LdsRdsTest, XdsRoutingApplyApplicationTimeoutWhenHttpTimeoutExplicit0) {
4743 const int64_t kTimeoutApplicationSecond = 4;
4744 SetNextResolution({});
4745 SetNextResolutionForLbChannelAllBalancers();
4746 // Populate new EDS resources.
4747 EdsResourceArgs args({{"locality0", {MakeNonExistantEndpoint()}}});
4748 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
4749 auto listener = default_listener_;
4750 HttpConnectionManager http_connection_manager;
4751 listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
4752 &http_connection_manager);
4753 // Set up HTTP max_stream_duration to be explicit 0
4754 auto* duration =
4755 http_connection_manager.mutable_common_http_protocol_options()
4756 ->mutable_max_stream_duration();
4757 duration->set_seconds(0);
4758 duration->set_nanos(0);
4759 listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
4760 http_connection_manager);
4761 // Set listener and route config.
4762 SetListenerAndRouteConfiguration(0, std::move(listener),
4763 default_route_config_);
4764 // Test application timeout is applied for route 1
4765 auto t0 = system_clock::now();
4766 CheckRpcSendFailure(
4767 CheckRpcSendFailureOptions()
4768 .set_rpc_options(RpcOptions().set_wait_for_ready(true).set_timeout_ms(
4769 kTimeoutApplicationSecond * 1000))
4770 .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
4771 auto ellapsed_nano_seconds =
4772 std::chrono::duration_cast<std::chrono::nanoseconds>(system_clock::now() -
4773 t0);
4774 EXPECT_GT(ellapsed_nano_seconds.count(),
4775 kTimeoutApplicationSecond * 1000000000);
4776 }
4777
4778 // Test to ensure application-specified deadline won't be affected when
4779 // the xDS config does not specify a timeout.
TEST_P(LdsRdsTest,XdsRoutingWithOnlyApplicationTimeout)4780 TEST_P(LdsRdsTest, XdsRoutingWithOnlyApplicationTimeout) {
4781 const int64_t kTimeoutApplicationSecond = 4;
4782 SetNextResolution({});
4783 SetNextResolutionForLbChannelAllBalancers();
4784 // Populate new EDS resources.
4785 EdsResourceArgs args({{"locality0", {MakeNonExistantEndpoint()}}});
4786 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
4787 auto t0 = system_clock::now();
4788 CheckRpcSendFailure(
4789 CheckRpcSendFailureOptions()
4790 .set_rpc_options(RpcOptions().set_wait_for_ready(true).set_timeout_ms(
4791 kTimeoutApplicationSecond * 1000))
4792 .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
4793 auto ellapsed_nano_seconds =
4794 std::chrono::duration_cast<std::chrono::nanoseconds>(system_clock::now() -
4795 t0);
4796 EXPECT_GT(ellapsed_nano_seconds.count(),
4797 kTimeoutApplicationSecond * 1000000000);
4798 }
4799
TEST_P(LdsRdsTest,XdsRetryPolicyNumRetries)4800 TEST_P(LdsRdsTest, XdsRetryPolicyNumRetries) {
4801 const size_t kNumRetries = 3;
4802 SetNextResolution({});
4803 SetNextResolutionForLbChannelAllBalancers();
4804 // Populate new EDS resources.
4805 EdsResourceArgs args({
4806 {"locality0", CreateEndpointsForBackends(0, 1)},
4807 });
4808 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
4809 // Construct route config to set retry policy.
4810 RouteConfiguration new_route_config = default_route_config_;
4811 auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4812 auto* retry_policy = route1->mutable_route()->mutable_retry_policy();
4813 retry_policy->set_retry_on(
4814 "5xx,cancelled,deadline-exceeded,internal,resource-exhausted,"
4815 "unavailable");
4816 retry_policy->mutable_num_retries()->set_value(kNumRetries);
4817 SetRouteConfiguration(0, new_route_config);
4818 // Ensure we retried the correct number of times on all supported status.
4819 CheckRpcSendFailure(
4820 CheckRpcSendFailureOptions()
4821 .set_rpc_options(
4822 RpcOptions().set_server_expected_error(StatusCode::CANCELLED))
4823 .set_expected_error_code(StatusCode::CANCELLED));
4824 EXPECT_EQ(kNumRetries + 1, backends_[0]->backend_service()->request_count());
4825 ResetBackendCounters();
4826 CheckRpcSendFailure(
4827 CheckRpcSendFailureOptions()
4828 .set_rpc_options(RpcOptions().set_server_expected_error(
4829 StatusCode::DEADLINE_EXCEEDED))
4830 .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
4831 EXPECT_EQ(kNumRetries + 1, backends_[0]->backend_service()->request_count());
4832 ResetBackendCounters();
4833 CheckRpcSendFailure(
4834 CheckRpcSendFailureOptions()
4835 .set_rpc_options(
4836 RpcOptions().set_server_expected_error(StatusCode::INTERNAL))
4837 .set_expected_error_code(StatusCode::INTERNAL));
4838 EXPECT_EQ(kNumRetries + 1, backends_[0]->backend_service()->request_count());
4839 ResetBackendCounters();
4840 CheckRpcSendFailure(
4841 CheckRpcSendFailureOptions()
4842 .set_rpc_options(RpcOptions().set_server_expected_error(
4843 StatusCode::RESOURCE_EXHAUSTED))
4844 .set_expected_error_code(StatusCode::RESOURCE_EXHAUSTED));
4845 EXPECT_EQ(kNumRetries + 1, backends_[0]->backend_service()->request_count());
4846 ResetBackendCounters();
4847 CheckRpcSendFailure(
4848 CheckRpcSendFailureOptions()
4849 .set_rpc_options(
4850 RpcOptions().set_server_expected_error(StatusCode::UNAVAILABLE))
4851 .set_expected_error_code(StatusCode::UNAVAILABLE));
4852 EXPECT_EQ(kNumRetries + 1, backends_[0]->backend_service()->request_count());
4853 ResetBackendCounters();
4854 // Ensure we don't retry on an unsupported status.
4855 CheckRpcSendFailure(
4856 CheckRpcSendFailureOptions()
4857 .set_rpc_options(RpcOptions().set_server_expected_error(
4858 StatusCode::UNAUTHENTICATED))
4859 .set_expected_error_code(StatusCode::UNAUTHENTICATED));
4860 EXPECT_EQ(1, backends_[0]->backend_service()->request_count());
4861 }
4862
TEST_P(LdsRdsTest,XdsRetryPolicyAtVirtualHostLevel)4863 TEST_P(LdsRdsTest, XdsRetryPolicyAtVirtualHostLevel) {
4864 const size_t kNumRetries = 3;
4865 SetNextResolution({});
4866 SetNextResolutionForLbChannelAllBalancers();
4867 // Populate new EDS resources.
4868 EdsResourceArgs args({
4869 {"locality0", CreateEndpointsForBackends(0, 1)},
4870 });
4871 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
4872 // Construct route config to set retry policy.
4873 RouteConfiguration new_route_config = default_route_config_;
4874 auto* retry_policy =
4875 new_route_config.mutable_virtual_hosts(0)->mutable_retry_policy();
4876 retry_policy->set_retry_on(
4877 "cancelled,deadline-exceeded,internal,resource-exhausted,unavailable");
4878 retry_policy->mutable_num_retries()->set_value(kNumRetries);
4879 SetRouteConfiguration(0, new_route_config);
4880 // Ensure we retried the correct number of times on a supported status.
4881 CheckRpcSendFailure(
4882 CheckRpcSendFailureOptions()
4883 .set_rpc_options(RpcOptions().set_server_expected_error(
4884 StatusCode::DEADLINE_EXCEEDED))
4885 .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
4886 EXPECT_EQ(kNumRetries + 1, backends_[0]->backend_service()->request_count());
4887 }
4888
TEST_P(LdsRdsTest,XdsRetryPolicyLongBackOff)4889 TEST_P(LdsRdsTest, XdsRetryPolicyLongBackOff) {
4890 // Set num retries to 3, but due to longer back off, we expect only 1 retry
4891 // will take place.
4892 const size_t kNumRetries = 3;
4893 SetNextResolution({});
4894 SetNextResolutionForLbChannelAllBalancers();
4895 // Populate new EDS resources.
4896 EdsResourceArgs args({
4897 {"locality0", CreateEndpointsForBackends(0, 1)},
4898 });
4899 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
4900 // Construct route config to set retry policy.
4901 RouteConfiguration new_route_config = default_route_config_;
4902 auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4903 auto* retry_policy = route1->mutable_route()->mutable_retry_policy();
4904 retry_policy->set_retry_on(
4905 "5xx,cancelled,deadline-exceeded,internal,resource-exhausted,"
4906 "unavailable");
4907 retry_policy->mutable_num_retries()->set_value(kNumRetries);
4908 auto base_interval =
4909 retry_policy->mutable_retry_back_off()->mutable_base_interval();
4910 // Set backoff to 1 second, 1/2 of rpc timeout of 2 second.
4911 base_interval->set_seconds(1 * grpc_test_slowdown_factor());
4912 base_interval->set_nanos(0);
4913 SetRouteConfiguration(0, new_route_config);
4914 // No need to set max interval and just let it be the default of 10x of base.
4915 // We expect 1 retry before the RPC times out with DEADLINE_EXCEEDED.
4916 CheckRpcSendFailure(
4917 CheckRpcSendFailureOptions()
4918 .set_rpc_options(
4919 RpcOptions().set_timeout_ms(2500).set_server_expected_error(
4920 StatusCode::CANCELLED))
4921 .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
4922 EXPECT_EQ(1 + 1, backends_[0]->backend_service()->request_count());
4923 }
4924
TEST_P(LdsRdsTest,XdsRetryPolicyMaxBackOff)4925 TEST_P(LdsRdsTest, XdsRetryPolicyMaxBackOff) {
4926 // Set num retries to 3, but due to longer back off, we expect only 2 retry
4927 // will take place, while the 2nd one will obey the max backoff.
4928 const size_t kNumRetries = 3;
4929 SetNextResolution({});
4930 SetNextResolutionForLbChannelAllBalancers();
4931 // Populate new EDS resources.
4932 EdsResourceArgs args({
4933 {"locality0", CreateEndpointsForBackends(0, 1)},
4934 });
4935 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
4936 // Construct route config to set retry policy.
4937 RouteConfiguration new_route_config = default_route_config_;
4938 auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4939 auto* retry_policy = route1->mutable_route()->mutable_retry_policy();
4940 retry_policy->set_retry_on(
4941 "5xx,cancelled,deadline-exceeded,internal,resource-exhausted,"
4942 "unavailable");
4943 retry_policy->mutable_num_retries()->set_value(kNumRetries);
4944 auto base_interval =
4945 retry_policy->mutable_retry_back_off()->mutable_base_interval();
4946 // Set backoff to 1 second.
4947 base_interval->set_seconds(1 * grpc_test_slowdown_factor());
4948 base_interval->set_nanos(0);
4949 auto max_interval =
4950 retry_policy->mutable_retry_back_off()->mutable_max_interval();
4951 // Set max interval to be the same as base, so 2 retries will take 2 seconds
4952 // and both retries will take place before the 2.5 seconds rpc timeout.
4953 // Tested to ensure if max is not set, this test will be the same as
4954 // XdsRetryPolicyLongBackOff and we will only see 1 retry in that case.
4955 max_interval->set_seconds(1 * grpc_test_slowdown_factor());
4956 max_interval->set_nanos(0);
4957 SetRouteConfiguration(0, new_route_config);
4958 // We expect 2 retry before the RPC times out with DEADLINE_EXCEEDED.
4959 CheckRpcSendFailure(
4960 CheckRpcSendFailureOptions()
4961 .set_rpc_options(
4962 RpcOptions().set_timeout_ms(2500).set_server_expected_error(
4963 StatusCode::CANCELLED))
4964 .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
4965 EXPECT_EQ(2 + 1, backends_[0]->backend_service()->request_count());
4966 }
4967
TEST_P(LdsRdsTest,XdsRetryPolicyUnsupportedStatusCode)4968 TEST_P(LdsRdsTest, XdsRetryPolicyUnsupportedStatusCode) {
4969 const size_t kNumRetries = 3;
4970 SetNextResolution({});
4971 SetNextResolutionForLbChannelAllBalancers();
4972 // Populate new EDS resources.
4973 EdsResourceArgs args({
4974 {"locality0", CreateEndpointsForBackends(0, 1)},
4975 });
4976 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
4977 // Construct route config to set retry policy.
4978 RouteConfiguration new_route_config = default_route_config_;
4979 auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4980 auto* retry_policy = route1->mutable_route()->mutable_retry_policy();
4981 retry_policy->set_retry_on("5xx");
4982 retry_policy->mutable_num_retries()->set_value(kNumRetries);
4983 SetRouteConfiguration(0, new_route_config);
4984 // We expect no retry.
4985 CheckRpcSendFailure(
4986 CheckRpcSendFailureOptions()
4987 .set_rpc_options(RpcOptions().set_server_expected_error(
4988 StatusCode::DEADLINE_EXCEEDED))
4989 .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
4990 EXPECT_EQ(1, backends_[0]->backend_service()->request_count());
4991 }
4992
TEST_P(LdsRdsTest,XdsRetryPolicyUnsupportedStatusCodeWithVirtualHostLevelRetry)4993 TEST_P(LdsRdsTest,
4994 XdsRetryPolicyUnsupportedStatusCodeWithVirtualHostLevelRetry) {
4995 const size_t kNumRetries = 3;
4996 SetNextResolution({});
4997 SetNextResolutionForLbChannelAllBalancers();
4998 // Populate new EDS resources.
4999 EdsResourceArgs args({
5000 {"locality0", CreateEndpointsForBackends(0, 1)},
5001 });
5002 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
5003 // Construct route config to set retry policy with no supported retry_on
5004 // statuses.
5005 RouteConfiguration new_route_config = default_route_config_;
5006 auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
5007 auto* retry_policy = route1->mutable_route()->mutable_retry_policy();
5008 retry_policy->set_retry_on("5xx");
5009 retry_policy->mutable_num_retries()->set_value(kNumRetries);
5010 // Construct a virtual host level retry policy with supported statuses.
5011 auto* virtual_host_retry_policy =
5012 new_route_config.mutable_virtual_hosts(0)->mutable_retry_policy();
5013 virtual_host_retry_policy->set_retry_on(
5014 "cancelled,deadline-exceeded,internal,resource-exhausted,unavailable");
5015 virtual_host_retry_policy->mutable_num_retries()->set_value(kNumRetries);
5016 SetRouteConfiguration(0, new_route_config);
5017 // We expect no retry.
5018 CheckRpcSendFailure(
5019 CheckRpcSendFailureOptions()
5020 .set_rpc_options(RpcOptions().set_server_expected_error(
5021 StatusCode::DEADLINE_EXCEEDED))
5022 .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
5023 EXPECT_EQ(1, backends_[0]->backend_service()->request_count());
5024 }
5025
TEST_P(LdsRdsTest,XdsRetryPolicyInvalidNumRetriesZero)5026 TEST_P(LdsRdsTest, XdsRetryPolicyInvalidNumRetriesZero) {
5027 SetNextResolution({});
5028 SetNextResolutionForLbChannelAllBalancers();
5029 // Populate new EDS resources.
5030 EdsResourceArgs args({
5031 {"locality0", CreateEndpointsForBackends(0, 1)},
5032 });
5033 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
5034 // Construct route config to set retry policy.
5035 RouteConfiguration new_route_config = default_route_config_;
5036 auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
5037 auto* retry_policy = route1->mutable_route()->mutable_retry_policy();
5038 retry_policy->set_retry_on("deadline-exceeded");
5039 // Setting num_retries to zero is not valid.
5040 retry_policy->mutable_num_retries()->set_value(0);
5041 SetRouteConfiguration(0, new_route_config);
5042 ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
5043 const auto response_state = RouteConfigurationResponseState(0);
5044 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
5045 EXPECT_THAT(
5046 response_state.error_message,
5047 ::testing::HasSubstr(
5048 "RouteAction RetryPolicy num_retries set to invalid value 0."));
5049 }
5050
TEST_P(LdsRdsTest,XdsRetryPolicyRetryBackOffMissingBaseInterval)5051 TEST_P(LdsRdsTest, XdsRetryPolicyRetryBackOffMissingBaseInterval) {
5052 SetNextResolution({});
5053 SetNextResolutionForLbChannelAllBalancers();
5054 // Populate new EDS resources.
5055 EdsResourceArgs args({
5056 {"locality0", CreateEndpointsForBackends(0, 1)},
5057 });
5058 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
5059 // Construct route config to set retry policy.
5060 RouteConfiguration new_route_config = default_route_config_;
5061 auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
5062 auto* retry_policy = route1->mutable_route()->mutable_retry_policy();
5063 retry_policy->set_retry_on("deadline-exceeded");
5064 retry_policy->mutable_num_retries()->set_value(1);
5065 // RetryBackoff is there but base interval is missing.
5066 auto max_interval =
5067 retry_policy->mutable_retry_back_off()->mutable_max_interval();
5068 max_interval->set_seconds(0);
5069 max_interval->set_nanos(250000000);
5070 SetRouteConfiguration(0, new_route_config);
5071 ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
5072 const auto response_state = RouteConfigurationResponseState(0);
5073 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
5074 EXPECT_THAT(
5075 response_state.error_message,
5076 ::testing::HasSubstr(
5077 "RouteAction RetryPolicy RetryBackoff missing base interval."));
5078 }
5079
TEST_P(LdsRdsTest,XdsRoutingHeadersMatching)5080 TEST_P(LdsRdsTest, XdsRoutingHeadersMatching) {
5081 const char* kNewClusterName = "new_cluster";
5082 const char* kNewEdsServiceName = "new_eds_service_name";
5083 const size_t kNumEcho1Rpcs = 100;
5084 const size_t kNumEchoRpcs = 5;
5085 SetNextResolution({});
5086 SetNextResolutionForLbChannelAllBalancers();
5087 // Populate new EDS resources.
5088 EdsResourceArgs args({
5089 {"locality0", CreateEndpointsForBackends(0, 1)},
5090 });
5091 EdsResourceArgs args1({
5092 {"locality0", CreateEndpointsForBackends(1, 2)},
5093 });
5094 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
5095 balancers_[0]->ads_service()->SetEdsResource(
5096 BuildEdsResource(args1, kNewEdsServiceName));
5097 // Populate new CDS resources.
5098 Cluster new_cluster = default_cluster_;
5099 new_cluster.set_name(kNewClusterName);
5100 new_cluster.mutable_eds_cluster_config()->set_service_name(
5101 kNewEdsServiceName);
5102 balancers_[0]->ads_service()->SetCdsResource(new_cluster);
5103 // Populating Route Configurations for LDS.
5104 RouteConfiguration route_config = default_route_config_;
5105 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
5106 route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
5107 auto* header_matcher1 = route1->mutable_match()->add_headers();
5108 header_matcher1->set_name("header1");
5109 header_matcher1->set_exact_match("POST,PUT,GET");
5110 auto* header_matcher2 = route1->mutable_match()->add_headers();
5111 header_matcher2->set_name("header2");
5112 header_matcher2->mutable_safe_regex_match()->set_regex("[a-z]*");
5113 auto* header_matcher3 = route1->mutable_match()->add_headers();
5114 header_matcher3->set_name("header3");
5115 header_matcher3->mutable_range_match()->set_start(1);
5116 header_matcher3->mutable_range_match()->set_end(1000);
5117 auto* header_matcher4 = route1->mutable_match()->add_headers();
5118 header_matcher4->set_name("header4");
5119 header_matcher4->set_present_match(false);
5120 auto* header_matcher5 = route1->mutable_match()->add_headers();
5121 header_matcher5->set_name("header5");
5122 header_matcher5->set_present_match(true);
5123 auto* header_matcher6 = route1->mutable_match()->add_headers();
5124 header_matcher6->set_name("header6");
5125 header_matcher6->set_prefix_match("/grpc");
5126 auto* header_matcher7 = route1->mutable_match()->add_headers();
5127 header_matcher7->set_name("header7");
5128 header_matcher7->set_suffix_match(".cc");
5129 header_matcher7->set_invert_match(true);
5130 route1->mutable_route()->set_cluster(kNewClusterName);
5131 auto* default_route = route_config.mutable_virtual_hosts(0)->add_routes();
5132 default_route->mutable_match()->set_prefix("");
5133 default_route->mutable_route()->set_cluster(kDefaultClusterName);
5134 SetRouteConfiguration(0, route_config);
5135 std::vector<std::pair<std::string, std::string>> metadata = {
5136 {"header1", "POST"},
5137 {"header2", "blah"},
5138 {"header3", "1"},
5139 {"header5", "anything"},
5140 {"header6", "/grpc.testing.EchoTest1Service/"},
5141 {"header1", "PUT"},
5142 {"header7", "grpc.java"},
5143 {"header1", "GET"},
5144 };
5145 const auto header_match_rpc_options = RpcOptions()
5146 .set_rpc_service(SERVICE_ECHO1)
5147 .set_rpc_method(METHOD_ECHO1)
5148 .set_metadata(std::move(metadata));
5149 // Make sure all backends are up.
5150 WaitForBackend(0);
5151 WaitForBackend(1, WaitForBackendOptions(), header_match_rpc_options);
5152 // Send RPCs.
5153 CheckRpcSendOk(kNumEchoRpcs);
5154 CheckRpcSendOk(kNumEcho1Rpcs, header_match_rpc_options);
5155 EXPECT_EQ(kNumEchoRpcs, backends_[0]->backend_service()->request_count());
5156 EXPECT_EQ(0, backends_[0]->backend_service1()->request_count());
5157 EXPECT_EQ(0, backends_[0]->backend_service2()->request_count());
5158 EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
5159 EXPECT_EQ(kNumEcho1Rpcs, backends_[1]->backend_service1()->request_count());
5160 EXPECT_EQ(0, backends_[1]->backend_service2()->request_count());
5161 const auto response_state = RouteConfigurationResponseState(0);
5162 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::ACKED);
5163 }
5164
TEST_P(LdsRdsTest,XdsRoutingHeadersMatchingSpecialHeaderContentType)5165 TEST_P(LdsRdsTest, XdsRoutingHeadersMatchingSpecialHeaderContentType) {
5166 const char* kNewClusterName = "new_cluster";
5167 const char* kNewEdsServiceName = "new_eds_service_name";
5168 const size_t kNumEchoRpcs = 100;
5169 SetNextResolution({});
5170 SetNextResolutionForLbChannelAllBalancers();
5171 // Populate new EDS resources.
5172 EdsResourceArgs args({
5173 {"locality0", CreateEndpointsForBackends(0, 1)},
5174 });
5175 EdsResourceArgs args1({
5176 {"locality0", CreateEndpointsForBackends(1, 2)},
5177 });
5178 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
5179 balancers_[0]->ads_service()->SetEdsResource(
5180 BuildEdsResource(args1, kNewEdsServiceName));
5181 // Populate new CDS resources.
5182 Cluster new_cluster = default_cluster_;
5183 new_cluster.set_name(kNewClusterName);
5184 new_cluster.mutable_eds_cluster_config()->set_service_name(
5185 kNewEdsServiceName);
5186 balancers_[0]->ads_service()->SetCdsResource(new_cluster);
5187 // Populating Route Configurations for LDS.
5188 RouteConfiguration route_config = default_route_config_;
5189 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
5190 route1->mutable_match()->set_prefix("");
5191 auto* header_matcher1 = route1->mutable_match()->add_headers();
5192 header_matcher1->set_name("content-type");
5193 header_matcher1->set_exact_match("notapplication/grpc");
5194 route1->mutable_route()->set_cluster(kNewClusterName);
5195 auto* default_route = route_config.mutable_virtual_hosts(0)->add_routes();
5196 default_route->mutable_match()->set_prefix("");
5197 auto* header_matcher2 = default_route->mutable_match()->add_headers();
5198 header_matcher2->set_name("content-type");
5199 header_matcher2->set_exact_match("application/grpc");
5200 default_route->mutable_route()->set_cluster(kDefaultClusterName);
5201 SetRouteConfiguration(0, route_config);
5202 // Make sure the backend is up.
5203 WaitForAllBackends(0, 1);
5204 // Send RPCs.
5205 CheckRpcSendOk(kNumEchoRpcs);
5206 EXPECT_EQ(kNumEchoRpcs, backends_[0]->backend_service()->request_count());
5207 EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
5208 const auto response_state = RouteConfigurationResponseState(0);
5209 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::ACKED);
5210 }
5211
TEST_P(LdsRdsTest,XdsRoutingHeadersMatchingSpecialCasesToIgnore)5212 TEST_P(LdsRdsTest, XdsRoutingHeadersMatchingSpecialCasesToIgnore) {
5213 const char* kNewCluster1Name = "new_cluster_1";
5214 const char* kNewEdsService1Name = "new_eds_service_name_1";
5215 const size_t kNumEchoRpcs = 100;
5216 SetNextResolution({});
5217 SetNextResolutionForLbChannelAllBalancers();
5218 // Populate new EDS resources.
5219 EdsResourceArgs args({
5220 {"locality0", CreateEndpointsForBackends(0, 1)},
5221 });
5222 EdsResourceArgs args1({
5223 {"locality0", CreateEndpointsForBackends(1, 2)},
5224 });
5225 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
5226 balancers_[0]->ads_service()->SetEdsResource(
5227 BuildEdsResource(args1, kNewEdsService1Name));
5228 // Populate new CDS resources.
5229 Cluster new_cluster1 = default_cluster_;
5230 new_cluster1.set_name(kNewCluster1Name);
5231 new_cluster1.mutable_eds_cluster_config()->set_service_name(
5232 kNewEdsService1Name);
5233 balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
5234 // Populating Route Configurations for LDS.
5235 RouteConfiguration route_config = default_route_config_;
5236 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
5237 route1->mutable_match()->set_prefix("");
5238 auto* header_matcher1 = route1->mutable_match()->add_headers();
5239 header_matcher1->set_name("grpc-foo-bin");
5240 header_matcher1->set_present_match(true);
5241 route1->mutable_route()->set_cluster(kNewCluster1Name);
5242 auto* default_route = route_config.mutable_virtual_hosts(0)->add_routes();
5243 default_route->mutable_match()->set_prefix("");
5244 default_route->mutable_route()->set_cluster(kDefaultClusterName);
5245 SetRouteConfiguration(0, route_config);
5246 // Send headers which will mismatch each route
5247 std::vector<std::pair<std::string, std::string>> metadata = {
5248 {"grpc-foo-bin", "grpc-foo-bin"},
5249 };
5250 WaitForAllBackends(0, 1);
5251 CheckRpcSendOk(kNumEchoRpcs, RpcOptions().set_metadata(metadata));
5252 // Verify that only the default backend got RPCs since all previous routes
5253 // were mismatched.
5254 EXPECT_EQ(kNumEchoRpcs, backends_[0]->backend_service()->request_count());
5255 EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
5256 const auto response_state = RouteConfigurationResponseState(0);
5257 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::ACKED);
5258 }
5259
TEST_P(LdsRdsTest,XdsRoutingRuntimeFractionMatching)5260 TEST_P(LdsRdsTest, XdsRoutingRuntimeFractionMatching) {
5261 const char* kNewClusterName = "new_cluster";
5262 const char* kNewEdsServiceName = "new_eds_service_name";
5263 const double kErrorTolerance = 0.05;
5264 const size_t kRouteMatchNumerator = 25;
5265 const double kRouteMatchPercent =
5266 static_cast<double>(kRouteMatchNumerator) / 100;
5267 const size_t kNumRpcs =
5268 ComputeIdealNumRpcs(kRouteMatchPercent, kErrorTolerance);
5269 SetNextResolution({});
5270 SetNextResolutionForLbChannelAllBalancers();
5271 // Populate new EDS resources.
5272 EdsResourceArgs args({
5273 {"locality0", CreateEndpointsForBackends(0, 1)},
5274 });
5275 EdsResourceArgs args1({
5276 {"locality0", CreateEndpointsForBackends(1, 2)},
5277 });
5278 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
5279 balancers_[0]->ads_service()->SetEdsResource(
5280 BuildEdsResource(args1, kNewEdsServiceName));
5281 // Populate new CDS resources.
5282 Cluster new_cluster = default_cluster_;
5283 new_cluster.set_name(kNewClusterName);
5284 new_cluster.mutable_eds_cluster_config()->set_service_name(
5285 kNewEdsServiceName);
5286 balancers_[0]->ads_service()->SetCdsResource(new_cluster);
5287 // Populating Route Configurations for LDS.
5288 RouteConfiguration route_config = default_route_config_;
5289 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
5290 route1->mutable_match()
5291 ->mutable_runtime_fraction()
5292 ->mutable_default_value()
5293 ->set_numerator(kRouteMatchNumerator);
5294 route1->mutable_route()->set_cluster(kNewClusterName);
5295 auto* default_route = route_config.mutable_virtual_hosts(0)->add_routes();
5296 default_route->mutable_match()->set_prefix("");
5297 default_route->mutable_route()->set_cluster(kDefaultClusterName);
5298 SetRouteConfiguration(0, route_config);
5299 WaitForAllBackends(0, 2);
5300 CheckRpcSendOk(kNumRpcs);
5301 const int default_backend_count =
5302 backends_[0]->backend_service()->request_count();
5303 const int matched_backend_count =
5304 backends_[1]->backend_service()->request_count();
5305 EXPECT_THAT(static_cast<double>(default_backend_count) / kNumRpcs,
5306 ::testing::DoubleNear(1 - kRouteMatchPercent, kErrorTolerance));
5307 EXPECT_THAT(static_cast<double>(matched_backend_count) / kNumRpcs,
5308 ::testing::DoubleNear(kRouteMatchPercent, kErrorTolerance));
5309 const auto response_state = RouteConfigurationResponseState(0);
5310 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::ACKED);
5311 }
5312
TEST_P(LdsRdsTest,XdsRoutingHeadersMatchingUnmatchCases)5313 TEST_P(LdsRdsTest, XdsRoutingHeadersMatchingUnmatchCases) {
5314 const char* kNewCluster1Name = "new_cluster_1";
5315 const char* kNewEdsService1Name = "new_eds_service_name_1";
5316 const char* kNewCluster2Name = "new_cluster_2";
5317 const char* kNewEdsService2Name = "new_eds_service_name_2";
5318 const char* kNewCluster3Name = "new_cluster_3";
5319 const char* kNewEdsService3Name = "new_eds_service_name_3";
5320 const size_t kNumEcho1Rpcs = 100;
5321 const size_t kNumEchoRpcs = 5;
5322 SetNextResolution({});
5323 SetNextResolutionForLbChannelAllBalancers();
5324 // Populate new EDS resources.
5325 EdsResourceArgs args({
5326 {"locality0", CreateEndpointsForBackends(0, 1)},
5327 });
5328 EdsResourceArgs args1({
5329 {"locality0", CreateEndpointsForBackends(1, 2)},
5330 });
5331 EdsResourceArgs args2({
5332 {"locality0", CreateEndpointsForBackends(2, 3)},
5333 });
5334 EdsResourceArgs args3({
5335 {"locality0", CreateEndpointsForBackends(3, 4)},
5336 });
5337 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
5338 balancers_[0]->ads_service()->SetEdsResource(
5339 BuildEdsResource(args1, kNewEdsService1Name));
5340 balancers_[0]->ads_service()->SetEdsResource(
5341 BuildEdsResource(args2, kNewEdsService2Name));
5342 balancers_[0]->ads_service()->SetEdsResource(
5343 BuildEdsResource(args3, kNewEdsService3Name));
5344 // Populate new CDS resources.
5345 Cluster new_cluster1 = default_cluster_;
5346 new_cluster1.set_name(kNewCluster1Name);
5347 new_cluster1.mutable_eds_cluster_config()->set_service_name(
5348 kNewEdsService1Name);
5349 balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
5350 Cluster new_cluster2 = default_cluster_;
5351 new_cluster2.set_name(kNewCluster2Name);
5352 new_cluster2.mutable_eds_cluster_config()->set_service_name(
5353 kNewEdsService2Name);
5354 balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
5355 Cluster new_cluster3 = default_cluster_;
5356 new_cluster3.set_name(kNewCluster3Name);
5357 new_cluster3.mutable_eds_cluster_config()->set_service_name(
5358 kNewEdsService3Name);
5359 balancers_[0]->ads_service()->SetCdsResource(new_cluster3);
5360 // Populating Route Configurations for LDS.
5361 RouteConfiguration route_config = default_route_config_;
5362 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
5363 route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
5364 auto* header_matcher1 = route1->mutable_match()->add_headers();
5365 header_matcher1->set_name("header1");
5366 header_matcher1->set_exact_match("POST");
5367 route1->mutable_route()->set_cluster(kNewCluster1Name);
5368 auto route2 = route_config.mutable_virtual_hosts(0)->add_routes();
5369 route2->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
5370 auto* header_matcher2 = route2->mutable_match()->add_headers();
5371 header_matcher2->set_name("header2");
5372 header_matcher2->mutable_range_match()->set_start(1);
5373 header_matcher2->mutable_range_match()->set_end(1000);
5374 route2->mutable_route()->set_cluster(kNewCluster2Name);
5375 auto route3 = route_config.mutable_virtual_hosts(0)->add_routes();
5376 route3->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
5377 auto* header_matcher3 = route3->mutable_match()->add_headers();
5378 header_matcher3->set_name("header3");
5379 header_matcher3->mutable_safe_regex_match()->set_regex("[a-z]*");
5380 route3->mutable_route()->set_cluster(kNewCluster3Name);
5381 auto* default_route = route_config.mutable_virtual_hosts(0)->add_routes();
5382 default_route->mutable_match()->set_prefix("");
5383 default_route->mutable_route()->set_cluster(kDefaultClusterName);
5384 SetRouteConfiguration(0, route_config);
5385 // Send headers which will mismatch each route
5386 std::vector<std::pair<std::string, std::string>> metadata = {
5387 {"header1", "POST"},
5388 {"header2", "1000"},
5389 {"header3", "123"},
5390 {"header1", "GET"},
5391 };
5392 WaitForAllBackends(0, 1);
5393 CheckRpcSendOk(kNumEchoRpcs, RpcOptions().set_metadata(metadata));
5394 CheckRpcSendOk(kNumEcho1Rpcs, RpcOptions()
5395 .set_rpc_service(SERVICE_ECHO1)
5396 .set_rpc_method(METHOD_ECHO1)
5397 .set_metadata(metadata));
5398 // Verify that only the default backend got RPCs since all previous routes
5399 // were mismatched.
5400 for (size_t i = 1; i < 4; ++i) {
5401 EXPECT_EQ(0, backends_[i]->backend_service()->request_count());
5402 EXPECT_EQ(0, backends_[i]->backend_service1()->request_count());
5403 EXPECT_EQ(0, backends_[i]->backend_service2()->request_count());
5404 }
5405 EXPECT_EQ(kNumEchoRpcs, backends_[0]->backend_service()->request_count());
5406 EXPECT_EQ(kNumEcho1Rpcs, backends_[0]->backend_service1()->request_count());
5407 EXPECT_EQ(0, backends_[0]->backend_service2()->request_count());
5408 const auto response_state = RouteConfigurationResponseState(0);
5409 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::ACKED);
5410 }
5411
TEST_P(LdsRdsTest,XdsRoutingChangeRoutesWithoutChangingClusters)5412 TEST_P(LdsRdsTest, XdsRoutingChangeRoutesWithoutChangingClusters) {
5413 const char* kNewClusterName = "new_cluster";
5414 const char* kNewEdsServiceName = "new_eds_service_name";
5415 SetNextResolution({});
5416 SetNextResolutionForLbChannelAllBalancers();
5417 // Populate new EDS resources.
5418 EdsResourceArgs args({
5419 {"locality0", CreateEndpointsForBackends(0, 1)},
5420 });
5421 EdsResourceArgs args1({
5422 {"locality0", CreateEndpointsForBackends(1, 2)},
5423 });
5424 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
5425 balancers_[0]->ads_service()->SetEdsResource(
5426 BuildEdsResource(args1, kNewEdsServiceName));
5427 // Populate new CDS resources.
5428 Cluster new_cluster = default_cluster_;
5429 new_cluster.set_name(kNewClusterName);
5430 new_cluster.mutable_eds_cluster_config()->set_service_name(
5431 kNewEdsServiceName);
5432 balancers_[0]->ads_service()->SetCdsResource(new_cluster);
5433 // Populating Route Configurations for LDS.
5434 RouteConfiguration route_config = default_route_config_;
5435 auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
5436 route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
5437 route1->mutable_route()->set_cluster(kNewClusterName);
5438 auto* default_route = route_config.mutable_virtual_hosts(0)->add_routes();
5439 default_route->mutable_match()->set_prefix("");
5440 default_route->mutable_route()->set_cluster(kDefaultClusterName);
5441 SetRouteConfiguration(0, route_config);
5442 // Make sure all backends are up and that requests for each RPC
5443 // service go to the right backends.
5444 WaitForBackend(0, WaitForBackendOptions().set_reset_counters(false));
5445 WaitForBackend(1, WaitForBackendOptions().set_reset_counters(false),
5446 RpcOptions().set_rpc_service(SERVICE_ECHO1));
5447 WaitForBackend(0, WaitForBackendOptions().set_reset_counters(false),
5448 RpcOptions().set_rpc_service(SERVICE_ECHO2));
5449 // Requests for services Echo and Echo2 should have gone to backend 0.
5450 EXPECT_EQ(1, backends_[0]->backend_service()->request_count());
5451 EXPECT_EQ(0, backends_[0]->backend_service1()->request_count());
5452 EXPECT_EQ(1, backends_[0]->backend_service2()->request_count());
5453 // Requests for service Echo1 should have gone to backend 1.
5454 EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
5455 EXPECT_EQ(1, backends_[1]->backend_service1()->request_count());
5456 EXPECT_EQ(0, backends_[1]->backend_service2()->request_count());
5457 // Now send an update that changes the first route to match a
5458 // different RPC service, and wait for the client to make the change.
5459 route1->mutable_match()->set_prefix("/grpc.testing.EchoTest2Service/");
5460 SetRouteConfiguration(0, route_config);
5461 WaitForBackend(1, WaitForBackendOptions(),
5462 RpcOptions().set_rpc_service(SERVICE_ECHO2));
5463 // Now repeat the earlier test, making sure all traffic goes to the
5464 // right place.
5465 WaitForBackend(0, WaitForBackendOptions().set_reset_counters(false));
5466 WaitForBackend(0, WaitForBackendOptions().set_reset_counters(false),
5467 RpcOptions().set_rpc_service(SERVICE_ECHO1));
5468 WaitForBackend(1, WaitForBackendOptions().set_reset_counters(false),
5469 RpcOptions().set_rpc_service(SERVICE_ECHO2));
5470 // Requests for services Echo and Echo1 should have gone to backend 0.
5471 EXPECT_EQ(1, backends_[0]->backend_service()->request_count());
5472 EXPECT_EQ(1, backends_[0]->backend_service1()->request_count());
5473 EXPECT_EQ(0, backends_[0]->backend_service2()->request_count());
5474 // Requests for service Echo2 should have gone to backend 1.
5475 EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
5476 EXPECT_EQ(0, backends_[1]->backend_service1()->request_count());
5477 EXPECT_EQ(1, backends_[1]->backend_service2()->request_count());
5478 }
5479
5480 // Test that we NACK unknown filter types in VirtualHost.
TEST_P(LdsRdsTest,RejectsUnknownHttpFilterTypeInVirtualHost)5481 TEST_P(LdsRdsTest, RejectsUnknownHttpFilterTypeInVirtualHost) {
5482 if (GetParam().use_v2()) return; // Filters supported in v3 only.
5483 RouteConfiguration route_config = default_route_config_;
5484 auto* per_filter_config =
5485 route_config.mutable_virtual_hosts(0)->mutable_typed_per_filter_config();
5486 (*per_filter_config)["unknown"].PackFrom(Listener());
5487 SetListenerAndRouteConfiguration(0, default_listener_, route_config);
5488 SetNextResolution({});
5489 SetNextResolutionForLbChannelAllBalancers();
5490 ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
5491 const auto response_state = RouteConfigurationResponseState(0);
5492 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
5493 EXPECT_THAT(response_state.error_message,
5494 ::testing::HasSubstr("no filter registered for config type "
5495 "envoy.config.listener.v3.Listener"));
5496 }
5497
5498 // Test that we ignore optional unknown filter types in VirtualHost.
TEST_P(LdsRdsTest,IgnoresOptionalUnknownHttpFilterTypeInVirtualHost)5499 TEST_P(LdsRdsTest, IgnoresOptionalUnknownHttpFilterTypeInVirtualHost) {
5500 if (GetParam().use_v2()) return; // Filters supported in v3 only.
5501 RouteConfiguration route_config = default_route_config_;
5502 auto* per_filter_config =
5503 route_config.mutable_virtual_hosts(0)->mutable_typed_per_filter_config();
5504 ::envoy::config::route::v3::FilterConfig filter_config;
5505 filter_config.mutable_config()->PackFrom(Listener());
5506 filter_config.set_is_optional(true);
5507 (*per_filter_config)["unknown"].PackFrom(filter_config);
5508 SetListenerAndRouteConfiguration(0, default_listener_, route_config);
5509 EdsResourceArgs args({
5510 {"locality0", CreateEndpointsForBackends()},
5511 });
5512 balancers_[0]->ads_service()->SetEdsResource(
5513 BuildEdsResource(args, DefaultEdsServiceName()));
5514 SetNextResolution({});
5515 SetNextResolutionForLbChannelAllBalancers();
5516 WaitForAllBackends();
5517 EXPECT_EQ(RouteConfigurationResponseState(0).state,
5518 AdsServiceImpl::ResponseState::ACKED);
5519 }
5520
5521 // Test that we NACK filters without configs in VirtualHost.
TEST_P(LdsRdsTest,RejectsHttpFilterWithoutConfigInVirtualHost)5522 TEST_P(LdsRdsTest, RejectsHttpFilterWithoutConfigInVirtualHost) {
5523 if (GetParam().use_v2()) return; // Filters supported in v3 only.
5524 RouteConfiguration route_config = default_route_config_;
5525 auto* per_filter_config =
5526 route_config.mutable_virtual_hosts(0)->mutable_typed_per_filter_config();
5527 (*per_filter_config)["unknown"];
5528 SetListenerAndRouteConfiguration(0, default_listener_, route_config);
5529 SetNextResolution({});
5530 SetNextResolutionForLbChannelAllBalancers();
5531 ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
5532 const auto response_state = RouteConfigurationResponseState(0);
5533 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
5534 EXPECT_THAT(response_state.error_message,
5535 ::testing::HasSubstr(
5536 "no filter config specified for filter name unknown"));
5537 }
5538
5539 // Test that we NACK filters without configs in FilterConfig in VirtualHost.
TEST_P(LdsRdsTest,RejectsHttpFilterWithoutConfigInFilterConfigInVirtualHost)5540 TEST_P(LdsRdsTest, RejectsHttpFilterWithoutConfigInFilterConfigInVirtualHost) {
5541 if (GetParam().use_v2()) return; // Filters supported in v3 only.
5542 RouteConfiguration route_config = default_route_config_;
5543 auto* per_filter_config =
5544 route_config.mutable_virtual_hosts(0)->mutable_typed_per_filter_config();
5545 (*per_filter_config)["unknown"].PackFrom(
5546 ::envoy::config::route::v3::FilterConfig());
5547 SetListenerAndRouteConfiguration(0, default_listener_, route_config);
5548 SetNextResolution({});
5549 SetNextResolutionForLbChannelAllBalancers();
5550 ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
5551 const auto response_state = RouteConfigurationResponseState(0);
5552 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
5553 EXPECT_THAT(response_state.error_message,
5554 ::testing::HasSubstr(
5555 "no filter config specified for filter name unknown"));
5556 }
5557
5558 // Test that we ignore optional filters without configs in VirtualHost.
TEST_P(LdsRdsTest,IgnoresOptionalHttpFilterWithoutConfigInVirtualHost)5559 TEST_P(LdsRdsTest, IgnoresOptionalHttpFilterWithoutConfigInVirtualHost) {
5560 if (GetParam().use_v2()) return; // Filters supported in v3 only.
5561 RouteConfiguration route_config = default_route_config_;
5562 auto* per_filter_config =
5563 route_config.mutable_virtual_hosts(0)->mutable_typed_per_filter_config();
5564 ::envoy::config::route::v3::FilterConfig filter_config;
5565 filter_config.set_is_optional(true);
5566 (*per_filter_config)["unknown"].PackFrom(filter_config);
5567 SetListenerAndRouteConfiguration(0, default_listener_, route_config);
5568 EdsResourceArgs args({
5569 {"locality0", CreateEndpointsForBackends()},
5570 });
5571 balancers_[0]->ads_service()->SetEdsResource(
5572 BuildEdsResource(args, DefaultEdsServiceName()));
5573 SetNextResolution({});
5574 SetNextResolutionForLbChannelAllBalancers();
5575 WaitForAllBackends();
5576 EXPECT_EQ(RouteConfigurationResponseState(0).state,
5577 AdsServiceImpl::ResponseState::ACKED);
5578 }
5579
5580 // Test that we NACK unparseable filter types in VirtualHost.
TEST_P(LdsRdsTest,RejectsUnparseableHttpFilterTypeInVirtualHost)5581 TEST_P(LdsRdsTest, RejectsUnparseableHttpFilterTypeInVirtualHost) {
5582 if (GetParam().use_v2()) return; // Filters supported in v3 only.
5583 RouteConfiguration route_config = default_route_config_;
5584 auto* per_filter_config =
5585 route_config.mutable_virtual_hosts(0)->mutable_typed_per_filter_config();
5586 (*per_filter_config)["unknown"].PackFrom(
5587 envoy::extensions::filters::http::router::v3::Router());
5588 SetListenerAndRouteConfiguration(0, default_listener_, route_config);
5589 SetNextResolution({});
5590 SetNextResolutionForLbChannelAllBalancers();
5591 ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
5592 const auto response_state = RouteConfigurationResponseState(0);
5593 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
5594 EXPECT_THAT(
5595 response_state.error_message,
5596 ::testing::HasSubstr("router filter does not support config override"));
5597 }
5598
5599 // Test that we NACK unknown filter types in Route.
TEST_P(LdsRdsTest,RejectsUnknownHttpFilterTypeInRoute)5600 TEST_P(LdsRdsTest, RejectsUnknownHttpFilterTypeInRoute) {
5601 if (GetParam().use_v2()) return; // Filters supported in v3 only.
5602 RouteConfiguration route_config = default_route_config_;
5603 auto* per_filter_config = route_config.mutable_virtual_hosts(0)
5604 ->mutable_routes(0)
5605 ->mutable_typed_per_filter_config();
5606 (*per_filter_config)["unknown"].PackFrom(Listener());
5607 SetListenerAndRouteConfiguration(0, default_listener_, route_config);
5608 SetNextResolution({});
5609 SetNextResolutionForLbChannelAllBalancers();
5610 ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
5611 const auto response_state = RouteConfigurationResponseState(0);
5612 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
5613 EXPECT_THAT(response_state.error_message,
5614 ::testing::HasSubstr("no filter registered for config type "
5615 "envoy.config.listener.v3.Listener"));
5616 }
5617
5618 // Test that we ignore optional unknown filter types in Route.
TEST_P(LdsRdsTest,IgnoresOptionalUnknownHttpFilterTypeInRoute)5619 TEST_P(LdsRdsTest, IgnoresOptionalUnknownHttpFilterTypeInRoute) {
5620 if (GetParam().use_v2()) return; // Filters supported in v3 only.
5621 RouteConfiguration route_config = default_route_config_;
5622 auto* per_filter_config = route_config.mutable_virtual_hosts(0)
5623 ->mutable_routes(0)
5624 ->mutable_typed_per_filter_config();
5625 ::envoy::config::route::v3::FilterConfig filter_config;
5626 filter_config.mutable_config()->PackFrom(Listener());
5627 filter_config.set_is_optional(true);
5628 (*per_filter_config)["unknown"].PackFrom(filter_config);
5629 SetListenerAndRouteConfiguration(0, default_listener_, route_config);
5630 EdsResourceArgs args({
5631 {"locality0", CreateEndpointsForBackends()},
5632 });
5633 balancers_[0]->ads_service()->SetEdsResource(
5634 BuildEdsResource(args, DefaultEdsServiceName()));
5635 SetNextResolution({});
5636 SetNextResolutionForLbChannelAllBalancers();
5637 WaitForAllBackends();
5638 EXPECT_EQ(RouteConfigurationResponseState(0).state,
5639 AdsServiceImpl::ResponseState::ACKED);
5640 }
5641
5642 // Test that we NACK filters without configs in Route.
TEST_P(LdsRdsTest,RejectsHttpFilterWithoutConfigInRoute)5643 TEST_P(LdsRdsTest, RejectsHttpFilterWithoutConfigInRoute) {
5644 if (GetParam().use_v2()) return; // Filters supported in v3 only.
5645 RouteConfiguration route_config = default_route_config_;
5646 auto* per_filter_config = route_config.mutable_virtual_hosts(0)
5647 ->mutable_routes(0)
5648 ->mutable_typed_per_filter_config();
5649 (*per_filter_config)["unknown"];
5650 SetListenerAndRouteConfiguration(0, default_listener_, route_config);
5651 SetNextResolution({});
5652 SetNextResolutionForLbChannelAllBalancers();
5653 ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
5654 const auto response_state = RouteConfigurationResponseState(0);
5655 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
5656 EXPECT_THAT(response_state.error_message,
5657 ::testing::HasSubstr(
5658 "no filter config specified for filter name unknown"));
5659 }
5660
5661 // Test that we NACK filters without configs in FilterConfig in Route.
TEST_P(LdsRdsTest,RejectsHttpFilterWithoutConfigInFilterConfigInRoute)5662 TEST_P(LdsRdsTest, RejectsHttpFilterWithoutConfigInFilterConfigInRoute) {
5663 if (GetParam().use_v2()) return; // Filters supported in v3 only.
5664 RouteConfiguration route_config = default_route_config_;
5665 auto* per_filter_config = route_config.mutable_virtual_hosts(0)
5666 ->mutable_routes(0)
5667 ->mutable_typed_per_filter_config();
5668 (*per_filter_config)["unknown"].PackFrom(
5669 ::envoy::config::route::v3::FilterConfig());
5670 SetListenerAndRouteConfiguration(0, default_listener_, route_config);
5671 SetNextResolution({});
5672 SetNextResolutionForLbChannelAllBalancers();
5673 ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
5674 const auto response_state = RouteConfigurationResponseState(0);
5675 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
5676 EXPECT_THAT(response_state.error_message,
5677 ::testing::HasSubstr(
5678 "no filter config specified for filter name unknown"));
5679 }
5680
5681 // Test that we ignore optional filters without configs in Route.
TEST_P(LdsRdsTest,IgnoresOptionalHttpFilterWithoutConfigInRoute)5682 TEST_P(LdsRdsTest, IgnoresOptionalHttpFilterWithoutConfigInRoute) {
5683 if (GetParam().use_v2()) return; // Filters supported in v3 only.
5684 RouteConfiguration route_config = default_route_config_;
5685 auto* per_filter_config = route_config.mutable_virtual_hosts(0)
5686 ->mutable_routes(0)
5687 ->mutable_typed_per_filter_config();
5688 ::envoy::config::route::v3::FilterConfig filter_config;
5689 filter_config.set_is_optional(true);
5690 (*per_filter_config)["unknown"].PackFrom(filter_config);
5691 SetListenerAndRouteConfiguration(0, default_listener_, route_config);
5692 EdsResourceArgs args({
5693 {"locality0", CreateEndpointsForBackends()},
5694 });
5695 balancers_[0]->ads_service()->SetEdsResource(
5696 BuildEdsResource(args, DefaultEdsServiceName()));
5697 SetNextResolution({});
5698 SetNextResolutionForLbChannelAllBalancers();
5699 WaitForAllBackends();
5700 EXPECT_EQ(RouteConfigurationResponseState(0).state,
5701 AdsServiceImpl::ResponseState::ACKED);
5702 }
5703
5704 // Test that we NACK unparseable filter types in Route.
TEST_P(LdsRdsTest,RejectsUnparseableHttpFilterTypeInRoute)5705 TEST_P(LdsRdsTest, RejectsUnparseableHttpFilterTypeInRoute) {
5706 if (GetParam().use_v2()) return; // Filters supported in v3 only.
5707 RouteConfiguration route_config = default_route_config_;
5708 auto* per_filter_config = route_config.mutable_virtual_hosts(0)
5709 ->mutable_routes(0)
5710 ->mutable_typed_per_filter_config();
5711 (*per_filter_config)["unknown"].PackFrom(
5712 envoy::extensions::filters::http::router::v3::Router());
5713 SetListenerAndRouteConfiguration(0, default_listener_, route_config);
5714 SetNextResolution({});
5715 SetNextResolutionForLbChannelAllBalancers();
5716 ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
5717 const auto response_state = RouteConfigurationResponseState(0);
5718 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
5719 EXPECT_THAT(
5720 response_state.error_message,
5721 ::testing::HasSubstr("router filter does not support config override"));
5722 }
5723
5724 // Test that we NACK unknown filter types in ClusterWeight.
TEST_P(LdsRdsTest,RejectsUnknownHttpFilterTypeInClusterWeight)5725 TEST_P(LdsRdsTest, RejectsUnknownHttpFilterTypeInClusterWeight) {
5726 if (GetParam().use_v2()) return; // Filters supported in v3 only.
5727 RouteConfiguration route_config = default_route_config_;
5728 auto* cluster_weight = route_config.mutable_virtual_hosts(0)
5729 ->mutable_routes(0)
5730 ->mutable_route()
5731 ->mutable_weighted_clusters()
5732 ->add_clusters();
5733 cluster_weight->set_name(kDefaultClusterName);
5734 cluster_weight->mutable_weight()->set_value(100);
5735 auto* per_filter_config = cluster_weight->mutable_typed_per_filter_config();
5736 (*per_filter_config)["unknown"].PackFrom(Listener());
5737 SetListenerAndRouteConfiguration(0, default_listener_, route_config);
5738 SetNextResolution({});
5739 SetNextResolutionForLbChannelAllBalancers();
5740 ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
5741 const auto response_state = RouteConfigurationResponseState(0);
5742 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
5743 EXPECT_THAT(response_state.error_message,
5744 ::testing::HasSubstr("no filter registered for config type "
5745 "envoy.config.listener.v3.Listener"));
5746 }
5747
5748 // Test that we ignore optional unknown filter types in ClusterWeight.
TEST_P(LdsRdsTest,IgnoresOptionalUnknownHttpFilterTypeInClusterWeight)5749 TEST_P(LdsRdsTest, IgnoresOptionalUnknownHttpFilterTypeInClusterWeight) {
5750 if (GetParam().use_v2()) return; // Filters supported in v3 only.
5751 RouteConfiguration route_config = default_route_config_;
5752 auto* cluster_weight = route_config.mutable_virtual_hosts(0)
5753 ->mutable_routes(0)
5754 ->mutable_route()
5755 ->mutable_weighted_clusters()
5756 ->add_clusters();
5757 cluster_weight->set_name(kDefaultClusterName);
5758 cluster_weight->mutable_weight()->set_value(100);
5759 auto* per_filter_config = cluster_weight->mutable_typed_per_filter_config();
5760 ::envoy::config::route::v3::FilterConfig filter_config;
5761 filter_config.mutable_config()->PackFrom(Listener());
5762 filter_config.set_is_optional(true);
5763 (*per_filter_config)["unknown"].PackFrom(filter_config);
5764 SetListenerAndRouteConfiguration(0, default_listener_, route_config);
5765 EdsResourceArgs args({
5766 {"locality0", CreateEndpointsForBackends()},
5767 });
5768 balancers_[0]->ads_service()->SetEdsResource(
5769 BuildEdsResource(args, DefaultEdsServiceName()));
5770 SetNextResolution({});
5771 SetNextResolutionForLbChannelAllBalancers();
5772 WaitForAllBackends();
5773 EXPECT_EQ(RouteConfigurationResponseState(0).state,
5774 AdsServiceImpl::ResponseState::ACKED);
5775 }
5776
5777 // Test that we NACK filters without configs in ClusterWeight.
TEST_P(LdsRdsTest,RejectsHttpFilterWithoutConfigInClusterWeight)5778 TEST_P(LdsRdsTest, RejectsHttpFilterWithoutConfigInClusterWeight) {
5779 if (GetParam().use_v2()) return; // Filters supported in v3 only.
5780 RouteConfiguration route_config = default_route_config_;
5781 auto* cluster_weight = route_config.mutable_virtual_hosts(0)
5782 ->mutable_routes(0)
5783 ->mutable_route()
5784 ->mutable_weighted_clusters()
5785 ->add_clusters();
5786 cluster_weight->set_name(kDefaultClusterName);
5787 cluster_weight->mutable_weight()->set_value(100);
5788 auto* per_filter_config = cluster_weight->mutable_typed_per_filter_config();
5789 (*per_filter_config)["unknown"];
5790 SetListenerAndRouteConfiguration(0, default_listener_, route_config);
5791 SetNextResolution({});
5792 SetNextResolutionForLbChannelAllBalancers();
5793 ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
5794 const auto response_state = RouteConfigurationResponseState(0);
5795 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
5796 EXPECT_THAT(response_state.error_message,
5797 ::testing::HasSubstr(
5798 "no filter config specified for filter name unknown"));
5799 }
5800
5801 // Test that we NACK filters without configs in FilterConfig in ClusterWeight.
TEST_P(LdsRdsTest,RejectsHttpFilterWithoutConfigInFilterConfigInClusterWeight)5802 TEST_P(LdsRdsTest,
5803 RejectsHttpFilterWithoutConfigInFilterConfigInClusterWeight) {
5804 if (GetParam().use_v2()) return; // Filters supported in v3 only.
5805 RouteConfiguration route_config = default_route_config_;
5806 auto* cluster_weight = route_config.mutable_virtual_hosts(0)
5807 ->mutable_routes(0)
5808 ->mutable_route()
5809 ->mutable_weighted_clusters()
5810 ->add_clusters();
5811 cluster_weight->set_name(kDefaultClusterName);
5812 cluster_weight->mutable_weight()->set_value(100);
5813 auto* per_filter_config = cluster_weight->mutable_typed_per_filter_config();
5814 (*per_filter_config)["unknown"].PackFrom(
5815 ::envoy::config::route::v3::FilterConfig());
5816 SetListenerAndRouteConfiguration(0, default_listener_, route_config);
5817 SetNextResolution({});
5818 SetNextResolutionForLbChannelAllBalancers();
5819 ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
5820 const auto response_state = RouteConfigurationResponseState(0);
5821 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
5822 EXPECT_THAT(response_state.error_message,
5823 ::testing::HasSubstr(
5824 "no filter config specified for filter name unknown"));
5825 }
5826
5827 // Test that we ignore optional filters without configs in ClusterWeight.
TEST_P(LdsRdsTest,IgnoresOptionalHttpFilterWithoutConfigInClusterWeight)5828 TEST_P(LdsRdsTest, IgnoresOptionalHttpFilterWithoutConfigInClusterWeight) {
5829 if (GetParam().use_v2()) return; // Filters supported in v3 only.
5830 RouteConfiguration route_config = default_route_config_;
5831 auto* cluster_weight = route_config.mutable_virtual_hosts(0)
5832 ->mutable_routes(0)
5833 ->mutable_route()
5834 ->mutable_weighted_clusters()
5835 ->add_clusters();
5836 cluster_weight->set_name(kDefaultClusterName);
5837 cluster_weight->mutable_weight()->set_value(100);
5838 auto* per_filter_config = cluster_weight->mutable_typed_per_filter_config();
5839 ::envoy::config::route::v3::FilterConfig filter_config;
5840 filter_config.set_is_optional(true);
5841 (*per_filter_config)["unknown"].PackFrom(filter_config);
5842 SetListenerAndRouteConfiguration(0, default_listener_, route_config);
5843 EdsResourceArgs args({
5844 {"locality0", CreateEndpointsForBackends()},
5845 });
5846 balancers_[0]->ads_service()->SetEdsResource(
5847 BuildEdsResource(args, DefaultEdsServiceName()));
5848 SetNextResolution({});
5849 SetNextResolutionForLbChannelAllBalancers();
5850 WaitForAllBackends();
5851 EXPECT_EQ(RouteConfigurationResponseState(0).state,
5852 AdsServiceImpl::ResponseState::ACKED);
5853 }
5854
5855 // Test that we NACK unparseable filter types in ClusterWeight.
TEST_P(LdsRdsTest,RejectsUnparseableHttpFilterTypeInClusterWeight)5856 TEST_P(LdsRdsTest, RejectsUnparseableHttpFilterTypeInClusterWeight) {
5857 if (GetParam().use_v2()) return; // Filters supported in v3 only.
5858 RouteConfiguration route_config = default_route_config_;
5859 auto* cluster_weight = route_config.mutable_virtual_hosts(0)
5860 ->mutable_routes(0)
5861 ->mutable_route()
5862 ->mutable_weighted_clusters()
5863 ->add_clusters();
5864 cluster_weight->set_name(kDefaultClusterName);
5865 cluster_weight->mutable_weight()->set_value(100);
5866 auto* per_filter_config = cluster_weight->mutable_typed_per_filter_config();
5867 (*per_filter_config)["unknown"].PackFrom(
5868 envoy::extensions::filters::http::router::v3::Router());
5869 SetListenerAndRouteConfiguration(0, default_listener_, route_config);
5870 SetNextResolution({});
5871 SetNextResolutionForLbChannelAllBalancers();
5872 ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
5873 const auto response_state = RouteConfigurationResponseState(0);
5874 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
5875 EXPECT_THAT(
5876 response_state.error_message,
5877 ::testing::HasSubstr("router filter does not support config override"));
5878 }
5879
5880 using CdsTest = BasicTest;
5881
5882 // Tests that CDS client should send an ACK upon correct CDS response.
TEST_P(CdsTest,Vanilla)5883 TEST_P(CdsTest, Vanilla) {
5884 SetNextResolution({});
5885 SetNextResolutionForLbChannelAllBalancers();
5886 (void)SendRpc();
5887 EXPECT_EQ(balancers_[0]->ads_service()->cds_response_state().state,
5888 AdsServiceImpl::ResponseState::ACKED);
5889 }
5890
TEST_P(CdsTest,LogicalDNSClusterType)5891 TEST_P(CdsTest, LogicalDNSClusterType) {
5892 gpr_setenv("GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER",
5893 "true");
5894 SetNextResolution({});
5895 SetNextResolutionForLbChannelAllBalancers();
5896 // Create Logical DNS Cluster
5897 auto cluster = default_cluster_;
5898 cluster.set_type(Cluster::LOGICAL_DNS);
5899 auto* address = cluster.mutable_load_assignment()
5900 ->add_endpoints()
5901 ->add_lb_endpoints()
5902 ->mutable_endpoint()
5903 ->mutable_address()
5904 ->mutable_socket_address();
5905 address->set_address(kServerName);
5906 address->set_port_value(443);
5907 balancers_[0]->ads_service()->SetCdsResource(cluster);
5908 // Set Logical DNS result
5909 {
5910 grpc_core::ExecCtx exec_ctx;
5911 grpc_core::Resolver::Result result;
5912 result.addresses = CreateAddressListFromPortList(GetBackendPorts(1, 2));
5913 logical_dns_cluster_resolver_response_generator_->SetResponse(
5914 std::move(result));
5915 }
5916 // Wait for traffic to go to backend 1.
5917 WaitForBackend(1);
5918 gpr_unsetenv(
5919 "GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER");
5920 }
5921
TEST_P(CdsTest,LogicalDNSClusterTypeMissingLoadAssignment)5922 TEST_P(CdsTest, LogicalDNSClusterTypeMissingLoadAssignment) {
5923 gpr_setenv("GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER",
5924 "true");
5925 SetNextResolution({});
5926 SetNextResolutionForLbChannelAllBalancers();
5927 // Create Logical DNS Cluster
5928 auto cluster = default_cluster_;
5929 cluster.set_type(Cluster::LOGICAL_DNS);
5930 balancers_[0]->ads_service()->SetCdsResource(cluster);
5931 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
5932 const auto response_state =
5933 balancers_[0]->ads_service()->cds_response_state();
5934 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
5935 EXPECT_THAT(response_state.error_message,
5936 ::testing::HasSubstr(
5937 "load_assignment not present for LOGICAL_DNS cluster"));
5938 gpr_unsetenv(
5939 "GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER");
5940 }
5941
TEST_P(CdsTest,LogicalDNSClusterTypeMissingLocalities)5942 TEST_P(CdsTest, LogicalDNSClusterTypeMissingLocalities) {
5943 gpr_setenv("GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER",
5944 "true");
5945 SetNextResolution({});
5946 SetNextResolutionForLbChannelAllBalancers();
5947 // Create Logical DNS Cluster
5948 auto cluster = default_cluster_;
5949 cluster.set_type(Cluster::LOGICAL_DNS);
5950 cluster.mutable_load_assignment();
5951 balancers_[0]->ads_service()->SetCdsResource(cluster);
5952 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
5953 const auto response_state =
5954 balancers_[0]->ads_service()->cds_response_state();
5955 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
5956 EXPECT_THAT(
5957 response_state.error_message,
5958 ::testing::HasSubstr("load_assignment for LOGICAL_DNS cluster must have "
5959 "exactly one locality, found 0"));
5960 gpr_unsetenv(
5961 "GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER");
5962 }
5963
TEST_P(CdsTest,LogicalDNSClusterTypeMultipleLocalities)5964 TEST_P(CdsTest, LogicalDNSClusterTypeMultipleLocalities) {
5965 gpr_setenv("GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER",
5966 "true");
5967 SetNextResolution({});
5968 SetNextResolutionForLbChannelAllBalancers();
5969 // Create Logical DNS Cluster
5970 auto cluster = default_cluster_;
5971 cluster.set_type(Cluster::LOGICAL_DNS);
5972 auto* load_assignment = cluster.mutable_load_assignment();
5973 load_assignment->add_endpoints();
5974 load_assignment->add_endpoints();
5975 balancers_[0]->ads_service()->SetCdsResource(cluster);
5976 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
5977 const auto response_state =
5978 balancers_[0]->ads_service()->cds_response_state();
5979 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
5980 EXPECT_THAT(
5981 response_state.error_message,
5982 ::testing::HasSubstr("load_assignment for LOGICAL_DNS cluster must have "
5983 "exactly one locality, found 2"));
5984 gpr_unsetenv(
5985 "GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER");
5986 }
5987
TEST_P(CdsTest,LogicalDNSClusterTypeMissingEndpoints)5988 TEST_P(CdsTest, LogicalDNSClusterTypeMissingEndpoints) {
5989 gpr_setenv("GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER",
5990 "true");
5991 SetNextResolution({});
5992 SetNextResolutionForLbChannelAllBalancers();
5993 // Create Logical DNS Cluster
5994 auto cluster = default_cluster_;
5995 cluster.set_type(Cluster::LOGICAL_DNS);
5996 cluster.mutable_load_assignment()->add_endpoints();
5997 balancers_[0]->ads_service()->SetCdsResource(cluster);
5998 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
5999 const auto response_state =
6000 balancers_[0]->ads_service()->cds_response_state();
6001 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
6002 EXPECT_THAT(response_state.error_message,
6003 ::testing::HasSubstr(
6004 "locality for LOGICAL_DNS cluster must have exactly one "
6005 "endpoint, found 0"));
6006 gpr_unsetenv(
6007 "GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER");
6008 }
6009
TEST_P(CdsTest,LogicalDNSClusterTypeMultipleEndpoints)6010 TEST_P(CdsTest, LogicalDNSClusterTypeMultipleEndpoints) {
6011 gpr_setenv("GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER",
6012 "true");
6013 SetNextResolution({});
6014 SetNextResolutionForLbChannelAllBalancers();
6015 // Create Logical DNS Cluster
6016 auto cluster = default_cluster_;
6017 cluster.set_type(Cluster::LOGICAL_DNS);
6018 auto* locality = cluster.mutable_load_assignment()->add_endpoints();
6019 locality->add_lb_endpoints();
6020 locality->add_lb_endpoints();
6021 balancers_[0]->ads_service()->SetCdsResource(cluster);
6022 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
6023 const auto response_state =
6024 balancers_[0]->ads_service()->cds_response_state();
6025 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
6026 EXPECT_THAT(response_state.error_message,
6027 ::testing::HasSubstr(
6028 "locality for LOGICAL_DNS cluster must have exactly one "
6029 "endpoint, found 2"));
6030 gpr_unsetenv(
6031 "GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER");
6032 }
6033
TEST_P(CdsTest,LogicalDNSClusterTypeEmptyEndpoint)6034 TEST_P(CdsTest, LogicalDNSClusterTypeEmptyEndpoint) {
6035 gpr_setenv("GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER",
6036 "true");
6037 SetNextResolution({});
6038 SetNextResolutionForLbChannelAllBalancers();
6039 // Create Logical DNS Cluster
6040 auto cluster = default_cluster_;
6041 cluster.set_type(Cluster::LOGICAL_DNS);
6042 cluster.mutable_load_assignment()->add_endpoints()->add_lb_endpoints();
6043 balancers_[0]->ads_service()->SetCdsResource(cluster);
6044 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
6045 const auto response_state =
6046 balancers_[0]->ads_service()->cds_response_state();
6047 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
6048 EXPECT_THAT(response_state.error_message,
6049 ::testing::HasSubstr("LbEndpoint endpoint field not set"));
6050 gpr_unsetenv(
6051 "GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER");
6052 }
6053
TEST_P(CdsTest,LogicalDNSClusterTypeEndpointMissingAddress)6054 TEST_P(CdsTest, LogicalDNSClusterTypeEndpointMissingAddress) {
6055 gpr_setenv("GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER",
6056 "true");
6057 SetNextResolution({});
6058 SetNextResolutionForLbChannelAllBalancers();
6059 // Create Logical DNS Cluster
6060 auto cluster = default_cluster_;
6061 cluster.set_type(Cluster::LOGICAL_DNS);
6062 cluster.mutable_load_assignment()
6063 ->add_endpoints()
6064 ->add_lb_endpoints()
6065 ->mutable_endpoint();
6066 balancers_[0]->ads_service()->SetCdsResource(cluster);
6067 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
6068 const auto response_state =
6069 balancers_[0]->ads_service()->cds_response_state();
6070 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
6071 EXPECT_THAT(response_state.error_message,
6072 ::testing::HasSubstr("Endpoint address field not set"));
6073 gpr_unsetenv(
6074 "GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER");
6075 }
6076
TEST_P(CdsTest,LogicalDNSClusterTypeAddressMissingSocketAddress)6077 TEST_P(CdsTest, LogicalDNSClusterTypeAddressMissingSocketAddress) {
6078 gpr_setenv("GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER",
6079 "true");
6080 SetNextResolution({});
6081 SetNextResolutionForLbChannelAllBalancers();
6082 // Create Logical DNS Cluster
6083 auto cluster = default_cluster_;
6084 cluster.set_type(Cluster::LOGICAL_DNS);
6085 cluster.mutable_load_assignment()
6086 ->add_endpoints()
6087 ->add_lb_endpoints()
6088 ->mutable_endpoint()
6089 ->mutable_address();
6090 balancers_[0]->ads_service()->SetCdsResource(cluster);
6091 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
6092 const auto response_state =
6093 balancers_[0]->ads_service()->cds_response_state();
6094 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
6095 EXPECT_THAT(response_state.error_message,
6096 ::testing::HasSubstr("Address socket_address field not set"));
6097 gpr_unsetenv(
6098 "GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER");
6099 }
6100
TEST_P(CdsTest,LogicalDNSClusterTypeSocketAddressHasResolverName)6101 TEST_P(CdsTest, LogicalDNSClusterTypeSocketAddressHasResolverName) {
6102 gpr_setenv("GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER",
6103 "true");
6104 SetNextResolution({});
6105 SetNextResolutionForLbChannelAllBalancers();
6106 // Create Logical DNS Cluster
6107 auto cluster = default_cluster_;
6108 cluster.set_type(Cluster::LOGICAL_DNS);
6109 cluster.mutable_load_assignment()
6110 ->add_endpoints()
6111 ->add_lb_endpoints()
6112 ->mutable_endpoint()
6113 ->mutable_address()
6114 ->mutable_socket_address()
6115 ->set_resolver_name("foo");
6116 balancers_[0]->ads_service()->SetCdsResource(cluster);
6117 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
6118 const auto response_state =
6119 balancers_[0]->ads_service()->cds_response_state();
6120 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
6121 EXPECT_THAT(response_state.error_message,
6122 ::testing::HasSubstr("LOGICAL_DNS clusters must NOT have a "
6123 "custom resolver name set"));
6124 gpr_unsetenv(
6125 "GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER");
6126 }
6127
TEST_P(CdsTest,LogicalDNSClusterTypeSocketAddressMissingAddress)6128 TEST_P(CdsTest, LogicalDNSClusterTypeSocketAddressMissingAddress) {
6129 gpr_setenv("GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER",
6130 "true");
6131 SetNextResolution({});
6132 SetNextResolutionForLbChannelAllBalancers();
6133 // Create Logical DNS Cluster
6134 auto cluster = default_cluster_;
6135 cluster.set_type(Cluster::LOGICAL_DNS);
6136 cluster.mutable_load_assignment()
6137 ->add_endpoints()
6138 ->add_lb_endpoints()
6139 ->mutable_endpoint()
6140 ->mutable_address()
6141 ->mutable_socket_address();
6142 balancers_[0]->ads_service()->SetCdsResource(cluster);
6143 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
6144 const auto response_state =
6145 balancers_[0]->ads_service()->cds_response_state();
6146 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
6147 EXPECT_THAT(response_state.error_message,
6148 ::testing::HasSubstr("SocketAddress address field not set"));
6149 gpr_unsetenv(
6150 "GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER");
6151 }
6152
TEST_P(CdsTest,LogicalDNSClusterTypeSocketAddressMissingPort)6153 TEST_P(CdsTest, LogicalDNSClusterTypeSocketAddressMissingPort) {
6154 gpr_setenv("GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER",
6155 "true");
6156 SetNextResolution({});
6157 SetNextResolutionForLbChannelAllBalancers();
6158 // Create Logical DNS Cluster
6159 auto cluster = default_cluster_;
6160 cluster.set_type(Cluster::LOGICAL_DNS);
6161 cluster.mutable_load_assignment()
6162 ->add_endpoints()
6163 ->add_lb_endpoints()
6164 ->mutable_endpoint()
6165 ->mutable_address()
6166 ->mutable_socket_address()
6167 ->set_address(kServerName);
6168 balancers_[0]->ads_service()->SetCdsResource(cluster);
6169 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
6170 const auto response_state =
6171 balancers_[0]->ads_service()->cds_response_state();
6172 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
6173 EXPECT_THAT(response_state.error_message,
6174 ::testing::HasSubstr("SocketAddress port_value field not set"));
6175 gpr_unsetenv(
6176 "GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER");
6177 }
6178
TEST_P(CdsTest,AggregateClusterType)6179 TEST_P(CdsTest, AggregateClusterType) {
6180 gpr_setenv("GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER",
6181 "true");
6182 const char* kNewCluster1Name = "new_cluster_1";
6183 const char* kNewEdsService1Name = "new_eds_service_name_1";
6184 const char* kNewCluster2Name = "new_cluster_2";
6185 const char* kNewEdsService2Name = "new_eds_service_name_2";
6186 SetNextResolution({});
6187 SetNextResolutionForLbChannelAllBalancers();
6188 // Populate new EDS resources.
6189 EdsResourceArgs args1({
6190 {"locality0", CreateEndpointsForBackends(1, 2)},
6191 });
6192 EdsResourceArgs args2({
6193 {"locality0", CreateEndpointsForBackends(2, 3)},
6194 });
6195 balancers_[0]->ads_service()->SetEdsResource(
6196 BuildEdsResource(args1, kNewEdsService1Name));
6197 balancers_[0]->ads_service()->SetEdsResource(
6198 BuildEdsResource(args2, kNewEdsService2Name));
6199 // Populate new CDS resources.
6200 Cluster new_cluster1 = default_cluster_;
6201 new_cluster1.set_name(kNewCluster1Name);
6202 new_cluster1.mutable_eds_cluster_config()->set_service_name(
6203 kNewEdsService1Name);
6204 balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
6205 Cluster new_cluster2 = default_cluster_;
6206 new_cluster2.set_name(kNewCluster2Name);
6207 new_cluster2.mutable_eds_cluster_config()->set_service_name(
6208 kNewEdsService2Name);
6209 balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
6210 // Create Aggregate Cluster
6211 auto cluster = default_cluster_;
6212 CustomClusterType* custom_cluster = cluster.mutable_cluster_type();
6213 custom_cluster->set_name("envoy.clusters.aggregate");
6214 ClusterConfig cluster_config;
6215 cluster_config.add_clusters(kNewCluster1Name);
6216 cluster_config.add_clusters(kNewCluster2Name);
6217 custom_cluster->mutable_typed_config()->PackFrom(cluster_config);
6218 balancers_[0]->ads_service()->SetCdsResource(cluster);
6219 // Wait for traffic to go to backend 1.
6220 WaitForBackend(1);
6221 // Shutdown backend 1 and wait for all traffic to go to backend 2.
6222 ShutdownBackend(1);
6223 WaitForBackend(2, WaitForBackendOptions().set_allow_failures(true));
6224 EXPECT_EQ(balancers_[0]->ads_service()->cds_response_state().state,
6225 AdsServiceImpl::ResponseState::ACKED);
6226 // Bring backend 1 back and ensure all traffic go back to it.
6227 StartBackend(1);
6228 WaitForBackend(1);
6229 gpr_unsetenv(
6230 "GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER");
6231 }
6232
TEST_P(CdsTest,AggregateClusterEdsToLogicalDns)6233 TEST_P(CdsTest, AggregateClusterEdsToLogicalDns) {
6234 gpr_setenv("GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER",
6235 "true");
6236 SetNextResolution({});
6237 SetNextResolutionForLbChannelAllBalancers();
6238 const char* kNewCluster1Name = "new_cluster_1";
6239 const char* kNewEdsService1Name = "new_eds_service_name_1";
6240 const char* kLogicalDNSClusterName = "logical_dns_cluster";
6241 // Populate new EDS resources.
6242 EdsResourceArgs args1({
6243 {"locality0", CreateEndpointsForBackends(1, 2)},
6244 });
6245 balancers_[0]->ads_service()->SetEdsResource(
6246 BuildEdsResource(args1, kNewEdsService1Name));
6247 // Populate new CDS resources.
6248 Cluster new_cluster1 = default_cluster_;
6249 new_cluster1.set_name(kNewCluster1Name);
6250 new_cluster1.mutable_eds_cluster_config()->set_service_name(
6251 kNewEdsService1Name);
6252 balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
6253 // Create Logical DNS Cluster
6254 auto logical_dns_cluster = default_cluster_;
6255 logical_dns_cluster.set_name(kLogicalDNSClusterName);
6256 logical_dns_cluster.set_type(Cluster::LOGICAL_DNS);
6257 auto* address = logical_dns_cluster.mutable_load_assignment()
6258 ->add_endpoints()
6259 ->add_lb_endpoints()
6260 ->mutable_endpoint()
6261 ->mutable_address()
6262 ->mutable_socket_address();
6263 address->set_address(kServerName);
6264 address->set_port_value(443);
6265 balancers_[0]->ads_service()->SetCdsResource(logical_dns_cluster);
6266 // Create Aggregate Cluster
6267 auto cluster = default_cluster_;
6268 CustomClusterType* custom_cluster = cluster.mutable_cluster_type();
6269 custom_cluster->set_name("envoy.clusters.aggregate");
6270 ClusterConfig cluster_config;
6271 cluster_config.add_clusters(kNewCluster1Name);
6272 cluster_config.add_clusters(kLogicalDNSClusterName);
6273 custom_cluster->mutable_typed_config()->PackFrom(cluster_config);
6274 balancers_[0]->ads_service()->SetCdsResource(cluster);
6275 // Set Logical DNS result
6276 {
6277 grpc_core::ExecCtx exec_ctx;
6278 grpc_core::Resolver::Result result;
6279 result.addresses = CreateAddressListFromPortList(GetBackendPorts(2, 3));
6280 logical_dns_cluster_resolver_response_generator_->SetResponse(
6281 std::move(result));
6282 }
6283 // Wait for traffic to go to backend 1.
6284 WaitForBackend(1);
6285 // Shutdown backend 1 and wait for all traffic to go to backend 2.
6286 ShutdownBackend(1);
6287 WaitForBackend(2, WaitForBackendOptions().set_allow_failures(true));
6288 EXPECT_EQ(balancers_[0]->ads_service()->cds_response_state().state,
6289 AdsServiceImpl::ResponseState::ACKED);
6290 // Bring backend 1 back and ensure all traffic go back to it.
6291 StartBackend(1);
6292 WaitForBackend(1);
6293 gpr_unsetenv(
6294 "GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER");
6295 }
6296
TEST_P(CdsTest,AggregateClusterLogicalDnsToEds)6297 TEST_P(CdsTest, AggregateClusterLogicalDnsToEds) {
6298 gpr_setenv("GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER",
6299 "true");
6300 SetNextResolution({});
6301 SetNextResolutionForLbChannelAllBalancers();
6302 const char* kNewCluster2Name = "new_cluster_2";
6303 const char* kNewEdsService2Name = "new_eds_service_name_2";
6304 const char* kLogicalDNSClusterName = "logical_dns_cluster";
6305 // Populate new EDS resources.
6306 EdsResourceArgs args2({
6307 {"locality0", CreateEndpointsForBackends(2, 3)},
6308 });
6309 balancers_[0]->ads_service()->SetEdsResource(
6310 BuildEdsResource(args2, kNewEdsService2Name));
6311 // Populate new CDS resources.
6312 Cluster new_cluster2 = default_cluster_;
6313 new_cluster2.set_name(kNewCluster2Name);
6314 new_cluster2.mutable_eds_cluster_config()->set_service_name(
6315 kNewEdsService2Name);
6316 balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
6317 // Create Logical DNS Cluster
6318 auto logical_dns_cluster = default_cluster_;
6319 logical_dns_cluster.set_name(kLogicalDNSClusterName);
6320 logical_dns_cluster.set_type(Cluster::LOGICAL_DNS);
6321 auto* address = logical_dns_cluster.mutable_load_assignment()
6322 ->add_endpoints()
6323 ->add_lb_endpoints()
6324 ->mutable_endpoint()
6325 ->mutable_address()
6326 ->mutable_socket_address();
6327 address->set_address(kServerName);
6328 address->set_port_value(443);
6329 balancers_[0]->ads_service()->SetCdsResource(logical_dns_cluster);
6330 // Create Aggregate Cluster
6331 auto cluster = default_cluster_;
6332 CustomClusterType* custom_cluster = cluster.mutable_cluster_type();
6333 custom_cluster->set_name("envoy.clusters.aggregate");
6334 ClusterConfig cluster_config;
6335 cluster_config.add_clusters(kLogicalDNSClusterName);
6336 cluster_config.add_clusters(kNewCluster2Name);
6337 custom_cluster->mutable_typed_config()->PackFrom(cluster_config);
6338 balancers_[0]->ads_service()->SetCdsResource(cluster);
6339 // Set Logical DNS result
6340 {
6341 grpc_core::ExecCtx exec_ctx;
6342 grpc_core::Resolver::Result result;
6343 result.addresses = CreateAddressListFromPortList(GetBackendPorts(1, 2));
6344 logical_dns_cluster_resolver_response_generator_->SetResponse(
6345 std::move(result));
6346 }
6347 // Wait for traffic to go to backend 1.
6348 WaitForBackend(1);
6349 // Shutdown backend 1 and wait for all traffic to go to backend 2.
6350 ShutdownBackend(1);
6351 WaitForBackend(2, WaitForBackendOptions().set_allow_failures(true));
6352 EXPECT_EQ(balancers_[0]->ads_service()->cds_response_state().state,
6353 AdsServiceImpl::ResponseState::ACKED);
6354 // Bring backend 1 back and ensure all traffic go back to it.
6355 StartBackend(1);
6356 WaitForBackend(1);
6357 gpr_unsetenv(
6358 "GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER");
6359 }
6360
6361 // Test that CDS client should send a NACK if cluster type is Logical DNS but
6362 // the feature is not yet supported.
TEST_P(CdsTest,LogicalDNSClusterTypeDisabled)6363 TEST_P(CdsTest, LogicalDNSClusterTypeDisabled) {
6364 auto cluster = default_cluster_;
6365 cluster.set_type(Cluster::LOGICAL_DNS);
6366 balancers_[0]->ads_service()->SetCdsResource(cluster);
6367 SetNextResolution({});
6368 SetNextResolutionForLbChannelAllBalancers();
6369 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
6370 const auto response_state =
6371 balancers_[0]->ads_service()->cds_response_state();
6372 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
6373 EXPECT_THAT(response_state.error_message,
6374 ::testing::HasSubstr("DiscoveryType is not valid."));
6375 }
6376
6377 // Test that CDS client should send a NACK if cluster type is AGGREGATE but
6378 // the feature is not yet supported.
TEST_P(CdsTest,AggregateClusterTypeDisabled)6379 TEST_P(CdsTest, AggregateClusterTypeDisabled) {
6380 auto cluster = default_cluster_;
6381 CustomClusterType* custom_cluster = cluster.mutable_cluster_type();
6382 custom_cluster->set_name("envoy.clusters.aggregate");
6383 ClusterConfig cluster_config;
6384 cluster_config.add_clusters("cluster1");
6385 cluster_config.add_clusters("cluster2");
6386 custom_cluster->mutable_typed_config()->PackFrom(cluster_config);
6387 cluster.set_type(Cluster::LOGICAL_DNS);
6388 balancers_[0]->ads_service()->SetCdsResource(cluster);
6389 SetNextResolution({});
6390 SetNextResolutionForLbChannelAllBalancers();
6391 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
6392 const auto response_state =
6393 balancers_[0]->ads_service()->cds_response_state();
6394 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
6395 EXPECT_THAT(response_state.error_message,
6396 ::testing::HasSubstr("DiscoveryType is not valid."));
6397 }
6398
6399 // Tests that CDS client should send a NACK if the cluster type in CDS
6400 // response is unsupported.
TEST_P(CdsTest,UnsupportedClusterType)6401 TEST_P(CdsTest, UnsupportedClusterType) {
6402 auto cluster = default_cluster_;
6403 cluster.set_type(Cluster::STATIC);
6404 balancers_[0]->ads_service()->SetCdsResource(cluster);
6405 SetNextResolution({});
6406 SetNextResolutionForLbChannelAllBalancers();
6407 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
6408 const auto response_state =
6409 balancers_[0]->ads_service()->cds_response_state();
6410 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
6411 EXPECT_THAT(response_state.error_message,
6412 ::testing::HasSubstr("DiscoveryType is not valid."));
6413 }
6414
6415 // Tests that the NACK for multiple bad resources includes both errors.
TEST_P(CdsTest,MultipleBadResources)6416 TEST_P(CdsTest, MultipleBadResources) {
6417 constexpr char kClusterName2[] = "cluster_name_2";
6418 constexpr char kClusterName3[] = "cluster_name_3";
6419 // Add cluster with unsupported type.
6420 auto cluster = default_cluster_;
6421 cluster.set_name(kClusterName2);
6422 cluster.set_type(Cluster::STATIC);
6423 balancers_[0]->ads_service()->SetCdsResource(cluster);
6424 // Add second cluster with the same error.
6425 cluster.set_name(kClusterName3);
6426 balancers_[0]->ads_service()->SetCdsResource(cluster);
6427 // Change RouteConfig to point to all clusters.
6428 RouteConfiguration route_config = default_route_config_;
6429 route_config.mutable_virtual_hosts(0)->clear_routes();
6430 // First route: default cluster, selected based on header.
6431 auto* route = route_config.mutable_virtual_hosts(0)->add_routes();
6432 route->mutable_match()->set_prefix("");
6433 auto* header_matcher = route->mutable_match()->add_headers();
6434 header_matcher->set_name("cluster");
6435 header_matcher->set_exact_match(kDefaultClusterName);
6436 route->mutable_route()->set_cluster(kDefaultClusterName);
6437 // Second route: cluster 2, selected based on header.
6438 route = route_config.mutable_virtual_hosts(0)->add_routes();
6439 route->mutable_match()->set_prefix("");
6440 header_matcher = route->mutable_match()->add_headers();
6441 header_matcher->set_name("cluster");
6442 header_matcher->set_exact_match(kClusterName2);
6443 route->mutable_route()->set_cluster(kClusterName2);
6444 // Third route: cluster 3, used by default.
6445 route = route_config.mutable_virtual_hosts(0)->add_routes();
6446 route->mutable_match()->set_prefix("");
6447 route->mutable_route()->set_cluster(kClusterName3);
6448 SetRouteConfiguration(0, route_config);
6449 // Add EDS resource.
6450 EdsResourceArgs args({
6451 {"locality0", CreateEndpointsForBackends(0, 1)},
6452 });
6453 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
6454 // Send RPC.
6455 SetNextResolution({});
6456 SetNextResolutionForLbChannelAllBalancers();
6457 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
6458 const auto response_state =
6459 balancers_[0]->ads_service()->cds_response_state();
6460 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
6461 EXPECT_THAT(
6462 response_state.error_message,
6463 ::testing::ContainsRegex(absl::StrCat(kClusterName2,
6464 ": validation error.*"
6465 "DiscoveryType is not valid.*",
6466 kClusterName3,
6467 ": validation error.*"
6468 "DiscoveryType is not valid")));
6469 // RPCs for default cluster should succeed.
6470 std::vector<std::pair<std::string, std::string>> metadata_default_cluster = {
6471 {"cluster", kDefaultClusterName},
6472 };
6473 CheckRpcSendOk(
6474 1, RpcOptions().set_metadata(std::move(metadata_default_cluster)));
6475 // RPCs for cluster 2 should fail.
6476 std::vector<std::pair<std::string, std::string>> metadata_cluster_2 = {
6477 {"cluster", kClusterName2},
6478 };
6479 CheckRpcSendFailure(CheckRpcSendFailureOptions().set_rpc_options(
6480 RpcOptions().set_metadata(std::move(metadata_cluster_2))));
6481 }
6482
6483 // Tests that we don't trigger does-not-exist callbacks for a resource
6484 // that was previously valid but is updated to be invalid.
TEST_P(CdsTest,InvalidClusterStillExistsIfPreviouslyCached)6485 TEST_P(CdsTest, InvalidClusterStillExistsIfPreviouslyCached) {
6486 EdsResourceArgs args({
6487 {"locality0", CreateEndpointsForBackends(0, 1)},
6488 });
6489 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
6490 SetNextResolution({});
6491 SetNextResolutionForLbChannelAllBalancers();
6492 // Check that everything works.
6493 CheckRpcSendOk();
6494 // Now send an update changing the Cluster to be invalid.
6495 auto cluster = default_cluster_;
6496 cluster.set_type(Cluster::STATIC);
6497 balancers_[0]->ads_service()->SetCdsResource(cluster);
6498 // Wait for xDS server to see NACK.
6499 auto deadline = absl::Now() + absl::Seconds(30);
6500 do {
6501 CheckRpcSendOk();
6502 ASSERT_LT(absl::Now(), deadline);
6503 } while (balancers_[0]->ads_service()->cds_response_state().state !=
6504 AdsServiceImpl::ResponseState::NACKED);
6505 EXPECT_THAT(balancers_[0]->ads_service()->cds_response_state().error_message,
6506 ::testing::ContainsRegex(absl::StrCat(
6507 kDefaultClusterName,
6508 ": validation error.*DiscoveryType is not valid")));
6509 // Check one more time, just to make sure it still works after NACK.
6510 CheckRpcSendOk();
6511 }
6512
6513 // Tests that CDS client should send a NACK if the eds_config in CDS response
6514 // is other than ADS.
TEST_P(CdsTest,WrongEdsConfig)6515 TEST_P(CdsTest, WrongEdsConfig) {
6516 auto cluster = default_cluster_;
6517 cluster.mutable_eds_cluster_config()->mutable_eds_config()->mutable_self();
6518 balancers_[0]->ads_service()->SetCdsResource(cluster);
6519 SetNextResolution({});
6520 SetNextResolutionForLbChannelAllBalancers();
6521 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
6522 const auto response_state =
6523 balancers_[0]->ads_service()->cds_response_state();
6524 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
6525 EXPECT_THAT(response_state.error_message,
6526 ::testing::HasSubstr("EDS ConfigSource is not ADS."));
6527 }
6528
6529 // Tests that CDS client should send a NACK if the lb_policy in CDS response
6530 // is other than ROUND_ROBIN.
TEST_P(CdsTest,WrongLbPolicy)6531 TEST_P(CdsTest, WrongLbPolicy) {
6532 auto cluster = default_cluster_;
6533 cluster.set_lb_policy(Cluster::LEAST_REQUEST);
6534 balancers_[0]->ads_service()->SetCdsResource(cluster);
6535 SetNextResolution({});
6536 SetNextResolutionForLbChannelAllBalancers();
6537 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
6538 const auto response_state =
6539 balancers_[0]->ads_service()->cds_response_state();
6540 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
6541 EXPECT_THAT(response_state.error_message,
6542 ::testing::HasSubstr("LB policy is not supported."));
6543 }
6544
6545 // Tests that CDS client should send a NACK if the lrs_server in CDS response
6546 // is other than SELF.
TEST_P(CdsTest,WrongLrsServer)6547 TEST_P(CdsTest, WrongLrsServer) {
6548 auto cluster = default_cluster_;
6549 cluster.mutable_lrs_server()->mutable_ads();
6550 balancers_[0]->ads_service()->SetCdsResource(cluster);
6551 SetNextResolution({});
6552 SetNextResolutionForLbChannelAllBalancers();
6553 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
6554 const auto response_state =
6555 balancers_[0]->ads_service()->cds_response_state();
6556 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
6557 EXPECT_THAT(response_state.error_message,
6558 ::testing::HasSubstr("LRS ConfigSource is not self."));
6559 }
6560
6561 // Tests that ring hash policy that hashes using channel id ensures all RPCs
6562 // to go 1 particular backend.
TEST_P(CdsTest,RingHashChannelIdHashing)6563 TEST_P(CdsTest, RingHashChannelIdHashing) {
6564 auto cluster = default_cluster_;
6565 cluster.set_lb_policy(Cluster::RING_HASH);
6566 balancers_[0]->ads_service()->SetCdsResource(cluster);
6567 auto new_route_config = default_route_config_;
6568 auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
6569 auto* hash_policy = route->mutable_route()->add_hash_policy();
6570 hash_policy->mutable_filter_state()->set_key("io.grpc.channel_id");
6571 SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
6572 EdsResourceArgs args({
6573 {"locality0", CreateEndpointsForBackends()},
6574 });
6575 balancers_[0]->ads_service()->SetEdsResource(
6576 BuildEdsResource(args, DefaultEdsServiceName()));
6577 SetNextResolutionForLbChannelAllBalancers();
6578 CheckRpcSendOk(100);
6579 bool found = false;
6580 for (size_t i = 0; i < backends_.size(); ++i) {
6581 if (backends_[i]->backend_service()->request_count() > 0) {
6582 EXPECT_EQ(backends_[i]->backend_service()->request_count(), 100)
6583 << "backend " << i;
6584 EXPECT_FALSE(found) << "backend " << i;
6585 found = true;
6586 }
6587 }
6588 EXPECT_TRUE(found);
6589 }
6590
6591 // Tests that ring hash policy that hashes using a header value can spread
6592 // RPCs across all the backends.
TEST_P(CdsTest,RingHashHeaderHashing)6593 TEST_P(CdsTest, RingHashHeaderHashing) {
6594 auto cluster = default_cluster_;
6595 cluster.set_lb_policy(Cluster::RING_HASH);
6596 balancers_[0]->ads_service()->SetCdsResource(cluster);
6597 auto new_route_config = default_route_config_;
6598 auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
6599 auto* hash_policy = route->mutable_route()->add_hash_policy();
6600 hash_policy->mutable_header()->set_header_name("address_hash");
6601 SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
6602 EdsResourceArgs args({
6603 {"locality0", CreateEndpointsForBackends()},
6604 });
6605 balancers_[0]->ads_service()->SetEdsResource(
6606 BuildEdsResource(args, DefaultEdsServiceName()));
6607 SetNextResolutionForLbChannelAllBalancers();
6608 // Note each type of RPC will contains a header value that will always be
6609 // hashed to a specific backend as the header value matches the value used
6610 // to create the entry in the ring.
6611 std::vector<std::pair<std::string, std::string>> metadata = {
6612 {"address_hash", CreateMetadataValueThatHashesToBackend(0)}};
6613 std::vector<std::pair<std::string, std::string>> metadata1 = {
6614 {"address_hash", CreateMetadataValueThatHashesToBackend(1)}};
6615 std::vector<std::pair<std::string, std::string>> metadata2 = {
6616 {"address_hash", CreateMetadataValueThatHashesToBackend(2)}};
6617 std::vector<std::pair<std::string, std::string>> metadata3 = {
6618 {"address_hash", CreateMetadataValueThatHashesToBackend(3)}};
6619 const auto rpc_options = RpcOptions().set_metadata(std::move(metadata));
6620 const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1));
6621 const auto rpc_options2 = RpcOptions().set_metadata(std::move(metadata2));
6622 const auto rpc_options3 = RpcOptions().set_metadata(std::move(metadata3));
6623 WaitForBackend(0, WaitForBackendOptions(), rpc_options);
6624 WaitForBackend(1, WaitForBackendOptions(), rpc_options1);
6625 WaitForBackend(2, WaitForBackendOptions(), rpc_options2);
6626 WaitForBackend(3, WaitForBackendOptions(), rpc_options3);
6627 CheckRpcSendOk(100, rpc_options);
6628 CheckRpcSendOk(100, rpc_options1);
6629 CheckRpcSendOk(100, rpc_options2);
6630 CheckRpcSendOk(100, rpc_options3);
6631 for (size_t i = 0; i < backends_.size(); ++i) {
6632 EXPECT_EQ(100, backends_[i]->backend_service()->request_count());
6633 }
6634 }
6635
6636 // Tests that ring hash policy that hashes using a header value and regex
6637 // rewrite to aggregate RPCs to 1 backend.
TEST_P(CdsTest,RingHashHeaderHashingWithRegexRewrite)6638 TEST_P(CdsTest, RingHashHeaderHashingWithRegexRewrite) {
6639 auto cluster = default_cluster_;
6640 cluster.set_lb_policy(Cluster::RING_HASH);
6641 balancers_[0]->ads_service()->SetCdsResource(cluster);
6642 auto new_route_config = default_route_config_;
6643 auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
6644 auto* hash_policy = route->mutable_route()->add_hash_policy();
6645 hash_policy->mutable_header()->set_header_name("address_hash");
6646 hash_policy->mutable_header()
6647 ->mutable_regex_rewrite()
6648 ->mutable_pattern()
6649 ->set_regex("[0-9]+");
6650 hash_policy->mutable_header()->mutable_regex_rewrite()->set_substitution(
6651 "foo");
6652 SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
6653 EdsResourceArgs args({
6654 {"locality0", CreateEndpointsForBackends()},
6655 });
6656 balancers_[0]->ads_service()->SetEdsResource(
6657 BuildEdsResource(args, DefaultEdsServiceName()));
6658 SetNextResolutionForLbChannelAllBalancers();
6659 std::vector<std::pair<std::string, std::string>> metadata = {
6660 {"address_hash", CreateMetadataValueThatHashesToBackend(0)}};
6661 std::vector<std::pair<std::string, std::string>> metadata1 = {
6662 {"address_hash", CreateMetadataValueThatHashesToBackend(1)}};
6663 std::vector<std::pair<std::string, std::string>> metadata2 = {
6664 {"address_hash", CreateMetadataValueThatHashesToBackend(2)}};
6665 std::vector<std::pair<std::string, std::string>> metadata3 = {
6666 {"address_hash", CreateMetadataValueThatHashesToBackend(3)}};
6667 const auto rpc_options = RpcOptions().set_metadata(std::move(metadata));
6668 const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1));
6669 const auto rpc_options2 = RpcOptions().set_metadata(std::move(metadata2));
6670 const auto rpc_options3 = RpcOptions().set_metadata(std::move(metadata3));
6671 CheckRpcSendOk(100, rpc_options);
6672 CheckRpcSendOk(100, rpc_options1);
6673 CheckRpcSendOk(100, rpc_options2);
6674 CheckRpcSendOk(100, rpc_options3);
6675 bool found = false;
6676 for (size_t i = 0; i < backends_.size(); ++i) {
6677 if (backends_[i]->backend_service()->request_count() > 0) {
6678 EXPECT_EQ(backends_[i]->backend_service()->request_count(), 400)
6679 << "backend " << i;
6680 EXPECT_FALSE(found) << "backend " << i;
6681 found = true;
6682 }
6683 }
6684 EXPECT_TRUE(found);
6685 }
6686
6687 // Tests that ring hash policy that hashes using a random value.
TEST_P(CdsTest,RingHashNoHashPolicy)6688 TEST_P(CdsTest, RingHashNoHashPolicy) {
6689 const double kDistribution50Percent = 0.5;
6690 const double kErrorTolerance = 0.05;
6691 const uint32_t kRpcTimeoutMs = 10000;
6692 const size_t kNumRpcs =
6693 ComputeIdealNumRpcs(kDistribution50Percent, kErrorTolerance);
6694 auto cluster = default_cluster_;
6695 // Increasing min ring size for random distribution.
6696 cluster.mutable_ring_hash_lb_config()->mutable_minimum_ring_size()->set_value(
6697 100000);
6698 cluster.set_lb_policy(Cluster::RING_HASH);
6699 balancers_[0]->ads_service()->SetCdsResource(cluster);
6700 EdsResourceArgs args({{"locality0", CreateEndpointsForBackends(0, 2)}});
6701 balancers_[0]->ads_service()->SetEdsResource(
6702 BuildEdsResource(args, DefaultEdsServiceName()));
6703 SetNextResolutionForLbChannelAllBalancers();
6704 // TODO(donnadionne): remove extended timeout after ring creation
6705 // optimization.
6706 WaitForAllBackends(0, 2, WaitForBackendOptions(),
6707 RpcOptions().set_timeout_ms(kRpcTimeoutMs));
6708 CheckRpcSendOk(kNumRpcs);
6709 const int request_count_1 = backends_[0]->backend_service()->request_count();
6710 const int request_count_2 = backends_[1]->backend_service()->request_count();
6711 EXPECT_THAT(static_cast<double>(request_count_1) / kNumRpcs,
6712 ::testing::DoubleNear(kDistribution50Percent, kErrorTolerance));
6713 EXPECT_THAT(static_cast<double>(request_count_2) / kNumRpcs,
6714 ::testing::DoubleNear(kDistribution50Percent, kErrorTolerance));
6715 }
6716
6717 // Test that ring hash policy evaluation will continue past the terminal
6718 // policy if no results are produced yet.
TEST_P(CdsTest,RingHashContinuesPastTerminalPolicyThatDoesNotProduceResult)6719 TEST_P(CdsTest, RingHashContinuesPastTerminalPolicyThatDoesNotProduceResult) {
6720 auto cluster = default_cluster_;
6721 cluster.set_lb_policy(Cluster::RING_HASH);
6722 balancers_[0]->ads_service()->SetCdsResource(cluster);
6723 auto new_route_config = default_route_config_;
6724 auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
6725 auto* hash_policy = route->mutable_route()->add_hash_policy();
6726 hash_policy->mutable_header()->set_header_name("header_not_present");
6727 hash_policy->set_terminal(true);
6728 auto* hash_policy2 = route->mutable_route()->add_hash_policy();
6729 hash_policy2->mutable_header()->set_header_name("address_hash");
6730 SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
6731 EdsResourceArgs args({{"locality0", CreateEndpointsForBackends(0, 2)}});
6732 balancers_[0]->ads_service()->SetEdsResource(
6733 BuildEdsResource(args, DefaultEdsServiceName()));
6734 SetNextResolutionForLbChannelAllBalancers();
6735 std::vector<std::pair<std::string, std::string>> metadata = {
6736 {"address_hash", CreateMetadataValueThatHashesToBackend(0)}};
6737 const auto rpc_options = RpcOptions().set_metadata(std::move(metadata));
6738 CheckRpcSendOk(100, rpc_options);
6739 EXPECT_EQ(backends_[0]->backend_service()->request_count(), 100);
6740 EXPECT_EQ(backends_[1]->backend_service()->request_count(), 0);
6741 }
6742
6743 // Test random hash is used when header hashing specified a header field that
6744 // the RPC did not have.
TEST_P(CdsTest,RingHashOnHeaderThatIsNotPresent)6745 TEST_P(CdsTest, RingHashOnHeaderThatIsNotPresent) {
6746 const double kDistribution50Percent = 0.5;
6747 const double kErrorTolerance = 0.05;
6748 const uint32_t kRpcTimeoutMs = 10000;
6749 const size_t kNumRpcs =
6750 ComputeIdealNumRpcs(kDistribution50Percent, kErrorTolerance);
6751 auto cluster = default_cluster_;
6752 // Increasing min ring size for random distribution.
6753 cluster.mutable_ring_hash_lb_config()->mutable_minimum_ring_size()->set_value(
6754 100000);
6755 cluster.set_lb_policy(Cluster::RING_HASH);
6756 balancers_[0]->ads_service()->SetCdsResource(cluster);
6757 auto new_route_config = default_route_config_;
6758 auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
6759 auto* hash_policy = route->mutable_route()->add_hash_policy();
6760 hash_policy->mutable_header()->set_header_name("header_not_present");
6761 SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
6762 EdsResourceArgs args({{"locality0", CreateEndpointsForBackends(0, 2)}});
6763 balancers_[0]->ads_service()->SetEdsResource(
6764 BuildEdsResource(args, DefaultEdsServiceName()));
6765 SetNextResolutionForLbChannelAllBalancers();
6766 std::vector<std::pair<std::string, std::string>> metadata = {
6767 {"unmatched_header", absl::StrFormat("%" PRIu32, rand())},
6768 };
6769 const auto rpc_options = RpcOptions().set_metadata(std::move(metadata));
6770 // TODO(donnadionne): remove extended timeout after ring creation
6771 // optimization.
6772 WaitForAllBackends(0, 2, WaitForBackendOptions(),
6773 RpcOptions().set_timeout_ms(kRpcTimeoutMs));
6774 CheckRpcSendOk(kNumRpcs, rpc_options);
6775 const int request_count_1 = backends_[0]->backend_service()->request_count();
6776 const int request_count_2 = backends_[1]->backend_service()->request_count();
6777 EXPECT_THAT(static_cast<double>(request_count_1) / kNumRpcs,
6778 ::testing::DoubleNear(kDistribution50Percent, kErrorTolerance));
6779 EXPECT_THAT(static_cast<double>(request_count_2) / kNumRpcs,
6780 ::testing::DoubleNear(kDistribution50Percent, kErrorTolerance));
6781 }
6782
6783 // Test random hash is used when only unsupported hash policies are
6784 // configured.
TEST_P(CdsTest,RingHashUnsupportedHashPolicyDefaultToRandomHashing)6785 TEST_P(CdsTest, RingHashUnsupportedHashPolicyDefaultToRandomHashing) {
6786 const double kDistribution50Percent = 0.5;
6787 const double kErrorTolerance = 0.05;
6788 const uint32_t kRpcTimeoutMs = 10000;
6789 const size_t kNumRpcs =
6790 ComputeIdealNumRpcs(kDistribution50Percent, kErrorTolerance);
6791 auto cluster = default_cluster_;
6792 // Increasing min ring size for random distribution.
6793 cluster.mutable_ring_hash_lb_config()->mutable_minimum_ring_size()->set_value(
6794 100000);
6795 cluster.set_lb_policy(Cluster::RING_HASH);
6796 balancers_[0]->ads_service()->SetCdsResource(cluster);
6797 auto new_route_config = default_route_config_;
6798 auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
6799 auto* hash_policy_unsupported_1 = route->mutable_route()->add_hash_policy();
6800 hash_policy_unsupported_1->mutable_cookie()->set_name("cookie");
6801 auto* hash_policy_unsupported_2 = route->mutable_route()->add_hash_policy();
6802 hash_policy_unsupported_2->mutable_connection_properties()->set_source_ip(
6803 true);
6804 auto* hash_policy_unsupported_3 = route->mutable_route()->add_hash_policy();
6805 hash_policy_unsupported_3->mutable_query_parameter()->set_name(
6806 "query_parameter");
6807 SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
6808 EdsResourceArgs args({{"locality0", CreateEndpointsForBackends(0, 2)}});
6809 balancers_[0]->ads_service()->SetEdsResource(
6810 BuildEdsResource(args, DefaultEdsServiceName()));
6811 SetNextResolutionForLbChannelAllBalancers();
6812 // TODO(donnadionne): remove extended timeout after ring creation
6813 // optimization.
6814 WaitForAllBackends(0, 2, WaitForBackendOptions(),
6815 RpcOptions().set_timeout_ms(kRpcTimeoutMs));
6816 CheckRpcSendOk(kNumRpcs);
6817 const int request_count_1 = backends_[0]->backend_service()->request_count();
6818 const int request_count_2 = backends_[1]->backend_service()->request_count();
6819 EXPECT_THAT(static_cast<double>(request_count_1) / kNumRpcs,
6820 ::testing::DoubleNear(kDistribution50Percent, kErrorTolerance));
6821 EXPECT_THAT(static_cast<double>(request_count_2) / kNumRpcs,
6822 ::testing::DoubleNear(kDistribution50Percent, kErrorTolerance));
6823 }
6824
6825 // Tests that ring hash policy that hashes using a random value can spread
6826 // RPCs across all the backends according to locality weight.
TEST_P(CdsTest,RingHashRandomHashingDistributionAccordingToEndpointWeight)6827 TEST_P(CdsTest, RingHashRandomHashingDistributionAccordingToEndpointWeight) {
6828 const size_t kWeight1 = 1;
6829 const size_t kWeight2 = 2;
6830 const size_t kWeightTotal = kWeight1 + kWeight2;
6831 const double kWeight33Percent = static_cast<double>(kWeight1) / kWeightTotal;
6832 const double kWeight66Percent = static_cast<double>(kWeight2) / kWeightTotal;
6833 const double kErrorTolerance = 0.05;
6834 const uint32_t kRpcTimeoutMs = 10000;
6835 const size_t kNumRpcs =
6836 ComputeIdealNumRpcs(kWeight33Percent, kErrorTolerance);
6837 auto cluster = default_cluster_;
6838 // Increasing min ring size for random distribution.
6839 cluster.mutable_ring_hash_lb_config()->mutable_minimum_ring_size()->set_value(
6840 100000);
6841 cluster.set_lb_policy(Cluster::RING_HASH);
6842 balancers_[0]->ads_service()->SetCdsResource(cluster);
6843 EdsResourceArgs args({{"locality0",
6844 {CreateEndpoint(0, HealthStatus::UNKNOWN, 1),
6845 CreateEndpoint(1, HealthStatus::UNKNOWN, 2)}}});
6846 balancers_[0]->ads_service()->SetEdsResource(
6847 BuildEdsResource(args, DefaultEdsServiceName()));
6848 SetNextResolutionForLbChannelAllBalancers();
6849 // TODO(donnadionne): remove extended timeout after ring creation
6850 // optimization.
6851 WaitForAllBackends(0, 2, WaitForBackendOptions(),
6852 RpcOptions().set_timeout_ms(kRpcTimeoutMs));
6853 CheckRpcSendOk(kNumRpcs);
6854 const int weight_33_request_count =
6855 backends_[0]->backend_service()->request_count();
6856 const int weight_66_request_count =
6857 backends_[1]->backend_service()->request_count();
6858 EXPECT_THAT(static_cast<double>(weight_33_request_count) / kNumRpcs,
6859 ::testing::DoubleNear(kWeight33Percent, kErrorTolerance));
6860 EXPECT_THAT(static_cast<double>(weight_66_request_count) / kNumRpcs,
6861 ::testing::DoubleNear(kWeight66Percent, kErrorTolerance));
6862 }
6863
6864 // Tests that ring hash policy that hashes using a random value can spread
6865 // RPCs across all the backends according to locality weight.
TEST_P(CdsTest,RingHashRandomHashingDistributionAccordingToLocalityAndEndpointWeight)6866 TEST_P(CdsTest,
6867 RingHashRandomHashingDistributionAccordingToLocalityAndEndpointWeight) {
6868 const size_t kWeight1 = 1 * 1;
6869 const size_t kWeight2 = 2 * 2;
6870 const size_t kWeightTotal = kWeight1 + kWeight2;
6871 const double kWeight20Percent = static_cast<double>(kWeight1) / kWeightTotal;
6872 const double kWeight80Percent = static_cast<double>(kWeight2) / kWeightTotal;
6873 const double kErrorTolerance = 0.05;
6874 const uint32_t kRpcTimeoutMs = 10000;
6875 const size_t kNumRpcs =
6876 ComputeIdealNumRpcs(kWeight20Percent, kErrorTolerance);
6877 auto cluster = default_cluster_;
6878 // Increasing min ring size for random distribution.
6879 cluster.mutable_ring_hash_lb_config()->mutable_minimum_ring_size()->set_value(
6880 100000);
6881 cluster.set_lb_policy(Cluster::RING_HASH);
6882 balancers_[0]->ads_service()->SetCdsResource(cluster);
6883 EdsResourceArgs args(
6884 {{"locality0", {CreateEndpoint(0, HealthStatus::UNKNOWN, 1)}, 1},
6885 {"locality1", {CreateEndpoint(1, HealthStatus::UNKNOWN, 2)}, 2}});
6886 balancers_[0]->ads_service()->SetEdsResource(
6887 BuildEdsResource(args, DefaultEdsServiceName()));
6888 SetNextResolutionForLbChannelAllBalancers();
6889 // TODO(donnadionne): remove extended timeout after ring creation
6890 // optimization.
6891 WaitForAllBackends(0, 2, WaitForBackendOptions(),
6892 RpcOptions().set_timeout_ms(kRpcTimeoutMs));
6893 CheckRpcSendOk(kNumRpcs);
6894 const int weight_20_request_count =
6895 backends_[0]->backend_service()->request_count();
6896 const int weight_80_request_count =
6897 backends_[1]->backend_service()->request_count();
6898 EXPECT_THAT(static_cast<double>(weight_20_request_count) / kNumRpcs,
6899 ::testing::DoubleNear(kWeight20Percent, kErrorTolerance));
6900 EXPECT_THAT(static_cast<double>(weight_80_request_count) / kNumRpcs,
6901 ::testing::DoubleNear(kWeight80Percent, kErrorTolerance));
6902 }
6903
6904 // Tests round robin is not implacted by the endpoint weight, and that the
6905 // localities in a locality map are picked according to their weights.
TEST_P(CdsTest,RingHashEndpointWeightDoesNotImpactWeightedRoundRobin)6906 TEST_P(CdsTest, RingHashEndpointWeightDoesNotImpactWeightedRoundRobin) {
6907 SetNextResolution({});
6908 SetNextResolutionForLbChannelAllBalancers();
6909 const int kLocalityWeight0 = 2;
6910 const int kLocalityWeight1 = 8;
6911 const int kTotalLocalityWeight = kLocalityWeight0 + kLocalityWeight1;
6912 const double kLocalityWeightRate0 =
6913 static_cast<double>(kLocalityWeight0) / kTotalLocalityWeight;
6914 const double kLocalityWeightRate1 =
6915 static_cast<double>(kLocalityWeight1) / kTotalLocalityWeight;
6916 const double kErrorTolerance = 0.05;
6917 const size_t kNumRpcs =
6918 ComputeIdealNumRpcs(kLocalityWeightRate0, kErrorTolerance);
6919 // ADS response contains 2 localities, each of which contains 1 backend.
6920 EdsResourceArgs args({
6921 {"locality0",
6922 {CreateEndpoint(0, HealthStatus::UNKNOWN, 8)},
6923 kLocalityWeight0},
6924 {"locality1",
6925 {CreateEndpoint(1, HealthStatus::UNKNOWN, 2)},
6926 kLocalityWeight1},
6927 });
6928 balancers_[0]->ads_service()->SetEdsResource(
6929 BuildEdsResource(args, DefaultEdsServiceName()));
6930 // Wait for both backends to be ready.
6931 WaitForAllBackends(0, 2);
6932 // Send kNumRpcs RPCs.
6933 CheckRpcSendOk(kNumRpcs);
6934 // The locality picking rates should be roughly equal to the expectation.
6935 const double locality_picked_rate_0 =
6936 static_cast<double>(backends_[0]->backend_service()->request_count()) /
6937 kNumRpcs;
6938 const double locality_picked_rate_1 =
6939 static_cast<double>(backends_[1]->backend_service()->request_count()) /
6940 kNumRpcs;
6941 EXPECT_THAT(locality_picked_rate_0,
6942 ::testing::DoubleNear(kLocalityWeightRate0, kErrorTolerance));
6943 EXPECT_THAT(locality_picked_rate_1,
6944 ::testing::DoubleNear(kLocalityWeightRate1, kErrorTolerance));
6945 }
6946
6947 // Tests that ring hash policy that hashes using a fixed string ensures all
6948 // RPCs to go 1 particular backend; and that subsequent hashing policies are
6949 // ignored due to the setting of terminal.
TEST_P(CdsTest,RingHashFixedHashingTerminalPolicy)6950 TEST_P(CdsTest, RingHashFixedHashingTerminalPolicy) {
6951 auto cluster = default_cluster_;
6952 cluster.set_lb_policy(Cluster::RING_HASH);
6953 balancers_[0]->ads_service()->SetCdsResource(cluster);
6954 auto new_route_config = default_route_config_;
6955 auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
6956 auto* hash_policy = route->mutable_route()->add_hash_policy();
6957 hash_policy->mutable_header()->set_header_name("fixed_string");
6958 hash_policy->set_terminal(true);
6959 auto* hash_policy_to_be_ignored = route->mutable_route()->add_hash_policy();
6960 hash_policy_to_be_ignored->mutable_header()->set_header_name("random_string");
6961 SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
6962 EdsResourceArgs args({
6963 {"locality0", CreateEndpointsForBackends()},
6964 });
6965 balancers_[0]->ads_service()->SetEdsResource(
6966 BuildEdsResource(args, DefaultEdsServiceName()));
6967 SetNextResolutionForLbChannelAllBalancers();
6968 std::vector<std::pair<std::string, std::string>> metadata = {
6969 {"fixed_string", "fixed_value"},
6970 {"random_string", absl::StrFormat("%" PRIu32, rand())},
6971 };
6972 const auto rpc_options = RpcOptions().set_metadata(std::move(metadata));
6973 CheckRpcSendOk(100, rpc_options);
6974 bool found = false;
6975 for (size_t i = 0; i < backends_.size(); ++i) {
6976 if (backends_[i]->backend_service()->request_count() > 0) {
6977 EXPECT_EQ(backends_[i]->backend_service()->request_count(), 100)
6978 << "backend " << i;
6979 EXPECT_FALSE(found) << "backend " << i;
6980 found = true;
6981 }
6982 }
6983 EXPECT_TRUE(found);
6984 }
6985
6986 // Test that the channel will go from idle to ready via connecting;
6987 // (tho it is not possible to catch the connecting state before moving to
6988 // ready)
TEST_P(CdsTest,RingHashIdleToReady)6989 TEST_P(CdsTest, RingHashIdleToReady) {
6990 auto cluster = default_cluster_;
6991 cluster.set_lb_policy(Cluster::RING_HASH);
6992 balancers_[0]->ads_service()->SetCdsResource(cluster);
6993 auto new_route_config = default_route_config_;
6994 auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
6995 auto* hash_policy = route->mutable_route()->add_hash_policy();
6996 hash_policy->mutable_filter_state()->set_key("io.grpc.channel_id");
6997 SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
6998 EdsResourceArgs args({
6999 {"locality0", CreateEndpointsForBackends()},
7000 });
7001 balancers_[0]->ads_service()->SetEdsResource(
7002 BuildEdsResource(args, DefaultEdsServiceName()));
7003 SetNextResolutionForLbChannelAllBalancers();
7004 EXPECT_EQ(GRPC_CHANNEL_IDLE, channel_->GetState(false));
7005 CheckRpcSendOk();
7006 EXPECT_EQ(GRPC_CHANNEL_READY, channel_->GetState(false));
7007 }
7008
7009 // Test that when the first pick is down leading to a transient failure, we
7010 // will move on to the next ring hash entry.
TEST_P(CdsTest,RingHashTransientFailureCheckNextOne)7011 TEST_P(CdsTest, RingHashTransientFailureCheckNextOne) {
7012 auto cluster = default_cluster_;
7013 cluster.set_lb_policy(Cluster::RING_HASH);
7014 balancers_[0]->ads_service()->SetCdsResource(cluster);
7015 auto new_route_config = default_route_config_;
7016 auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
7017 auto* hash_policy = route->mutable_route()->add_hash_policy();
7018 hash_policy->mutable_header()->set_header_name("address_hash");
7019 SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
7020 std::vector<EdsResourceArgs::Endpoint> endpoints;
7021 const int unused_port = grpc_pick_unused_port_or_die();
7022 endpoints.emplace_back(unused_port);
7023 endpoints.emplace_back(backends_[1]->port());
7024 EdsResourceArgs args({
7025 {"locality0", std::move(endpoints)},
7026 });
7027 balancers_[0]->ads_service()->SetEdsResource(
7028 BuildEdsResource(args, DefaultEdsServiceName()));
7029 SetNextResolutionForLbChannelAllBalancers();
7030 std::vector<std::pair<std::string, std::string>> metadata = {
7031 {"address_hash",
7032 CreateMetadataValueThatHashesToBackendPort(unused_port)}};
7033 const auto rpc_options = RpcOptions().set_metadata(std::move(metadata));
7034 WaitForBackend(1, WaitForBackendOptions(), rpc_options);
7035 CheckRpcSendOk(100, rpc_options);
7036 EXPECT_EQ(0, backends_[0]->backend_service()->request_count());
7037 EXPECT_EQ(100, backends_[1]->backend_service()->request_count());
7038 }
7039
7040 // Test that when a backend goes down, we will move on to the next subchannel
7041 // (with a lower priority). When the backend comes back up, traffic will move
7042 // back.
TEST_P(CdsTest,RingHashSwitchToLowerPrioirtyAndThenBack)7043 TEST_P(CdsTest, RingHashSwitchToLowerPrioirtyAndThenBack) {
7044 auto cluster = default_cluster_;
7045 cluster.set_lb_policy(Cluster::RING_HASH);
7046 balancers_[0]->ads_service()->SetCdsResource(cluster);
7047 auto new_route_config = default_route_config_;
7048 auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
7049 auto* hash_policy = route->mutable_route()->add_hash_policy();
7050 hash_policy->mutable_header()->set_header_name("address_hash");
7051 SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
7052 EdsResourceArgs args({
7053 {"locality0", CreateEndpointsForBackends(0, 1), kDefaultLocalityWeight,
7054 0},
7055 {"locality1", CreateEndpointsForBackends(1, 2), kDefaultLocalityWeight,
7056 1},
7057 });
7058 balancers_[0]->ads_service()->SetEdsResource(
7059 BuildEdsResource(args, DefaultEdsServiceName()));
7060 SetNextResolutionForLbChannelAllBalancers();
7061 std::vector<std::pair<std::string, std::string>> metadata = {
7062 {"address_hash", CreateMetadataValueThatHashesToBackend(0)}};
7063 const auto rpc_options = RpcOptions().set_metadata(std::move(metadata));
7064 WaitForBackend(0, WaitForBackendOptions(), rpc_options);
7065 ShutdownBackend(0);
7066 WaitForBackend(1, WaitForBackendOptions().set_allow_failures(true),
7067 rpc_options);
7068 StartBackend(0);
7069 WaitForBackend(0, WaitForBackendOptions(), rpc_options);
7070 CheckRpcSendOk(100, rpc_options);
7071 EXPECT_EQ(100, backends_[0]->backend_service()->request_count());
7072 EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
7073 }
7074
7075 // Test that when all backends are down, we will keep reattempting.
TEST_P(CdsTest,RingHashAllFailReattempt)7076 TEST_P(CdsTest, RingHashAllFailReattempt) {
7077 const uint32_t kConnectionTimeoutMilliseconds = 5000;
7078 auto cluster = default_cluster_;
7079 cluster.set_lb_policy(Cluster::RING_HASH);
7080 balancers_[0]->ads_service()->SetCdsResource(cluster);
7081 auto new_route_config = default_route_config_;
7082 auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
7083 auto* hash_policy = route->mutable_route()->add_hash_policy();
7084 hash_policy->mutable_header()->set_header_name("address_hash");
7085 SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
7086 std::vector<EdsResourceArgs::Endpoint> endpoints;
7087 endpoints.emplace_back(grpc_pick_unused_port_or_die());
7088 endpoints.emplace_back(backends_[1]->port());
7089 EdsResourceArgs args({
7090 {"locality0", std::move(endpoints)},
7091 });
7092 balancers_[0]->ads_service()->SetEdsResource(
7093 BuildEdsResource(args, DefaultEdsServiceName()));
7094 SetNextResolutionForLbChannelAllBalancers();
7095 std::vector<std::pair<std::string, std::string>> metadata = {
7096 {"address_hash", CreateMetadataValueThatHashesToBackend(0)}};
7097 EXPECT_EQ(GRPC_CHANNEL_IDLE, channel_->GetState(false));
7098 ShutdownBackend(1);
7099 CheckRpcSendFailure(CheckRpcSendFailureOptions().set_rpc_options(
7100 RpcOptions().set_metadata(std::move(metadata))));
7101 StartBackend(1);
7102 // Ensure we are actively connecting without any traffic.
7103 EXPECT_TRUE(channel_->WaitForConnected(
7104 grpc_timeout_milliseconds_to_deadline(kConnectionTimeoutMilliseconds)));
7105 }
7106
7107 // Test that when all backends are down and then up, we may pick a TF backend
7108 // and we will then jump to ready backend.
TEST_P(CdsTest,RingHashTransientFailureSkipToAvailableReady)7109 TEST_P(CdsTest, RingHashTransientFailureSkipToAvailableReady) {
7110 const uint32_t kConnectionTimeoutMilliseconds = 5000;
7111 auto cluster = default_cluster_;
7112 cluster.set_lb_policy(Cluster::RING_HASH);
7113 balancers_[0]->ads_service()->SetCdsResource(cluster);
7114 auto new_route_config = default_route_config_;
7115 auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
7116 auto* hash_policy = route->mutable_route()->add_hash_policy();
7117 hash_policy->mutable_header()->set_header_name("address_hash");
7118 SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
7119 std::vector<EdsResourceArgs::Endpoint> endpoints;
7120 // Make sure we include some unused ports to fill the ring.
7121 endpoints.emplace_back(backends_[0]->port());
7122 endpoints.emplace_back(backends_[1]->port());
7123 endpoints.emplace_back(grpc_pick_unused_port_or_die());
7124 endpoints.emplace_back(grpc_pick_unused_port_or_die());
7125 EdsResourceArgs args({
7126 {"locality0", std::move(endpoints)},
7127 });
7128 balancers_[0]->ads_service()->SetEdsResource(
7129 BuildEdsResource(args, DefaultEdsServiceName()));
7130 SetNextResolutionForLbChannelAllBalancers();
7131 std::vector<std::pair<std::string, std::string>> metadata = {
7132 {"address_hash", CreateMetadataValueThatHashesToBackend(0)}};
7133 const auto rpc_options = RpcOptions().set_metadata(std::move(metadata));
7134 EXPECT_EQ(GRPC_CHANNEL_IDLE, channel_->GetState(false));
7135 ShutdownBackend(0);
7136 ShutdownBackend(1);
7137 CheckRpcSendFailure(
7138 CheckRpcSendFailureOptions().set_rpc_options(rpc_options));
7139 EXPECT_EQ(GRPC_CHANNEL_TRANSIENT_FAILURE, channel_->GetState(false));
7140 // Bring up 0, should be picked as the RPC is hashed to it.
7141 StartBackend(0);
7142 EXPECT_TRUE(channel_->WaitForConnected(
7143 grpc_timeout_milliseconds_to_deadline(kConnectionTimeoutMilliseconds)));
7144 WaitForBackend(0, WaitForBackendOptions(), rpc_options);
7145 // Bring down 0 and bring up 1.
7146 // Note the RPC contains a header value that will always be hashed to
7147 // backend 0. So by purposely bring down backend 0 and bring up another
7148 // backend, this will ensure Picker's first choice of backend 0 will fail
7149 // and it will
7150 // 1. reattempt backend 0 and
7151 // 2. go through the remaining subchannels to find one in READY.
7152 // Since the the entries in the ring is pretty distributed and we have
7153 // unused ports to fill the ring, it is almost guaranteed that the Picker
7154 // will go through some non-READY entries and skip them as per design.
7155 ShutdownBackend(0);
7156 CheckRpcSendFailure(
7157 CheckRpcSendFailureOptions().set_rpc_options(rpc_options));
7158 StartBackend(1);
7159 EXPECT_TRUE(channel_->WaitForConnected(
7160 grpc_timeout_milliseconds_to_deadline(kConnectionTimeoutMilliseconds)));
7161 WaitForBackend(1, WaitForBackendOptions(), rpc_options);
7162 }
7163
7164 // Test unspported hash policy types are all ignored before a supported
7165 // policy.
TEST_P(CdsTest,RingHashUnsupportedHashPolicyUntilChannelIdHashing)7166 TEST_P(CdsTest, RingHashUnsupportedHashPolicyUntilChannelIdHashing) {
7167 auto cluster = default_cluster_;
7168 cluster.set_lb_policy(Cluster::RING_HASH);
7169 balancers_[0]->ads_service()->SetCdsResource(cluster);
7170 auto new_route_config = default_route_config_;
7171 auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
7172 auto* hash_policy_unsupported_1 = route->mutable_route()->add_hash_policy();
7173 hash_policy_unsupported_1->mutable_cookie()->set_name("cookie");
7174 auto* hash_policy_unsupported_2 = route->mutable_route()->add_hash_policy();
7175 hash_policy_unsupported_2->mutable_connection_properties()->set_source_ip(
7176 true);
7177 auto* hash_policy_unsupported_3 = route->mutable_route()->add_hash_policy();
7178 hash_policy_unsupported_3->mutable_query_parameter()->set_name(
7179 "query_parameter");
7180 auto* hash_policy = route->mutable_route()->add_hash_policy();
7181 hash_policy->mutable_filter_state()->set_key("io.grpc.channel_id");
7182 SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
7183 EdsResourceArgs args({
7184 {"locality0", CreateEndpointsForBackends()},
7185 });
7186 balancers_[0]->ads_service()->SetEdsResource(
7187 BuildEdsResource(args, DefaultEdsServiceName()));
7188 SetNextResolutionForLbChannelAllBalancers();
7189 CheckRpcSendOk(100);
7190 bool found = false;
7191 for (size_t i = 0; i < backends_.size(); ++i) {
7192 if (backends_[i]->backend_service()->request_count() > 0) {
7193 EXPECT_EQ(backends_[i]->backend_service()->request_count(), 100)
7194 << "backend " << i;
7195 EXPECT_FALSE(found) << "backend " << i;
7196 found = true;
7197 }
7198 }
7199 EXPECT_TRUE(found);
7200 }
7201
7202 // Test we nack when ring hash policy has invalid hash function (something
7203 // other than XX_HASH.
TEST_P(CdsTest,RingHashPolicyHasInvalidHashFunction)7204 TEST_P(CdsTest, RingHashPolicyHasInvalidHashFunction) {
7205 auto cluster = default_cluster_;
7206 cluster.set_lb_policy(Cluster::RING_HASH);
7207 cluster.mutable_ring_hash_lb_config()->set_hash_function(
7208 Cluster::RingHashLbConfig::MURMUR_HASH_2);
7209 balancers_[0]->ads_service()->SetCdsResource(cluster);
7210 auto new_route_config = default_route_config_;
7211 auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
7212 auto* hash_policy = route->mutable_route()->add_hash_policy();
7213 hash_policy->mutable_filter_state()->set_key("io.grpc.channel_id");
7214 SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
7215 EdsResourceArgs args({
7216 {"locality0", CreateEndpointsForBackends()},
7217 });
7218 balancers_[0]->ads_service()->SetEdsResource(
7219 BuildEdsResource(args, DefaultEdsServiceName()));
7220 SetNextResolutionForLbChannelAllBalancers();
7221 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
7222 const auto response_state =
7223 balancers_[0]->ads_service()->cds_response_state();
7224 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
7225 EXPECT_THAT(
7226 response_state.error_message,
7227 ::testing::HasSubstr("ring hash lb config has invalid hash function."));
7228 }
7229
7230 // Test we nack when ring hash policy has invalid ring size.
TEST_P(CdsTest,RingHashPolicyHasInvalidMinimumRingSize)7231 TEST_P(CdsTest, RingHashPolicyHasInvalidMinimumRingSize) {
7232 auto cluster = default_cluster_;
7233 cluster.set_lb_policy(Cluster::RING_HASH);
7234 cluster.mutable_ring_hash_lb_config()->mutable_minimum_ring_size()->set_value(
7235 0);
7236 balancers_[0]->ads_service()->SetCdsResource(cluster);
7237 auto new_route_config = default_route_config_;
7238 auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
7239 auto* hash_policy = route->mutable_route()->add_hash_policy();
7240 hash_policy->mutable_filter_state()->set_key("io.grpc.channel_id");
7241 SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
7242 EdsResourceArgs args({
7243 {"locality0", CreateEndpointsForBackends()},
7244 });
7245 balancers_[0]->ads_service()->SetEdsResource(
7246 BuildEdsResource(args, DefaultEdsServiceName()));
7247 SetNextResolutionForLbChannelAllBalancers();
7248 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
7249 const auto response_state =
7250 balancers_[0]->ads_service()->cds_response_state();
7251 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
7252 EXPECT_THAT(response_state.error_message,
7253 ::testing::HasSubstr(
7254 "min_ring_size is not in the range of 1 to 8388608."));
7255 }
7256
7257 // Test we nack when ring hash policy has invalid ring size.
TEST_P(CdsTest,RingHashPolicyHasInvalidMaxmumRingSize)7258 TEST_P(CdsTest, RingHashPolicyHasInvalidMaxmumRingSize) {
7259 auto cluster = default_cluster_;
7260 cluster.set_lb_policy(Cluster::RING_HASH);
7261 cluster.mutable_ring_hash_lb_config()->mutable_maximum_ring_size()->set_value(
7262 8388609);
7263 balancers_[0]->ads_service()->SetCdsResource(cluster);
7264 auto new_route_config = default_route_config_;
7265 auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
7266 auto* hash_policy = route->mutable_route()->add_hash_policy();
7267 hash_policy->mutable_filter_state()->set_key("io.grpc.channel_id");
7268 SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
7269 EdsResourceArgs args({
7270 {"locality0", CreateEndpointsForBackends()},
7271 });
7272 balancers_[0]->ads_service()->SetEdsResource(
7273 BuildEdsResource(args, DefaultEdsServiceName()));
7274 SetNextResolutionForLbChannelAllBalancers();
7275 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
7276 const auto response_state =
7277 balancers_[0]->ads_service()->cds_response_state();
7278 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
7279 EXPECT_THAT(response_state.error_message,
7280 ::testing::HasSubstr(
7281 "max_ring_size is not in the range of 1 to 8388608."));
7282 }
7283
7284 // Test we nack when ring hash policy has invalid ring size.
TEST_P(CdsTest,RingHashPolicyHasInvalidRingSizeMinGreaterThanMax)7285 TEST_P(CdsTest, RingHashPolicyHasInvalidRingSizeMinGreaterThanMax) {
7286 auto cluster = default_cluster_;
7287 cluster.set_lb_policy(Cluster::RING_HASH);
7288 cluster.mutable_ring_hash_lb_config()->mutable_maximum_ring_size()->set_value(
7289 5000);
7290 cluster.mutable_ring_hash_lb_config()->mutable_minimum_ring_size()->set_value(
7291 5001);
7292 balancers_[0]->ads_service()->SetCdsResource(cluster);
7293 auto new_route_config = default_route_config_;
7294 auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
7295 auto* hash_policy = route->mutable_route()->add_hash_policy();
7296 hash_policy->mutable_filter_state()->set_key("io.grpc.channel_id");
7297 SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
7298 EdsResourceArgs args({
7299 {"locality0", CreateEndpointsForBackends()},
7300 });
7301 balancers_[0]->ads_service()->SetEdsResource(
7302 BuildEdsResource(args, DefaultEdsServiceName()));
7303 SetNextResolutionForLbChannelAllBalancers();
7304 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
7305 const auto response_state =
7306 balancers_[0]->ads_service()->cds_response_state();
7307 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
7308 EXPECT_THAT(response_state.error_message,
7309 ::testing::HasSubstr(
7310 "min_ring_size cannot be greater than max_ring_size."));
7311 }
7312
7313 class XdsSecurityTest : public BasicTest {
7314 protected:
SetUp()7315 void SetUp() override {
7316 BasicTest::SetUp();
7317 root_cert_ = ReadFile(kCaCertPath);
7318 bad_root_cert_ = ReadFile(kBadClientCertPath);
7319 identity_pair_ = ReadTlsIdentityPair(kClientKeyPath, kClientCertPath);
7320 // TODO(yashykt): Use different client certs here instead of reusing
7321 // server certs after https://github.com/grpc/grpc/pull/24876 is merged
7322 fallback_identity_pair_ =
7323 ReadTlsIdentityPair(kServerKeyPath, kServerCertPath);
7324 bad_identity_pair_ =
7325 ReadTlsIdentityPair(kBadClientKeyPath, kBadClientCertPath);
7326 server_san_exact_.set_exact("*.test.google.fr");
7327 server_san_prefix_.set_prefix("waterzooi.test.google");
7328 server_san_suffix_.set_suffix("google.fr");
7329 server_san_contains_.set_contains("google");
7330 server_san_regex_.mutable_safe_regex()->mutable_google_re2();
7331 server_san_regex_.mutable_safe_regex()->set_regex(
7332 "(foo|waterzooi).test.google.(fr|be)");
7333 bad_san_1_.set_exact("192.168.1.4");
7334 bad_san_2_.set_exact("foo.test.google.in");
7335 authenticated_identity_ = {"testclient"};
7336 fallback_authenticated_identity_ = {"*.test.google.fr",
7337 "waterzooi.test.google.be",
7338 "*.test.youtube.com", "192.168.1.3"};
7339 EdsResourceArgs args({
7340 {"locality0", CreateEndpointsForBackends(0, 1)},
7341 });
7342 balancers_[0]->ads_service()->SetEdsResource(
7343 BuildEdsResource(args, DefaultEdsServiceName()));
7344 SetNextResolutionForLbChannelAllBalancers();
7345 }
7346
TearDown()7347 void TearDown() override {
7348 g_fake1_cert_data_map = nullptr;
7349 g_fake2_cert_data_map = nullptr;
7350 BasicTest::TearDown();
7351 }
7352
7353 // Sends CDS updates with the new security configuration and verifies that
7354 // after propagation, this new configuration is used for connections. If \a
7355 // identity_instance_name and \a root_instance_name are both empty,
7356 // connections are expected to use fallback credentials.
UpdateAndVerifyXdsSecurityConfiguration(absl::string_view root_instance_name,absl::string_view root_certificate_name,absl::string_view identity_instance_name,absl::string_view identity_certificate_name,const std::vector<StringMatcher> & san_matchers,const std::vector<std::string> & expected_authenticated_identity,bool test_expects_failure=false)7357 void UpdateAndVerifyXdsSecurityConfiguration(
7358 absl::string_view root_instance_name,
7359 absl::string_view root_certificate_name,
7360 absl::string_view identity_instance_name,
7361 absl::string_view identity_certificate_name,
7362 const std::vector<StringMatcher>& san_matchers,
7363 const std::vector<std::string>& expected_authenticated_identity,
7364 bool test_expects_failure = false) {
7365 auto cluster = default_cluster_;
7366 if (!identity_instance_name.empty() || !root_instance_name.empty()) {
7367 auto* transport_socket = cluster.mutable_transport_socket();
7368 transport_socket->set_name("envoy.transport_sockets.tls");
7369 UpstreamTlsContext upstream_tls_context;
7370 if (!identity_instance_name.empty()) {
7371 upstream_tls_context.mutable_common_tls_context()
7372 ->mutable_tls_certificate_provider_instance()
7373 ->set_instance_name(std::string(identity_instance_name));
7374 upstream_tls_context.mutable_common_tls_context()
7375 ->mutable_tls_certificate_provider_instance()
7376 ->set_certificate_name(std::string(identity_certificate_name));
7377 }
7378 if (!root_instance_name.empty()) {
7379 upstream_tls_context.mutable_common_tls_context()
7380 ->mutable_validation_context()
7381 ->mutable_ca_certificate_provider_instance()
7382 ->set_instance_name(std::string(root_instance_name));
7383 upstream_tls_context.mutable_common_tls_context()
7384 ->mutable_validation_context()
7385 ->mutable_ca_certificate_provider_instance()
7386 ->set_certificate_name(std::string(root_certificate_name));
7387 }
7388 if (!san_matchers.empty()) {
7389 auto* validation_context =
7390 upstream_tls_context.mutable_common_tls_context()
7391 ->mutable_validation_context();
7392 for (const auto& san_matcher : san_matchers) {
7393 *validation_context->add_match_subject_alt_names() = san_matcher;
7394 }
7395 }
7396 transport_socket->mutable_typed_config()->PackFrom(upstream_tls_context);
7397 }
7398 balancers_[0]->ads_service()->SetCdsResource(cluster);
7399 // The updates might take time to have an effect, so use a retry loop.
7400 constexpr int kRetryCount = 100;
7401 int num_tries = 0;
7402 for (; num_tries < kRetryCount; num_tries++) {
7403 // Give some time for the updates to propagate.
7404 gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100));
7405 if (test_expects_failure) {
7406 // Restart the servers to force a reconnection so that previously
7407 // connected subchannels are not used for the RPC.
7408 ShutdownBackend(0);
7409 StartBackend(0);
7410 if (SendRpc().ok()) {
7411 gpr_log(GPR_ERROR, "RPC succeeded. Failure expected. Trying again.");
7412 continue;
7413 }
7414 } else {
7415 WaitForBackend(0, WaitForBackendOptions().set_allow_failures(true));
7416 Status status = SendRpc();
7417 if (!status.ok()) {
7418 gpr_log(GPR_ERROR, "RPC failed. code=%d message=%s Trying again.",
7419 status.error_code(), status.error_message().c_str());
7420 continue;
7421 }
7422 if (backends_[0]->backend_service()->last_peer_identity() !=
7423 expected_authenticated_identity) {
7424 gpr_log(
7425 GPR_ERROR,
7426 "Expected client identity does not match. (actual) %s vs "
7427 "(expected) %s Trying again.",
7428 absl::StrJoin(
7429 backends_[0]->backend_service()->last_peer_identity(), ",")
7430 .c_str(),
7431 absl::StrJoin(expected_authenticated_identity, ",").c_str());
7432 continue;
7433 }
7434 }
7435 break;
7436 }
7437 EXPECT_LT(num_tries, kRetryCount);
7438 }
7439
7440 std::string root_cert_;
7441 std::string bad_root_cert_;
7442 grpc_core::PemKeyCertPairList identity_pair_;
7443 grpc_core::PemKeyCertPairList fallback_identity_pair_;
7444 grpc_core::PemKeyCertPairList bad_identity_pair_;
7445 StringMatcher server_san_exact_;
7446 StringMatcher server_san_prefix_;
7447 StringMatcher server_san_suffix_;
7448 StringMatcher server_san_contains_;
7449 StringMatcher server_san_regex_;
7450 StringMatcher bad_san_1_;
7451 StringMatcher bad_san_2_;
7452 std::vector<std::string> authenticated_identity_;
7453 std::vector<std::string> fallback_authenticated_identity_;
7454 };
7455
TEST_P(XdsSecurityTest,UnknownTransportSocket)7456 TEST_P(XdsSecurityTest, UnknownTransportSocket) {
7457 auto cluster = default_cluster_;
7458 auto* transport_socket = cluster.mutable_transport_socket();
7459 transport_socket->set_name("unknown_transport_socket");
7460 balancers_[0]->ads_service()->SetCdsResource(cluster);
7461 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
7462 const auto response_state =
7463 balancers_[0]->ads_service()->cds_response_state();
7464 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
7465 EXPECT_THAT(response_state.error_message,
7466 ::testing::HasSubstr(
7467 "Unrecognized transport socket: unknown_transport_socket"));
7468 }
7469
TEST_P(XdsSecurityTest,TLSConfigurationWithoutValidationContextCertificateProviderInstance)7470 TEST_P(XdsSecurityTest,
7471 TLSConfigurationWithoutValidationContextCertificateProviderInstance) {
7472 auto cluster = default_cluster_;
7473 auto* transport_socket = cluster.mutable_transport_socket();
7474 transport_socket->set_name("envoy.transport_sockets.tls");
7475 balancers_[0]->ads_service()->SetCdsResource(cluster);
7476 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
7477 const auto response_state =
7478 balancers_[0]->ads_service()->cds_response_state();
7479 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
7480 EXPECT_THAT(response_state.error_message,
7481 ::testing::HasSubstr("TLS configuration provided but no "
7482 "ca_certificate_provider_instance found."));
7483 }
7484
TEST_P(XdsSecurityTest,MatchSubjectAltNamesProvidedWithoutValidationContextCertificateProviderInstance)7485 TEST_P(
7486 XdsSecurityTest,
7487 MatchSubjectAltNamesProvidedWithoutValidationContextCertificateProviderInstance) {
7488 auto cluster = default_cluster_;
7489 auto* transport_socket = cluster.mutable_transport_socket();
7490 transport_socket->set_name("envoy.transport_sockets.tls");
7491 UpstreamTlsContext upstream_tls_context;
7492 auto* validation_context = upstream_tls_context.mutable_common_tls_context()
7493 ->mutable_validation_context();
7494 *validation_context->add_match_subject_alt_names() = server_san_exact_;
7495 transport_socket->mutable_typed_config()->PackFrom(upstream_tls_context);
7496 balancers_[0]->ads_service()->SetCdsResource(cluster);
7497 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
7498 const auto response_state =
7499 balancers_[0]->ads_service()->cds_response_state();
7500 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
7501 EXPECT_THAT(response_state.error_message,
7502 ::testing::HasSubstr("TLS configuration provided but no "
7503 "ca_certificate_provider_instance found."));
7504 }
7505
TEST_P(XdsSecurityTest,TlsCertificateProviderInstanceWithoutValidationContextCertificateProviderInstance)7506 TEST_P(
7507 XdsSecurityTest,
7508 TlsCertificateProviderInstanceWithoutValidationContextCertificateProviderInstance) {
7509 auto cluster = default_cluster_;
7510 auto* transport_socket = cluster.mutable_transport_socket();
7511 transport_socket->set_name("envoy.transport_sockets.tls");
7512 UpstreamTlsContext upstream_tls_context;
7513 upstream_tls_context.mutable_common_tls_context()
7514 ->mutable_tls_certificate_provider_instance()
7515 ->set_instance_name(std::string("fake_plugin1"));
7516 transport_socket->mutable_typed_config()->PackFrom(upstream_tls_context);
7517 balancers_[0]->ads_service()->SetCdsResource(cluster);
7518 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
7519 const auto response_state =
7520 balancers_[0]->ads_service()->cds_response_state();
7521 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
7522 EXPECT_THAT(response_state.error_message,
7523 ::testing::HasSubstr("TLS configuration provided but no "
7524 "ca_certificate_provider_instance found."));
7525 }
7526
TEST_P(XdsSecurityTest,RegexSanMatcherDoesNotAllowIgnoreCase)7527 TEST_P(XdsSecurityTest, RegexSanMatcherDoesNotAllowIgnoreCase) {
7528 auto cluster = default_cluster_;
7529 auto* transport_socket = cluster.mutable_transport_socket();
7530 transport_socket->set_name("envoy.transport_sockets.tls");
7531 UpstreamTlsContext upstream_tls_context;
7532 upstream_tls_context.mutable_common_tls_context()
7533 ->mutable_validation_context()
7534 ->mutable_ca_certificate_provider_instance()
7535 ->set_instance_name(std::string("fake_plugin1"));
7536 auto* validation_context = upstream_tls_context.mutable_common_tls_context()
7537 ->mutable_validation_context();
7538 StringMatcher matcher;
7539 matcher.mutable_safe_regex()->mutable_google_re2();
7540 matcher.mutable_safe_regex()->set_regex(
7541 "(foo|waterzooi).test.google.(fr|be)");
7542 matcher.set_ignore_case(true);
7543 *validation_context->add_match_subject_alt_names() = matcher;
7544 transport_socket->mutable_typed_config()->PackFrom(upstream_tls_context);
7545 balancers_[0]->ads_service()->SetCdsResource(cluster);
7546 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
7547 const auto response_state =
7548 balancers_[0]->ads_service()->cds_response_state();
7549 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
7550 EXPECT_THAT(response_state.error_message,
7551 ::testing::HasSubstr(
7552 "StringMatcher: ignore_case has no effect for SAFE_REGEX."));
7553 }
7554
TEST_P(XdsSecurityTest,UnknownRootCertificateProvider)7555 TEST_P(XdsSecurityTest, UnknownRootCertificateProvider) {
7556 auto cluster = default_cluster_;
7557 auto* transport_socket = cluster.mutable_transport_socket();
7558 transport_socket->set_name("envoy.transport_sockets.tls");
7559 UpstreamTlsContext upstream_tls_context;
7560 upstream_tls_context.mutable_common_tls_context()
7561 ->mutable_validation_context()
7562 ->mutable_ca_certificate_provider_instance()
7563 ->set_instance_name("unknown");
7564 transport_socket->mutable_typed_config()->PackFrom(upstream_tls_context);
7565 balancers_[0]->ads_service()->SetCdsResource(cluster);
7566 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
7567 const auto response_state =
7568 balancers_[0]->ads_service()->cds_response_state();
7569 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
7570 EXPECT_THAT(response_state.error_message,
7571 ::testing::HasSubstr(
7572 "Unrecognized certificate provider instance name: unknown"));
7573 }
7574
TEST_P(XdsSecurityTest,UnknownIdentityCertificateProvider)7575 TEST_P(XdsSecurityTest, UnknownIdentityCertificateProvider) {
7576 FakeCertificateProvider::CertDataMap fake1_cert_map = {
7577 {"", {root_cert_, identity_pair_}}};
7578 g_fake1_cert_data_map = &fake1_cert_map;
7579 auto cluster = default_cluster_;
7580 auto* transport_socket = cluster.mutable_transport_socket();
7581 transport_socket->set_name("envoy.transport_sockets.tls");
7582 UpstreamTlsContext upstream_tls_context;
7583 upstream_tls_context.mutable_common_tls_context()
7584 ->mutable_tls_certificate_provider_instance()
7585 ->set_instance_name("unknown");
7586 upstream_tls_context.mutable_common_tls_context()
7587 ->mutable_validation_context()
7588 ->mutable_ca_certificate_provider_instance()
7589 ->set_instance_name("fake_plugin1");
7590 transport_socket->mutable_typed_config()->PackFrom(upstream_tls_context);
7591 balancers_[0]->ads_service()->SetCdsResource(cluster);
7592 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
7593 const auto response_state =
7594 balancers_[0]->ads_service()->cds_response_state();
7595 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
7596 EXPECT_THAT(response_state.error_message,
7597 ::testing::HasSubstr(
7598 "Unrecognized certificate provider instance name: unknown"));
7599 g_fake1_cert_data_map = nullptr;
7600 }
7601
TEST_P(XdsSecurityTest,NacksCertificateValidationContextWithVerifyCertificateSpki)7602 TEST_P(XdsSecurityTest,
7603 NacksCertificateValidationContextWithVerifyCertificateSpki) {
7604 FakeCertificateProvider::CertDataMap fake1_cert_map = {
7605 {"", {root_cert_, identity_pair_}}};
7606 g_fake1_cert_data_map = &fake1_cert_map;
7607 auto cluster = default_cluster_;
7608 auto* transport_socket = cluster.mutable_transport_socket();
7609 transport_socket->set_name("envoy.transport_sockets.tls");
7610 UpstreamTlsContext upstream_tls_context;
7611 upstream_tls_context.mutable_common_tls_context()
7612 ->mutable_validation_context()
7613 ->mutable_ca_certificate_provider_instance()
7614 ->set_instance_name("fake_plugin1");
7615 upstream_tls_context.mutable_common_tls_context()
7616 ->mutable_validation_context()
7617 ->add_verify_certificate_spki("spki");
7618 transport_socket->mutable_typed_config()->PackFrom(upstream_tls_context);
7619 balancers_[0]->ads_service()->SetCdsResource(cluster);
7620 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
7621 const auto response_state =
7622 balancers_[0]->ads_service()->cds_response_state();
7623 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
7624 EXPECT_THAT(
7625 response_state.error_message,
7626 ::testing::HasSubstr(
7627 "CertificateValidationContext: verify_certificate_spki unsupported"));
7628 }
7629
TEST_P(XdsSecurityTest,NacksCertificateValidationContextWithVerifyCertificateHash)7630 TEST_P(XdsSecurityTest,
7631 NacksCertificateValidationContextWithVerifyCertificateHash) {
7632 FakeCertificateProvider::CertDataMap fake1_cert_map = {
7633 {"", {root_cert_, identity_pair_}}};
7634 g_fake1_cert_data_map = &fake1_cert_map;
7635 auto cluster = default_cluster_;
7636 auto* transport_socket = cluster.mutable_transport_socket();
7637 transport_socket->set_name("envoy.transport_sockets.tls");
7638 UpstreamTlsContext upstream_tls_context;
7639 upstream_tls_context.mutable_common_tls_context()
7640 ->mutable_validation_context()
7641 ->mutable_ca_certificate_provider_instance()
7642 ->set_instance_name("fake_plugin1");
7643 upstream_tls_context.mutable_common_tls_context()
7644 ->mutable_validation_context()
7645 ->add_verify_certificate_hash("hash");
7646 transport_socket->mutable_typed_config()->PackFrom(upstream_tls_context);
7647 balancers_[0]->ads_service()->SetCdsResource(cluster);
7648 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
7649 const auto response_state =
7650 balancers_[0]->ads_service()->cds_response_state();
7651 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
7652 EXPECT_THAT(
7653 response_state.error_message,
7654 ::testing::HasSubstr(
7655 "CertificateValidationContext: verify_certificate_hash unsupported"));
7656 }
7657
TEST_P(XdsSecurityTest,NacksCertificateValidationContextWithRequireSignedCertificateTimes)7658 TEST_P(XdsSecurityTest,
7659 NacksCertificateValidationContextWithRequireSignedCertificateTimes) {
7660 FakeCertificateProvider::CertDataMap fake1_cert_map = {
7661 {"", {root_cert_, identity_pair_}}};
7662 g_fake1_cert_data_map = &fake1_cert_map;
7663 auto cluster = default_cluster_;
7664 auto* transport_socket = cluster.mutable_transport_socket();
7665 transport_socket->set_name("envoy.transport_sockets.tls");
7666 UpstreamTlsContext upstream_tls_context;
7667 upstream_tls_context.mutable_common_tls_context()
7668 ->mutable_validation_context()
7669 ->mutable_ca_certificate_provider_instance()
7670 ->set_instance_name("fake_plugin1");
7671 upstream_tls_context.mutable_common_tls_context()
7672 ->mutable_validation_context()
7673 ->mutable_require_signed_certificate_timestamp()
7674 ->set_value(true);
7675 transport_socket->mutable_typed_config()->PackFrom(upstream_tls_context);
7676 balancers_[0]->ads_service()->SetCdsResource(cluster);
7677 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
7678 const auto response_state =
7679 balancers_[0]->ads_service()->cds_response_state();
7680 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
7681 EXPECT_THAT(
7682 response_state.error_message,
7683 ::testing::HasSubstr("CertificateValidationContext: "
7684 "require_signed_certificate_timestamp unsupported"));
7685 }
7686
TEST_P(XdsSecurityTest,NacksCertificateValidationContextWithCrl)7687 TEST_P(XdsSecurityTest, NacksCertificateValidationContextWithCrl) {
7688 FakeCertificateProvider::CertDataMap fake1_cert_map = {
7689 {"", {root_cert_, identity_pair_}}};
7690 g_fake1_cert_data_map = &fake1_cert_map;
7691 auto cluster = default_cluster_;
7692 auto* transport_socket = cluster.mutable_transport_socket();
7693 transport_socket->set_name("envoy.transport_sockets.tls");
7694 UpstreamTlsContext upstream_tls_context;
7695 upstream_tls_context.mutable_common_tls_context()
7696 ->mutable_validation_context()
7697 ->mutable_ca_certificate_provider_instance()
7698 ->set_instance_name("fake_plugin1");
7699 upstream_tls_context.mutable_common_tls_context()
7700 ->mutable_validation_context()
7701 ->mutable_crl();
7702 transport_socket->mutable_typed_config()->PackFrom(upstream_tls_context);
7703 balancers_[0]->ads_service()->SetCdsResource(cluster);
7704 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
7705 const auto response_state =
7706 balancers_[0]->ads_service()->cds_response_state();
7707 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
7708 EXPECT_THAT(
7709 response_state.error_message,
7710 ::testing::HasSubstr("CertificateValidationContext: crl unsupported"));
7711 }
7712
TEST_P(XdsSecurityTest,NacksCertificateValidationContextWithCustomValidatorConfig)7713 TEST_P(XdsSecurityTest,
7714 NacksCertificateValidationContextWithCustomValidatorConfig) {
7715 FakeCertificateProvider::CertDataMap fake1_cert_map = {
7716 {"", {root_cert_, identity_pair_}}};
7717 g_fake1_cert_data_map = &fake1_cert_map;
7718 auto cluster = default_cluster_;
7719 auto* transport_socket = cluster.mutable_transport_socket();
7720 transport_socket->set_name("envoy.transport_sockets.tls");
7721 UpstreamTlsContext upstream_tls_context;
7722 upstream_tls_context.mutable_common_tls_context()
7723 ->mutable_validation_context()
7724 ->mutable_ca_certificate_provider_instance()
7725 ->set_instance_name("fake_plugin1");
7726 upstream_tls_context.mutable_common_tls_context()
7727 ->mutable_validation_context()
7728 ->mutable_custom_validator_config();
7729 transport_socket->mutable_typed_config()->PackFrom(upstream_tls_context);
7730 balancers_[0]->ads_service()->SetCdsResource(cluster);
7731 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
7732 const auto response_state =
7733 balancers_[0]->ads_service()->cds_response_state();
7734 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
7735 EXPECT_THAT(
7736 response_state.error_message,
7737 ::testing::HasSubstr(
7738 "CertificateValidationContext: custom_validator_config unsupported"));
7739 }
7740
TEST_P(XdsSecurityTest,NacksValidationContextSdsSecretConfig)7741 TEST_P(XdsSecurityTest, NacksValidationContextSdsSecretConfig) {
7742 FakeCertificateProvider::CertDataMap fake1_cert_map = {
7743 {"", {root_cert_, identity_pair_}}};
7744 g_fake1_cert_data_map = &fake1_cert_map;
7745 auto cluster = default_cluster_;
7746 auto* transport_socket = cluster.mutable_transport_socket();
7747 transport_socket->set_name("envoy.transport_sockets.tls");
7748 UpstreamTlsContext upstream_tls_context;
7749 upstream_tls_context.mutable_common_tls_context()
7750 ->mutable_validation_context_sds_secret_config();
7751 transport_socket->mutable_typed_config()->PackFrom(upstream_tls_context);
7752 balancers_[0]->ads_service()->SetCdsResource(cluster);
7753 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
7754 const auto response_state =
7755 balancers_[0]->ads_service()->cds_response_state();
7756 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
7757 EXPECT_THAT(
7758 response_state.error_message,
7759 ::testing::HasSubstr("validation_context_sds_secret_config unsupported"));
7760 }
7761
TEST_P(XdsSecurityTest,NacksTlsParams)7762 TEST_P(XdsSecurityTest, NacksTlsParams) {
7763 FakeCertificateProvider::CertDataMap fake1_cert_map = {
7764 {"", {root_cert_, identity_pair_}}};
7765 g_fake1_cert_data_map = &fake1_cert_map;
7766 auto cluster = default_cluster_;
7767 auto* transport_socket = cluster.mutable_transport_socket();
7768 transport_socket->set_name("envoy.transport_sockets.tls");
7769 UpstreamTlsContext upstream_tls_context;
7770 upstream_tls_context.mutable_common_tls_context()
7771 ->mutable_validation_context()
7772 ->mutable_ca_certificate_provider_instance()
7773 ->set_instance_name("fake_plugin1");
7774 upstream_tls_context.mutable_common_tls_context()->mutable_tls_params();
7775 transport_socket->mutable_typed_config()->PackFrom(upstream_tls_context);
7776 balancers_[0]->ads_service()->SetCdsResource(cluster);
7777 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
7778 const auto response_state =
7779 balancers_[0]->ads_service()->cds_response_state();
7780 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
7781 EXPECT_THAT(response_state.error_message,
7782 ::testing::HasSubstr("tls_params unsupported"));
7783 }
7784
TEST_P(XdsSecurityTest,NacksCustomHandshaker)7785 TEST_P(XdsSecurityTest, NacksCustomHandshaker) {
7786 FakeCertificateProvider::CertDataMap fake1_cert_map = {
7787 {"", {root_cert_, identity_pair_}}};
7788 g_fake1_cert_data_map = &fake1_cert_map;
7789 auto cluster = default_cluster_;
7790 auto* transport_socket = cluster.mutable_transport_socket();
7791 transport_socket->set_name("envoy.transport_sockets.tls");
7792 UpstreamTlsContext upstream_tls_context;
7793 upstream_tls_context.mutable_common_tls_context()
7794 ->mutable_validation_context()
7795 ->mutable_ca_certificate_provider_instance()
7796 ->set_instance_name("fake_plugin1");
7797 upstream_tls_context.mutable_common_tls_context()
7798 ->mutable_custom_handshaker();
7799 transport_socket->mutable_typed_config()->PackFrom(upstream_tls_context);
7800 balancers_[0]->ads_service()->SetCdsResource(cluster);
7801 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
7802 const auto response_state =
7803 balancers_[0]->ads_service()->cds_response_state();
7804 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
7805 EXPECT_THAT(response_state.error_message,
7806 ::testing::HasSubstr("custom_handshaker unsupported"));
7807 }
7808
TEST_P(XdsSecurityTest,NacksTlsCertificates)7809 TEST_P(XdsSecurityTest, NacksTlsCertificates) {
7810 FakeCertificateProvider::CertDataMap fake1_cert_map = {
7811 {"", {root_cert_, identity_pair_}}};
7812 g_fake1_cert_data_map = &fake1_cert_map;
7813 auto cluster = default_cluster_;
7814 auto* transport_socket = cluster.mutable_transport_socket();
7815 transport_socket->set_name("envoy.transport_sockets.tls");
7816 UpstreamTlsContext upstream_tls_context;
7817 upstream_tls_context.mutable_common_tls_context()
7818 ->mutable_validation_context()
7819 ->mutable_ca_certificate_provider_instance()
7820 ->set_instance_name("fake_plugin1");
7821 upstream_tls_context.mutable_common_tls_context()->add_tls_certificates();
7822 transport_socket->mutable_typed_config()->PackFrom(upstream_tls_context);
7823 balancers_[0]->ads_service()->SetCdsResource(cluster);
7824 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
7825 const auto response_state =
7826 balancers_[0]->ads_service()->cds_response_state();
7827 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
7828 EXPECT_THAT(response_state.error_message,
7829 ::testing::HasSubstr("tls_certificates unsupported"));
7830 }
7831
TEST_P(XdsSecurityTest,NacksTlsCertificateSdsSecretConfigs)7832 TEST_P(XdsSecurityTest, NacksTlsCertificateSdsSecretConfigs) {
7833 FakeCertificateProvider::CertDataMap fake1_cert_map = {
7834 {"", {root_cert_, identity_pair_}}};
7835 g_fake1_cert_data_map = &fake1_cert_map;
7836 auto cluster = default_cluster_;
7837 auto* transport_socket = cluster.mutable_transport_socket();
7838 transport_socket->set_name("envoy.transport_sockets.tls");
7839 UpstreamTlsContext upstream_tls_context;
7840 upstream_tls_context.mutable_common_tls_context()
7841 ->mutable_validation_context()
7842 ->mutable_ca_certificate_provider_instance()
7843 ->set_instance_name("fake_plugin1");
7844 upstream_tls_context.mutable_common_tls_context()
7845 ->add_tls_certificate_sds_secret_configs();
7846 transport_socket->mutable_typed_config()->PackFrom(upstream_tls_context);
7847 balancers_[0]->ads_service()->SetCdsResource(cluster);
7848 ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
7849 const auto response_state =
7850 balancers_[0]->ads_service()->cds_response_state();
7851 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
7852 EXPECT_THAT(
7853 response_state.error_message,
7854 ::testing::HasSubstr("tls_certificate_sds_secret_configs unsupported"));
7855 }
7856
TEST_P(XdsSecurityTest,TestTlsConfigurationInCombinedValidationContext)7857 TEST_P(XdsSecurityTest, TestTlsConfigurationInCombinedValidationContext) {
7858 FakeCertificateProvider::CertDataMap fake1_cert_map = {
7859 {"", {root_cert_, identity_pair_}}};
7860 g_fake1_cert_data_map = &fake1_cert_map;
7861 auto cluster = default_cluster_;
7862 auto* transport_socket = cluster.mutable_transport_socket();
7863 transport_socket->set_name("envoy.transport_sockets.tls");
7864 UpstreamTlsContext upstream_tls_context;
7865 upstream_tls_context.mutable_common_tls_context()
7866 ->mutable_combined_validation_context()
7867 ->mutable_default_validation_context()
7868 ->mutable_ca_certificate_provider_instance()
7869 ->set_instance_name("fake_plugin1");
7870 transport_socket->mutable_typed_config()->PackFrom(upstream_tls_context);
7871 balancers_[0]->ads_service()->SetCdsResource(cluster);
7872 WaitForBackend(0, WaitForBackendOptions().set_allow_failures(true));
7873 Status status = SendRpc();
7874 EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
7875 << " message=" << status.error_message();
7876 }
7877
7878 // TODO(yashykt): Remove this test once we stop supporting old fields
TEST_P(XdsSecurityTest,TestTlsConfigurationInValidationContextCertificateProviderInstance)7879 TEST_P(XdsSecurityTest,
7880 TestTlsConfigurationInValidationContextCertificateProviderInstance) {
7881 FakeCertificateProvider::CertDataMap fake1_cert_map = {
7882 {"", {root_cert_, identity_pair_}}};
7883 g_fake1_cert_data_map = &fake1_cert_map;
7884 auto cluster = default_cluster_;
7885 auto* transport_socket = cluster.mutable_transport_socket();
7886 transport_socket->set_name("envoy.transport_sockets.tls");
7887 UpstreamTlsContext upstream_tls_context;
7888 upstream_tls_context.mutable_common_tls_context()
7889 ->mutable_combined_validation_context()
7890 ->mutable_validation_context_certificate_provider_instance()
7891 ->set_instance_name("fake_plugin1");
7892 transport_socket->mutable_typed_config()->PackFrom(upstream_tls_context);
7893 balancers_[0]->ads_service()->SetCdsResource(cluster);
7894 WaitForBackend(0, WaitForBackendOptions().set_allow_failures(true));
7895 Status status = SendRpc();
7896 EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
7897 << " message=" << status.error_message();
7898 }
7899
TEST_P(XdsSecurityTest,TestMtlsConfigurationWithNoSanMatchers)7900 TEST_P(XdsSecurityTest, TestMtlsConfigurationWithNoSanMatchers) {
7901 FakeCertificateProvider::CertDataMap fake1_cert_map = {
7902 {"", {root_cert_, identity_pair_}}};
7903 g_fake1_cert_data_map = &fake1_cert_map;
7904 UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
7905 "", {}, authenticated_identity_);
7906 g_fake1_cert_data_map = nullptr;
7907 }
7908
TEST_P(XdsSecurityTest,TestMtlsConfigurationWithExactSanMatcher)7909 TEST_P(XdsSecurityTest, TestMtlsConfigurationWithExactSanMatcher) {
7910 FakeCertificateProvider::CertDataMap fake1_cert_map = {
7911 {"", {root_cert_, identity_pair_}}};
7912 g_fake1_cert_data_map = &fake1_cert_map;
7913 UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
7914 "", {server_san_exact_},
7915 authenticated_identity_);
7916 g_fake1_cert_data_map = nullptr;
7917 }
7918
TEST_P(XdsSecurityTest,TestMtlsConfigurationWithPrefixSanMatcher)7919 TEST_P(XdsSecurityTest, TestMtlsConfigurationWithPrefixSanMatcher) {
7920 FakeCertificateProvider::CertDataMap fake1_cert_map = {
7921 {"", {root_cert_, identity_pair_}}};
7922 g_fake1_cert_data_map = &fake1_cert_map;
7923 UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
7924 "", {server_san_prefix_},
7925 authenticated_identity_);
7926 g_fake1_cert_data_map = nullptr;
7927 }
7928
TEST_P(XdsSecurityTest,TestMtlsConfigurationWithSuffixSanMatcher)7929 TEST_P(XdsSecurityTest, TestMtlsConfigurationWithSuffixSanMatcher) {
7930 FakeCertificateProvider::CertDataMap fake1_cert_map = {
7931 {"", {root_cert_, identity_pair_}}};
7932 g_fake1_cert_data_map = &fake1_cert_map;
7933 UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
7934 "", {server_san_suffix_},
7935 authenticated_identity_);
7936 g_fake1_cert_data_map = nullptr;
7937 }
7938
TEST_P(XdsSecurityTest,TestMtlsConfigurationWithContainsSanMatcher)7939 TEST_P(XdsSecurityTest, TestMtlsConfigurationWithContainsSanMatcher) {
7940 FakeCertificateProvider::CertDataMap fake1_cert_map = {
7941 {"", {root_cert_, identity_pair_}}};
7942 g_fake1_cert_data_map = &fake1_cert_map;
7943 UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
7944 "", {server_san_contains_},
7945 authenticated_identity_);
7946 g_fake1_cert_data_map = nullptr;
7947 }
7948
TEST_P(XdsSecurityTest,TestMtlsConfigurationWithRegexSanMatcher)7949 TEST_P(XdsSecurityTest, TestMtlsConfigurationWithRegexSanMatcher) {
7950 FakeCertificateProvider::CertDataMap fake1_cert_map = {
7951 {"", {root_cert_, identity_pair_}}};
7952 g_fake1_cert_data_map = &fake1_cert_map;
7953 UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
7954 "", {server_san_regex_},
7955 authenticated_identity_);
7956 g_fake1_cert_data_map = nullptr;
7957 }
7958
TEST_P(XdsSecurityTest,TestMtlsConfigurationWithSanMatchersUpdate)7959 TEST_P(XdsSecurityTest, TestMtlsConfigurationWithSanMatchersUpdate) {
7960 FakeCertificateProvider::CertDataMap fake1_cert_map = {
7961 {"", {root_cert_, identity_pair_}}};
7962 g_fake1_cert_data_map = &fake1_cert_map;
7963 UpdateAndVerifyXdsSecurityConfiguration(
7964 "fake_plugin1", "", "fake_plugin1", "",
7965 {server_san_exact_, server_san_prefix_}, authenticated_identity_);
7966 UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
7967 "", {bad_san_1_, bad_san_2_}, {},
7968 true /* failure */);
7969 UpdateAndVerifyXdsSecurityConfiguration(
7970 "fake_plugin1", "", "fake_plugin1", "",
7971 {server_san_prefix_, server_san_regex_}, authenticated_identity_);
7972 g_fake1_cert_data_map = nullptr;
7973 }
7974
TEST_P(XdsSecurityTest,TestMtlsConfigurationWithRootPluginUpdate)7975 TEST_P(XdsSecurityTest, TestMtlsConfigurationWithRootPluginUpdate) {
7976 FakeCertificateProvider::CertDataMap fake1_cert_map = {
7977 {"", {root_cert_, identity_pair_}}};
7978 g_fake1_cert_data_map = &fake1_cert_map;
7979 FakeCertificateProvider::CertDataMap fake2_cert_map = {
7980 {"", {bad_root_cert_, bad_identity_pair_}}};
7981 g_fake2_cert_data_map = &fake2_cert_map;
7982 UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
7983 "", {server_san_exact_},
7984 authenticated_identity_);
7985 UpdateAndVerifyXdsSecurityConfiguration("fake_plugin2" /* bad root */, "",
7986 "fake_plugin1", "", {}, {},
7987 true /* failure */);
7988 UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
7989 "", {server_san_exact_},
7990 authenticated_identity_);
7991 g_fake1_cert_data_map = nullptr;
7992 g_fake2_cert_data_map = nullptr;
7993 }
7994
TEST_P(XdsSecurityTest,TestMtlsConfigurationWithIdentityPluginUpdate)7995 TEST_P(XdsSecurityTest, TestMtlsConfigurationWithIdentityPluginUpdate) {
7996 FakeCertificateProvider::CertDataMap fake1_cert_map = {
7997 {"", {root_cert_, identity_pair_}}};
7998 g_fake1_cert_data_map = &fake1_cert_map;
7999 FakeCertificateProvider::CertDataMap fake2_cert_map = {
8000 {"", {root_cert_, fallback_identity_pair_}}};
8001 g_fake2_cert_data_map = &fake2_cert_map;
8002 UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
8003 "", {server_san_exact_},
8004 authenticated_identity_);
8005 UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin2",
8006 "", {server_san_exact_},
8007 fallback_authenticated_identity_);
8008 g_fake1_cert_data_map = nullptr;
8009 g_fake2_cert_data_map = nullptr;
8010 }
8011
TEST_P(XdsSecurityTest,TestMtlsConfigurationWithBothPluginsUpdated)8012 TEST_P(XdsSecurityTest, TestMtlsConfigurationWithBothPluginsUpdated) {
8013 FakeCertificateProvider::CertDataMap fake1_cert_map = {
8014 {"", {root_cert_, identity_pair_}}};
8015 g_fake1_cert_data_map = &fake1_cert_map;
8016 FakeCertificateProvider::CertDataMap fake2_cert_map = {
8017 {"", {bad_root_cert_, bad_identity_pair_}},
8018 {"good", {root_cert_, fallback_identity_pair_}}};
8019 g_fake2_cert_data_map = &fake2_cert_map;
8020 UpdateAndVerifyXdsSecurityConfiguration("fake_plugin2", "", "fake_plugin2",
8021 "", {}, {}, true /* failure */);
8022 UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
8023 "", {server_san_prefix_},
8024 authenticated_identity_);
8025 UpdateAndVerifyXdsSecurityConfiguration(
8026 "fake_plugin2", "good", "fake_plugin2", "good", {server_san_prefix_},
8027 fallback_authenticated_identity_);
8028 g_fake1_cert_data_map = nullptr;
8029 g_fake2_cert_data_map = nullptr;
8030 }
8031
TEST_P(XdsSecurityTest,TestMtlsConfigurationWithRootCertificateNameUpdate)8032 TEST_P(XdsSecurityTest, TestMtlsConfigurationWithRootCertificateNameUpdate) {
8033 FakeCertificateProvider::CertDataMap fake1_cert_map = {
8034 {"", {root_cert_, identity_pair_}},
8035 {"bad", {bad_root_cert_, bad_identity_pair_}}};
8036 g_fake1_cert_data_map = &fake1_cert_map;
8037 UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
8038 "", {server_san_regex_},
8039 authenticated_identity_);
8040 UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "bad", "fake_plugin1",
8041 "", {server_san_regex_}, {},
8042 true /* failure */);
8043 g_fake1_cert_data_map = nullptr;
8044 }
8045
TEST_P(XdsSecurityTest,TestMtlsConfigurationWithIdentityCertificateNameUpdate)8046 TEST_P(XdsSecurityTest,
8047 TestMtlsConfigurationWithIdentityCertificateNameUpdate) {
8048 FakeCertificateProvider::CertDataMap fake1_cert_map = {
8049 {"", {root_cert_, identity_pair_}},
8050 {"bad", {bad_root_cert_, bad_identity_pair_}}};
8051 g_fake1_cert_data_map = &fake1_cert_map;
8052 UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
8053 "", {server_san_exact_},
8054 authenticated_identity_);
8055 UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
8056 "bad", {server_san_exact_}, {},
8057 true /* failure */);
8058 g_fake1_cert_data_map = nullptr;
8059 }
8060
TEST_P(XdsSecurityTest,TestMtlsConfigurationWithIdentityCertificateNameUpdateGoodCerts)8061 TEST_P(XdsSecurityTest,
8062 TestMtlsConfigurationWithIdentityCertificateNameUpdateGoodCerts) {
8063 FakeCertificateProvider::CertDataMap fake1_cert_map = {
8064 {"", {root_cert_, identity_pair_}},
8065 {"good", {root_cert_, fallback_identity_pair_}}};
8066 g_fake1_cert_data_map = &fake1_cert_map;
8067 UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
8068 "", {server_san_exact_},
8069 authenticated_identity_);
8070 UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
8071 "good", {server_san_exact_},
8072 fallback_authenticated_identity_);
8073 g_fake1_cert_data_map = nullptr;
8074 }
8075
TEST_P(XdsSecurityTest,TestMtlsConfigurationWithBothCertificateNamesUpdated)8076 TEST_P(XdsSecurityTest, TestMtlsConfigurationWithBothCertificateNamesUpdated) {
8077 FakeCertificateProvider::CertDataMap fake1_cert_map = {
8078 {"", {root_cert_, identity_pair_}},
8079 {"bad", {bad_root_cert_, bad_identity_pair_}}};
8080 g_fake1_cert_data_map = &fake1_cert_map;
8081 UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "bad", "fake_plugin1",
8082 "bad", {server_san_prefix_}, {},
8083 true /* failure */);
8084 UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
8085 "", {server_san_prefix_},
8086 authenticated_identity_);
8087 g_fake1_cert_data_map = nullptr;
8088 }
8089
TEST_P(XdsSecurityTest,TestTlsConfigurationWithNoSanMatchers)8090 TEST_P(XdsSecurityTest, TestTlsConfigurationWithNoSanMatchers) {
8091 FakeCertificateProvider::CertDataMap fake1_cert_map = {
8092 {"", {root_cert_, identity_pair_}}};
8093 g_fake1_cert_data_map = &fake1_cert_map;
8094 UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "", "", {},
8095 {} /* unauthenticated */);
8096 g_fake1_cert_data_map = nullptr;
8097 }
8098
TEST_P(XdsSecurityTest,TestTlsConfigurationWithSanMatchers)8099 TEST_P(XdsSecurityTest, TestTlsConfigurationWithSanMatchers) {
8100 FakeCertificateProvider::CertDataMap fake1_cert_map = {
8101 {"", {root_cert_, identity_pair_}}};
8102 g_fake1_cert_data_map = &fake1_cert_map;
8103 UpdateAndVerifyXdsSecurityConfiguration(
8104 "fake_plugin1", "", "", "",
8105 {server_san_exact_, server_san_prefix_, server_san_regex_},
8106 {} /* unauthenticated */);
8107 g_fake1_cert_data_map = nullptr;
8108 }
8109
TEST_P(XdsSecurityTest,TestTlsConfigurationWithSanMatchersUpdate)8110 TEST_P(XdsSecurityTest, TestTlsConfigurationWithSanMatchersUpdate) {
8111 FakeCertificateProvider::CertDataMap fake1_cert_map = {
8112 {"", {root_cert_, identity_pair_}}};
8113 g_fake1_cert_data_map = &fake1_cert_map;
8114 UpdateAndVerifyXdsSecurityConfiguration(
8115 "fake_plugin1", "", "", "", {server_san_exact_, server_san_prefix_},
8116 {} /* unauthenticated */);
8117 UpdateAndVerifyXdsSecurityConfiguration(
8118 "fake_plugin1", "", "", "", {bad_san_1_, bad_san_2_},
8119 {} /* unauthenticated */, true /* failure */);
8120 UpdateAndVerifyXdsSecurityConfiguration(
8121 "fake_plugin1", "", "", "", {server_san_prefix_, server_san_regex_},
8122 {} /* unauthenticated */);
8123 g_fake1_cert_data_map = nullptr;
8124 }
8125
TEST_P(XdsSecurityTest,TestTlsConfigurationWithRootCertificateNameUpdate)8126 TEST_P(XdsSecurityTest, TestTlsConfigurationWithRootCertificateNameUpdate) {
8127 FakeCertificateProvider::CertDataMap fake1_cert_map = {
8128 {"", {root_cert_, identity_pair_}},
8129 {"bad", {bad_root_cert_, bad_identity_pair_}}};
8130 g_fake1_cert_data_map = &fake1_cert_map;
8131 UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "", "",
8132 {server_san_exact_},
8133 {} /* unauthenticated */);
8134 UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "bad", "", "",
8135 {server_san_exact_}, {},
8136 true /* failure */);
8137 g_fake1_cert_data_map = nullptr;
8138 }
8139
TEST_P(XdsSecurityTest,TestTlsConfigurationWithRootPluginUpdate)8140 TEST_P(XdsSecurityTest, TestTlsConfigurationWithRootPluginUpdate) {
8141 FakeCertificateProvider::CertDataMap fake1_cert_map = {
8142 {"", {root_cert_, identity_pair_}}};
8143 g_fake1_cert_data_map = &fake1_cert_map;
8144 FakeCertificateProvider::CertDataMap fake2_cert_map = {
8145 {"", {bad_root_cert_, bad_identity_pair_}}};
8146 g_fake2_cert_data_map = &fake2_cert_map;
8147 UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "", "",
8148 {server_san_exact_},
8149 {} /* unauthenticated */);
8150 UpdateAndVerifyXdsSecurityConfiguration(
8151 "fake_plugin2", "", "", "", {server_san_exact_}, {}, true /* failure */);
8152 g_fake1_cert_data_map = nullptr;
8153 g_fake2_cert_data_map = nullptr;
8154 }
8155
TEST_P(XdsSecurityTest,TestFallbackConfiguration)8156 TEST_P(XdsSecurityTest, TestFallbackConfiguration) {
8157 UpdateAndVerifyXdsSecurityConfiguration("", "", "", "", {},
8158 fallback_authenticated_identity_);
8159 g_fake1_cert_data_map = nullptr;
8160 }
8161
TEST_P(XdsSecurityTest,TestMtlsToTls)8162 TEST_P(XdsSecurityTest, TestMtlsToTls) {
8163 FakeCertificateProvider::CertDataMap fake1_cert_map = {
8164 {"", {root_cert_, identity_pair_}}};
8165 g_fake1_cert_data_map = &fake1_cert_map;
8166 UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
8167 "", {server_san_exact_},
8168 authenticated_identity_);
8169 UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "", "",
8170 {server_san_exact_},
8171 {} /* unauthenticated */);
8172 g_fake1_cert_data_map = nullptr;
8173 }
8174
TEST_P(XdsSecurityTest,TestMtlsToFallback)8175 TEST_P(XdsSecurityTest, TestMtlsToFallback) {
8176 FakeCertificateProvider::CertDataMap fake1_cert_map = {
8177 {"", {root_cert_, identity_pair_}}};
8178 g_fake1_cert_data_map = &fake1_cert_map;
8179 UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
8180 "", {server_san_exact_},
8181 authenticated_identity_);
8182 UpdateAndVerifyXdsSecurityConfiguration("", "", "", "", {},
8183 fallback_authenticated_identity_);
8184 g_fake1_cert_data_map = nullptr;
8185 }
8186
TEST_P(XdsSecurityTest,TestTlsToMtls)8187 TEST_P(XdsSecurityTest, TestTlsToMtls) {
8188 FakeCertificateProvider::CertDataMap fake1_cert_map = {
8189 {"", {root_cert_, identity_pair_}}};
8190 g_fake1_cert_data_map = &fake1_cert_map;
8191 UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "", "",
8192 {server_san_exact_},
8193 {} /* unauthenticated */);
8194 UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
8195 "", {server_san_exact_},
8196 authenticated_identity_);
8197 g_fake1_cert_data_map = nullptr;
8198 }
8199
TEST_P(XdsSecurityTest,TestTlsToFallback)8200 TEST_P(XdsSecurityTest, TestTlsToFallback) {
8201 FakeCertificateProvider::CertDataMap fake1_cert_map = {
8202 {"", {root_cert_, identity_pair_}}};
8203 g_fake1_cert_data_map = &fake1_cert_map;
8204 UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "", "",
8205 {server_san_exact_},
8206 {} /* unauthenticated */);
8207 UpdateAndVerifyXdsSecurityConfiguration("", "", "", "", {},
8208 fallback_authenticated_identity_);
8209 g_fake1_cert_data_map = nullptr;
8210 }
8211
TEST_P(XdsSecurityTest,TestFallbackToMtls)8212 TEST_P(XdsSecurityTest, TestFallbackToMtls) {
8213 FakeCertificateProvider::CertDataMap fake1_cert_map = {
8214 {"", {root_cert_, identity_pair_}}};
8215 g_fake1_cert_data_map = &fake1_cert_map;
8216 UpdateAndVerifyXdsSecurityConfiguration("", "", "", "", {},
8217 fallback_authenticated_identity_);
8218 UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
8219 "", {server_san_exact_},
8220 authenticated_identity_);
8221 g_fake1_cert_data_map = nullptr;
8222 }
8223
TEST_P(XdsSecurityTest,TestFallbackToTls)8224 TEST_P(XdsSecurityTest, TestFallbackToTls) {
8225 FakeCertificateProvider::CertDataMap fake1_cert_map = {
8226 {"", {root_cert_, identity_pair_}}};
8227 g_fake1_cert_data_map = &fake1_cert_map;
8228 UpdateAndVerifyXdsSecurityConfiguration("", "", "", "", {},
8229 fallback_authenticated_identity_);
8230 UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "", "",
8231 {server_san_exact_},
8232 {} /* unauthenticated */);
8233 g_fake1_cert_data_map = nullptr;
8234 }
8235
TEST_P(XdsSecurityTest,TestFileWatcherCertificateProvider)8236 TEST_P(XdsSecurityTest, TestFileWatcherCertificateProvider) {
8237 UpdateAndVerifyXdsSecurityConfiguration("file_plugin", "", "file_plugin", "",
8238 {server_san_exact_},
8239 authenticated_identity_);
8240 }
8241
8242 class XdsEnabledServerTest : public XdsEnd2endTest {
8243 protected:
XdsEnabledServerTest()8244 XdsEnabledServerTest()
8245 : XdsEnd2endTest(1, 1, 100, true /* use_xds_enabled_server */) {}
8246
SetUp()8247 void SetUp() override {
8248 XdsEnd2endTest::SetUp();
8249 EdsResourceArgs args({
8250 {"locality0", CreateEndpointsForBackends(0, 1)},
8251 });
8252 balancers_[0]->ads_service()->SetEdsResource(
8253 BuildEdsResource(args, DefaultEdsServiceName()));
8254 SetNextResolution({});
8255 SetNextResolutionForLbChannelAllBalancers();
8256 }
8257 };
8258
TEST_P(XdsEnabledServerTest,Basic)8259 TEST_P(XdsEnabledServerTest, Basic) { WaitForBackend(0); }
8260
TEST_P(XdsEnabledServerTest,BadLdsUpdateNoApiListenerNorAddress)8261 TEST_P(XdsEnabledServerTest, BadLdsUpdateNoApiListenerNorAddress) {
8262 Listener listener = default_server_listener_;
8263 listener.clear_address();
8264 listener.set_name(
8265 absl::StrCat("grpc/server?xds.resource.listening_address=",
8266 ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()));
8267 balancers_[0]->ads_service()->SetLdsResource(listener);
8268 ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK";
8269 const auto response_state =
8270 balancers_[0]->ads_service()->lds_response_state();
8271 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
8272 EXPECT_THAT(
8273 response_state.error_message,
8274 ::testing::HasSubstr("Listener has neither address nor ApiListener"));
8275 }
8276
TEST_P(XdsEnabledServerTest,BadLdsUpdateBothApiListenerAndAddress)8277 TEST_P(XdsEnabledServerTest, BadLdsUpdateBothApiListenerAndAddress) {
8278 Listener listener = default_server_listener_;
8279 listener.mutable_api_listener();
8280 SetServerListenerNameAndRouteConfiguration(0, listener, backends_[0]->port(),
8281 default_server_route_config_);
8282 ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK";
8283 const auto response_state =
8284 balancers_[0]->ads_service()->lds_response_state();
8285 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
8286 EXPECT_THAT(
8287 response_state.error_message,
8288 ::testing::HasSubstr("Listener has both address and ApiListener"));
8289 }
8290
TEST_P(XdsEnabledServerTest,UnsupportedL4Filter)8291 TEST_P(XdsEnabledServerTest, UnsupportedL4Filter) {
8292 Listener listener = default_server_listener_;
8293 listener.mutable_default_filter_chain()->clear_filters();
8294 listener.mutable_default_filter_chain()->add_filters()->mutable_typed_config()->PackFrom(default_listener_ /* any proto object other than HttpConnectionManager */);
8295 balancers_[0]->ads_service()->SetLdsResource(
8296 PopulateServerListenerNameAndPort(listener, backends_[0]->port()));
8297 ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK";
8298 const auto response_state =
8299 balancers_[0]->ads_service()->lds_response_state();
8300 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
8301 EXPECT_THAT(response_state.error_message,
8302 ::testing::HasSubstr("Unsupported filter type"));
8303 }
8304
TEST_P(XdsEnabledServerTest,NacksEmptyHttpFilterList)8305 TEST_P(XdsEnabledServerTest, NacksEmptyHttpFilterList) {
8306 Listener listener = default_server_listener_;
8307 HttpConnectionManager http_connection_manager =
8308 ServerHcmAccessor().Unpack(listener);
8309 http_connection_manager.clear_http_filters();
8310 ServerHcmAccessor().Pack(http_connection_manager, &listener);
8311 SetServerListenerNameAndRouteConfiguration(0, listener, backends_[0]->port(),
8312 default_server_route_config_);
8313 ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK";
8314 const auto response_state =
8315 balancers_[0]->ads_service()->lds_response_state();
8316 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
8317 EXPECT_THAT(response_state.error_message,
8318 ::testing::HasSubstr("Expected at least one HTTP filter"));
8319 }
8320
TEST_P(XdsEnabledServerTest,UnsupportedHttpFilter)8321 TEST_P(XdsEnabledServerTest, UnsupportedHttpFilter) {
8322 Listener listener = default_server_listener_;
8323 HttpConnectionManager http_connection_manager =
8324 ServerHcmAccessor().Unpack(listener);
8325 http_connection_manager.clear_http_filters();
8326 auto* http_filter = http_connection_manager.add_http_filters();
8327 http_filter->set_name("grpc.testing.unsupported_http_filter");
8328 http_filter->mutable_typed_config()->set_type_url(
8329 "grpc.testing.unsupported_http_filter");
8330 http_filter = http_connection_manager.add_http_filters();
8331 http_filter->set_name("router");
8332 http_filter->mutable_typed_config()->PackFrom(
8333 envoy::extensions::filters::http::router::v3::Router());
8334 ServerHcmAccessor().Pack(http_connection_manager, &listener);
8335 SetServerListenerNameAndRouteConfiguration(0, listener, backends_[0]->port(),
8336 default_server_route_config_);
8337 ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK";
8338 const auto response_state =
8339 balancers_[0]->ads_service()->lds_response_state();
8340 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
8341 EXPECT_THAT(response_state.error_message,
8342 ::testing::HasSubstr("no filter registered for config type "
8343 "grpc.testing.unsupported_http_filter"));
8344 }
8345
TEST_P(XdsEnabledServerTest,HttpFilterNotSupportedOnServer)8346 TEST_P(XdsEnabledServerTest, HttpFilterNotSupportedOnServer) {
8347 Listener listener = default_server_listener_;
8348 HttpConnectionManager http_connection_manager =
8349 ServerHcmAccessor().Unpack(listener);
8350 http_connection_manager.clear_http_filters();
8351 auto* http_filter = http_connection_manager.add_http_filters();
8352 http_filter->set_name("grpc.testing.client_only_http_filter");
8353 http_filter->mutable_typed_config()->set_type_url(
8354 "grpc.testing.client_only_http_filter");
8355 http_filter = http_connection_manager.add_http_filters();
8356 http_filter->set_name("router");
8357 http_filter->mutable_typed_config()->PackFrom(
8358 envoy::extensions::filters::http::router::v3::Router());
8359 ServerHcmAccessor().Pack(http_connection_manager, &listener);
8360 SetServerListenerNameAndRouteConfiguration(0, listener, backends_[0]->port(),
8361 default_server_route_config_);
8362 ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK";
8363 const auto response_state =
8364 balancers_[0]->ads_service()->lds_response_state();
8365 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
8366 EXPECT_THAT(
8367 response_state.error_message,
8368 ::testing::HasSubstr("Filter grpc.testing.client_only_http_filter is not "
8369 "supported on servers"));
8370 }
8371
TEST_P(XdsEnabledServerTest,HttpFilterNotSupportedOnServerIgnoredWhenOptional)8372 TEST_P(XdsEnabledServerTest,
8373 HttpFilterNotSupportedOnServerIgnoredWhenOptional) {
8374 Listener listener = default_server_listener_;
8375 HttpConnectionManager http_connection_manager =
8376 ServerHcmAccessor().Unpack(listener);
8377 http_connection_manager.clear_http_filters();
8378 auto* http_filter = http_connection_manager.add_http_filters();
8379 http_filter->set_name("grpc.testing.client_only_http_filter");
8380 http_filter->mutable_typed_config()->set_type_url(
8381 "grpc.testing.client_only_http_filter");
8382 http_filter->set_is_optional(true);
8383 http_filter = http_connection_manager.add_http_filters();
8384 http_filter->set_name("router");
8385 http_filter->mutable_typed_config()->PackFrom(
8386 envoy::extensions::filters::http::router::v3::Router());
8387 ServerHcmAccessor().Pack(http_connection_manager, &listener);
8388 SetServerListenerNameAndRouteConfiguration(0, listener, backends_[0]->port(),
8389 default_server_route_config_);
8390 WaitForBackend(0);
8391 const auto response_state =
8392 balancers_[0]->ads_service()->lds_response_state();
8393 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::ACKED);
8394 }
8395
8396 // Verify that a mismatch of listening address results in "not serving"
8397 // status.
TEST_P(XdsEnabledServerTest,ListenerAddressMismatch)8398 TEST_P(XdsEnabledServerTest, ListenerAddressMismatch) {
8399 Listener listener = default_server_listener_;
8400 // Set a different listening address in the LDS update
8401 listener.mutable_address()->mutable_socket_address()->set_address(
8402 "192.168.1.1");
8403 SetServerListenerNameAndRouteConfiguration(0, listener, backends_[0]->port(),
8404 default_server_route_config_);
8405 backends_[0]->notifier()->WaitOnServingStatusChange(
8406 absl::StrCat(ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()),
8407 grpc::StatusCode::FAILED_PRECONDITION);
8408 }
8409
TEST_P(XdsEnabledServerTest,UseOriginalDstNotSupported)8410 TEST_P(XdsEnabledServerTest, UseOriginalDstNotSupported) {
8411 Listener listener = default_server_listener_;
8412 listener.mutable_use_original_dst()->set_value(true);
8413 SetServerListenerNameAndRouteConfiguration(0, listener, backends_[0]->port(),
8414 default_server_route_config_);
8415 ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK";
8416 const auto response_state =
8417 balancers_[0]->ads_service()->lds_response_state();
8418 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
8419 EXPECT_THAT(
8420 response_state.error_message,
8421 ::testing::HasSubstr("Field \'use_original_dst\' is not supported."));
8422 }
8423
8424 class XdsServerSecurityTest : public XdsEnd2endTest {
8425 protected:
XdsServerSecurityTest()8426 XdsServerSecurityTest()
8427 : XdsEnd2endTest(1, 1, 100, true /* use_xds_enabled_server */) {}
8428
SetUp()8429 void SetUp() override {
8430 XdsEnd2endTest::SetUp();
8431 root_cert_ = ReadFile(kCaCertPath);
8432 bad_root_cert_ = ReadFile(kBadClientCertPath);
8433 identity_pair_ = ReadTlsIdentityPair(kServerKeyPath, kServerCertPath);
8434 bad_identity_pair_ =
8435 ReadTlsIdentityPair(kBadClientKeyPath, kBadClientCertPath);
8436 identity_pair_2_ = ReadTlsIdentityPair(kClientKeyPath, kClientCertPath);
8437 server_authenticated_identity_ = {"*.test.google.fr",
8438 "waterzooi.test.google.be",
8439 "*.test.youtube.com", "192.168.1.3"};
8440 server_authenticated_identity_2_ = {"testclient"};
8441 client_authenticated_identity_ = {"*.test.google.fr",
8442 "waterzooi.test.google.be",
8443 "*.test.youtube.com", "192.168.1.3"};
8444 EdsResourceArgs args({
8445 {"locality0", CreateEndpointsForBackends(0, 1)},
8446 });
8447 balancers_[0]->ads_service()->SetEdsResource(
8448 BuildEdsResource(args, DefaultEdsServiceName()));
8449 SetNextResolution({});
8450 SetNextResolutionForLbChannelAllBalancers();
8451 }
8452
TearDown()8453 void TearDown() override {
8454 g_fake1_cert_data_map = nullptr;
8455 g_fake2_cert_data_map = nullptr;
8456 XdsEnd2endTest::TearDown();
8457 }
8458
SetLdsUpdate(absl::string_view root_instance_name,absl::string_view root_certificate_name,absl::string_view identity_instance_name,absl::string_view identity_certificate_name,bool require_client_certificates)8459 void SetLdsUpdate(absl::string_view root_instance_name,
8460 absl::string_view root_certificate_name,
8461 absl::string_view identity_instance_name,
8462 absl::string_view identity_certificate_name,
8463 bool require_client_certificates) {
8464 Listener listener = default_server_listener_;
8465 auto* filter_chain = listener.mutable_default_filter_chain();
8466 if (!identity_instance_name.empty()) {
8467 auto* transport_socket = filter_chain->mutable_transport_socket();
8468 transport_socket->set_name("envoy.transport_sockets.tls");
8469 DownstreamTlsContext downstream_tls_context;
8470 downstream_tls_context.mutable_common_tls_context()
8471 ->mutable_tls_certificate_provider_instance()
8472 ->set_instance_name(std::string(identity_instance_name));
8473 downstream_tls_context.mutable_common_tls_context()
8474 ->mutable_tls_certificate_provider_instance()
8475 ->set_certificate_name(std::string(identity_certificate_name));
8476 if (!root_instance_name.empty()) {
8477 downstream_tls_context.mutable_common_tls_context()
8478 ->mutable_validation_context()
8479 ->mutable_ca_certificate_provider_instance()
8480 ->set_instance_name(std::string(root_instance_name));
8481 downstream_tls_context.mutable_common_tls_context()
8482 ->mutable_validation_context()
8483 ->mutable_ca_certificate_provider_instance()
8484 ->set_certificate_name(std::string(root_certificate_name));
8485 downstream_tls_context.mutable_require_client_certificate()->set_value(
8486 require_client_certificates);
8487 }
8488 transport_socket->mutable_typed_config()->PackFrom(
8489 downstream_tls_context);
8490 }
8491 SetServerListenerNameAndRouteConfiguration(
8492 0, listener, backends_[0]->port(), default_server_route_config_);
8493 }
8494
CreateMtlsChannel()8495 std::shared_ptr<grpc::Channel> CreateMtlsChannel() {
8496 ChannelArguments args;
8497 // Override target name for host name check
8498 args.SetString(GRPC_SSL_TARGET_NAME_OVERRIDE_ARG,
8499 ipv6_only_ ? "::1" : "127.0.0.1");
8500 args.SetInt(GRPC_ARG_USE_LOCAL_SUBCHANNEL_POOL, 1);
8501 std::string uri = absl::StrCat(
8502 ipv6_only_ ? "ipv6:[::1]:" : "ipv4:127.0.0.1:", backends_[0]->port());
8503 // TODO(yashykt): Switch to using C++ API once b/173823806 is fixed.
8504 grpc_tls_credentials_options* options =
8505 grpc_tls_credentials_options_create();
8506 grpc_tls_credentials_options_set_server_verification_option(
8507 options, GRPC_TLS_SKIP_HOSTNAME_VERIFICATION);
8508 grpc_tls_credentials_options_set_certificate_provider(
8509 options,
8510 grpc_core::MakeRefCounted<grpc_core::StaticDataCertificateProvider>(
8511 ReadFile(kCaCertPath),
8512 ReadTlsIdentityPair(kServerKeyPath, kServerCertPath))
8513 .get());
8514 grpc_tls_credentials_options_watch_root_certs(options);
8515 grpc_tls_credentials_options_watch_identity_key_cert_pairs(options);
8516 grpc_tls_server_authorization_check_config* check_config =
8517 grpc_tls_server_authorization_check_config_create(
8518 nullptr, ServerAuthCheckSchedule, nullptr, nullptr);
8519 grpc_tls_credentials_options_set_server_authorization_check_config(
8520 options, check_config);
8521 auto channel_creds = std::make_shared<SecureChannelCredentials>(
8522 grpc_tls_credentials_create(options));
8523 grpc_tls_server_authorization_check_config_release(check_config);
8524 return CreateCustomChannel(uri, channel_creds, args);
8525 }
8526
CreateTlsChannel()8527 std::shared_ptr<grpc::Channel> CreateTlsChannel() {
8528 ChannelArguments args;
8529 // Override target name for host name check
8530 args.SetString(GRPC_SSL_TARGET_NAME_OVERRIDE_ARG,
8531 ipv6_only_ ? "::1" : "127.0.0.1");
8532 args.SetInt(GRPC_ARG_USE_LOCAL_SUBCHANNEL_POOL, 1);
8533 std::string uri = absl::StrCat(
8534 ipv6_only_ ? "ipv6:[::1]:" : "ipv4:127.0.0.1:", backends_[0]->port());
8535 // TODO(yashykt): Switch to using C++ API once b/173823806 is fixed.
8536 grpc_tls_credentials_options* options =
8537 grpc_tls_credentials_options_create();
8538 grpc_tls_credentials_options_set_server_verification_option(
8539 options, GRPC_TLS_SKIP_HOSTNAME_VERIFICATION);
8540 grpc_tls_credentials_options_set_certificate_provider(
8541 options,
8542 grpc_core::MakeRefCounted<grpc_core::StaticDataCertificateProvider>(
8543 ReadFile(kCaCertPath),
8544 ReadTlsIdentityPair(kServerKeyPath, kServerCertPath))
8545 .get());
8546 grpc_tls_credentials_options_watch_root_certs(options);
8547 grpc_tls_server_authorization_check_config* check_config =
8548 grpc_tls_server_authorization_check_config_create(
8549 nullptr, ServerAuthCheckSchedule, nullptr, nullptr);
8550 grpc_tls_credentials_options_set_server_authorization_check_config(
8551 options, check_config);
8552 auto channel_creds = std::make_shared<SecureChannelCredentials>(
8553 grpc_tls_credentials_create(options));
8554 grpc_tls_server_authorization_check_config_release(check_config);
8555 return CreateCustomChannel(uri, channel_creds, args);
8556 }
8557
CreateInsecureChannel()8558 std::shared_ptr<grpc::Channel> CreateInsecureChannel() {
8559 ChannelArguments args;
8560 // Override target name for host name check
8561 args.SetString(GRPC_SSL_TARGET_NAME_OVERRIDE_ARG,
8562 ipv6_only_ ? "::1" : "127.0.0.1");
8563 args.SetInt(GRPC_ARG_USE_LOCAL_SUBCHANNEL_POOL, 1);
8564 std::string uri = absl::StrCat(
8565 ipv6_only_ ? "ipv6:[::1]:" : "ipv4:127.0.0.1:", backends_[0]->port());
8566 return CreateCustomChannel(uri, InsecureChannelCredentials(), args);
8567 }
8568
SendRpc(std::function<std::shared_ptr<grpc::Channel> ()> channel_creator,std::vector<std::string> expected_server_identity,std::vector<std::string> expected_client_identity,bool test_expects_failure=false)8569 void SendRpc(std::function<std::shared_ptr<grpc::Channel>()> channel_creator,
8570 std::vector<std::string> expected_server_identity,
8571 std::vector<std::string> expected_client_identity,
8572 bool test_expects_failure = false) {
8573 gpr_log(GPR_INFO, "Sending RPC");
8574 int num_tries = 0;
8575 constexpr int kRetryCount = 100;
8576 for (; num_tries < kRetryCount; num_tries++) {
8577 auto channel = channel_creator();
8578 auto stub = grpc::testing::EchoTestService::NewStub(channel);
8579 ClientContext context;
8580 context.set_wait_for_ready(true);
8581 context.set_deadline(grpc_timeout_milliseconds_to_deadline(2000));
8582 EchoRequest request;
8583 request.set_message(kRequestMessage);
8584 EchoResponse response;
8585 Status status = stub->Echo(&context, request, &response);
8586 if (test_expects_failure) {
8587 if (status.ok()) {
8588 gpr_log(GPR_ERROR, "RPC succeeded. Failure expected. Trying again.");
8589 continue;
8590 }
8591 } else {
8592 if (!status.ok()) {
8593 gpr_log(GPR_ERROR, "RPC failed. code=%d message=%s Trying again.",
8594 status.error_code(), status.error_message().c_str());
8595 continue;
8596 }
8597 EXPECT_EQ(response.message(), kRequestMessage);
8598 std::vector<std::string> peer_identity;
8599 for (const auto& entry : context.auth_context()->GetPeerIdentity()) {
8600 peer_identity.emplace_back(
8601 std::string(entry.data(), entry.size()).c_str());
8602 }
8603 if (peer_identity != expected_server_identity) {
8604 gpr_log(GPR_ERROR,
8605 "Expected server identity does not match. (actual) %s vs "
8606 "(expected) %s Trying again.",
8607 absl::StrJoin(peer_identity, ",").c_str(),
8608 absl::StrJoin(expected_server_identity, ",").c_str());
8609 continue;
8610 }
8611 if (backends_[0]->backend_service()->last_peer_identity() !=
8612 expected_client_identity) {
8613 gpr_log(
8614 GPR_ERROR,
8615 "Expected client identity does not match. (actual) %s vs "
8616 "(expected) %s Trying again.",
8617 absl::StrJoin(
8618 backends_[0]->backend_service()->last_peer_identity(), ",")
8619 .c_str(),
8620 absl::StrJoin(expected_client_identity, ",").c_str());
8621 continue;
8622 }
8623 }
8624 break;
8625 }
8626 EXPECT_LT(num_tries, kRetryCount);
8627 }
8628
8629 std::string root_cert_;
8630 std::string bad_root_cert_;
8631 grpc_core::PemKeyCertPairList identity_pair_;
8632 grpc_core::PemKeyCertPairList bad_identity_pair_;
8633 grpc_core::PemKeyCertPairList identity_pair_2_;
8634 std::vector<std::string> server_authenticated_identity_;
8635 std::vector<std::string> server_authenticated_identity_2_;
8636 std::vector<std::string> client_authenticated_identity_;
8637 };
8638
TEST_P(XdsServerSecurityTest,UnknownTransportSocket)8639 TEST_P(XdsServerSecurityTest, UnknownTransportSocket) {
8640 Listener listener = default_server_listener_;
8641 auto* filter_chain = listener.mutable_default_filter_chain();
8642 auto* transport_socket = filter_chain->mutable_transport_socket();
8643 transport_socket->set_name("unknown_transport_socket");
8644 SetServerListenerNameAndRouteConfiguration(0, listener, backends_[0]->port(),
8645 default_server_route_config_);
8646 ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED))
8647 << "timed out waiting for NACK";
8648 const auto response_state =
8649 balancers_[0]->ads_service()->lds_response_state();
8650 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
8651 EXPECT_THAT(response_state.error_message,
8652 ::testing::HasSubstr(
8653 "Unrecognized transport socket: unknown_transport_socket"));
8654 }
8655
TEST_P(XdsServerSecurityTest,NacksRequireSNI)8656 TEST_P(XdsServerSecurityTest, NacksRequireSNI) {
8657 Listener listener = default_server_listener_;
8658 auto* filter_chain = listener.mutable_default_filter_chain();
8659 auto* transport_socket = filter_chain->mutable_transport_socket();
8660 transport_socket->set_name("envoy.transport_sockets.tls");
8661 DownstreamTlsContext downstream_tls_context;
8662 downstream_tls_context.mutable_common_tls_context()
8663 ->mutable_tls_certificate_provider_instance()
8664 ->set_instance_name("fake_plugin1");
8665 downstream_tls_context.mutable_require_sni()->set_value(true);
8666 transport_socket->mutable_typed_config()->PackFrom(downstream_tls_context);
8667 SetServerListenerNameAndRouteConfiguration(0, listener, backends_[0]->port(),
8668 default_server_route_config_);
8669 ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED))
8670 << "timed out waiting for NACK";
8671 const auto response_state =
8672 balancers_[0]->ads_service()->lds_response_state();
8673 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
8674 EXPECT_THAT(response_state.error_message,
8675 ::testing::HasSubstr("require_sni: unsupported"));
8676 }
8677
TEST_P(XdsServerSecurityTest,NacksOcspStaplePolicyOtherThanLenientStapling)8678 TEST_P(XdsServerSecurityTest, NacksOcspStaplePolicyOtherThanLenientStapling) {
8679 Listener listener = default_server_listener_;
8680 auto* filter_chain = listener.mutable_default_filter_chain();
8681 auto* transport_socket = filter_chain->mutable_transport_socket();
8682 transport_socket->set_name("envoy.transport_sockets.tls");
8683 DownstreamTlsContext downstream_tls_context;
8684 downstream_tls_context.mutable_common_tls_context()
8685 ->mutable_tls_certificate_provider_instance()
8686 ->set_instance_name("fake_plugin1");
8687 downstream_tls_context.set_ocsp_staple_policy(
8688 envoy::extensions::transport_sockets::tls::v3::
8689 DownstreamTlsContext_OcspStaplePolicy_STRICT_STAPLING);
8690 transport_socket->mutable_typed_config()->PackFrom(downstream_tls_context);
8691 SetServerListenerNameAndRouteConfiguration(0, listener, backends_[0]->port(),
8692 default_server_route_config_);
8693 ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED))
8694 << "timed out waiting for NACK";
8695 const auto response_state =
8696 balancers_[0]->ads_service()->lds_response_state();
8697 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
8698 EXPECT_THAT(response_state.error_message,
8699 ::testing::HasSubstr(
8700 "ocsp_staple_policy: Only LENIENT_STAPLING supported"));
8701 }
8702
TEST_P(XdsServerSecurityTest,NacksRequiringClientCertificateWithoutValidationCertificateProviderInstance)8703 TEST_P(
8704 XdsServerSecurityTest,
8705 NacksRequiringClientCertificateWithoutValidationCertificateProviderInstance) {
8706 Listener listener = default_server_listener_;
8707 auto* filter_chain = listener.mutable_default_filter_chain();
8708 auto* transport_socket = filter_chain->mutable_transport_socket();
8709 transport_socket->set_name("envoy.transport_sockets.tls");
8710 DownstreamTlsContext downstream_tls_context;
8711 downstream_tls_context.mutable_common_tls_context()
8712 ->mutable_tls_certificate_provider_instance()
8713 ->set_instance_name("fake_plugin1");
8714 downstream_tls_context.mutable_require_client_certificate()->set_value(true);
8715 transport_socket->mutable_typed_config()->PackFrom(downstream_tls_context);
8716 SetServerListenerNameAndRouteConfiguration(0, listener, backends_[0]->port(),
8717 default_server_route_config_);
8718 ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED))
8719 << "timed out waiting for NACK";
8720 const auto response_state =
8721 balancers_[0]->ads_service()->lds_response_state();
8722 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
8723 EXPECT_THAT(response_state.error_message,
8724 ::testing::HasSubstr(
8725 "TLS configuration requires client certificates but no "
8726 "certificate provider instance specified for validation."));
8727 }
8728
TEST_P(XdsServerSecurityTest,NacksTlsConfigurationWithoutIdentityProviderInstance)8729 TEST_P(XdsServerSecurityTest,
8730 NacksTlsConfigurationWithoutIdentityProviderInstance) {
8731 Listener listener = default_server_listener_;
8732 auto* filter_chain = listener.mutable_default_filter_chain();
8733 auto* transport_socket = filter_chain->mutable_transport_socket();
8734 transport_socket->set_name("envoy.transport_sockets.tls");
8735 DownstreamTlsContext downstream_tls_context;
8736 transport_socket->mutable_typed_config()->PackFrom(downstream_tls_context);
8737 SetServerListenerNameAndRouteConfiguration(0, listener, backends_[0]->port(),
8738 default_server_route_config_);
8739 ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED))
8740 << "timed out waiting for NACK";
8741 const auto response_state =
8742 balancers_[0]->ads_service()->lds_response_state();
8743 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
8744 EXPECT_THAT(response_state.error_message,
8745 ::testing::HasSubstr("TLS configuration provided but no "
8746 "tls_certificate_provider_instance found."));
8747 }
8748
TEST_P(XdsServerSecurityTest,NacksMatchSubjectAltNames)8749 TEST_P(XdsServerSecurityTest, NacksMatchSubjectAltNames) {
8750 Listener listener = default_server_listener_;
8751 auto* filter_chain = listener.mutable_default_filter_chain();
8752 auto* transport_socket = filter_chain->mutable_transport_socket();
8753 transport_socket->set_name("envoy.transport_sockets.tls");
8754 DownstreamTlsContext downstream_tls_context;
8755 downstream_tls_context.mutable_common_tls_context()
8756 ->mutable_tls_certificate_provider_instance()
8757 ->set_instance_name("fake_plugin1");
8758 downstream_tls_context.mutable_common_tls_context()
8759 ->mutable_validation_context()
8760 ->add_match_subject_alt_names()
8761 ->set_exact("*.test.google.fr");
8762 transport_socket->mutable_typed_config()->PackFrom(downstream_tls_context);
8763 SetServerListenerNameAndRouteConfiguration(0, listener, backends_[0]->port(),
8764 default_server_route_config_);
8765 ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED))
8766 << "timed out waiting for NACK";
8767 const auto response_state =
8768 balancers_[0]->ads_service()->lds_response_state();
8769 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
8770 EXPECT_THAT(
8771 response_state.error_message,
8772 ::testing::HasSubstr("match_subject_alt_names not supported on servers"));
8773 }
8774
TEST_P(XdsServerSecurityTest,UnknownIdentityCertificateProvider)8775 TEST_P(XdsServerSecurityTest, UnknownIdentityCertificateProvider) {
8776 SetLdsUpdate("", "", "unknown", "", false);
8777 SendRpc([this]() { return CreateTlsChannel(); }, {}, {},
8778 true /* test_expects_failure */);
8779 ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED))
8780 << "timed out waiting for NACK";
8781 const auto response_state =
8782 balancers_[0]->ads_service()->lds_response_state();
8783 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
8784 EXPECT_THAT(response_state.error_message,
8785 ::testing::HasSubstr(
8786 "Unrecognized certificate provider instance name: unknown"));
8787 }
8788
TEST_P(XdsServerSecurityTest,UnknownRootCertificateProvider)8789 TEST_P(XdsServerSecurityTest, UnknownRootCertificateProvider) {
8790 FakeCertificateProvider::CertDataMap fake1_cert_map = {
8791 {"", {root_cert_, identity_pair_}}};
8792 SetLdsUpdate("unknown", "", "fake_plugin1", "", false);
8793 ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED))
8794 << "timed out waiting for NACK";
8795 const auto response_state =
8796 balancers_[0]->ads_service()->lds_response_state();
8797 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
8798 EXPECT_THAT(response_state.error_message,
8799 ::testing::HasSubstr(
8800 "Unrecognized certificate provider instance name: unknown"));
8801 }
8802
TEST_P(XdsServerSecurityTest,TestDeprecateTlsCertificateCertificateProviderInstanceField)8803 TEST_P(XdsServerSecurityTest,
8804 TestDeprecateTlsCertificateCertificateProviderInstanceField) {
8805 FakeCertificateProvider::CertDataMap fake1_cert_map = {
8806 {"", {root_cert_, identity_pair_}}};
8807 g_fake1_cert_data_map = &fake1_cert_map;
8808 Listener listener = default_server_listener_;
8809 auto* filter_chain = listener.mutable_default_filter_chain();
8810 filter_chain->mutable_filters()->at(0).mutable_typed_config()->PackFrom(
8811 ServerHcmAccessor().Unpack(listener));
8812 auto* transport_socket = filter_chain->mutable_transport_socket();
8813 transport_socket->set_name("envoy.transport_sockets.tls");
8814 DownstreamTlsContext downstream_tls_context;
8815 downstream_tls_context.mutable_common_tls_context()
8816 ->mutable_tls_certificate_certificate_provider_instance()
8817 ->set_instance_name("fake_plugin1");
8818 transport_socket->mutable_typed_config()->PackFrom(downstream_tls_context);
8819 SetServerListenerNameAndRouteConfiguration(0, listener, backends_[0]->port(),
8820 default_server_route_config_);
8821 SendRpc([this]() { return CreateTlsChannel(); },
8822 server_authenticated_identity_, {});
8823 }
8824
TEST_P(XdsServerSecurityTest,CertificatesNotAvailable)8825 TEST_P(XdsServerSecurityTest, CertificatesNotAvailable) {
8826 FakeCertificateProvider::CertDataMap fake1_cert_map;
8827 g_fake1_cert_data_map = &fake1_cert_map;
8828 SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", true);
8829 SendRpc([this]() { return CreateMtlsChannel(); }, {}, {},
8830 true /* test_expects_failure */);
8831 }
8832
TEST_P(XdsServerSecurityTest,TestMtls)8833 TEST_P(XdsServerSecurityTest, TestMtls) {
8834 FakeCertificateProvider::CertDataMap fake1_cert_map = {
8835 {"", {root_cert_, identity_pair_}}};
8836 g_fake1_cert_data_map = &fake1_cert_map;
8837 SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", true);
8838 SendRpc([this]() { return CreateMtlsChannel(); },
8839 server_authenticated_identity_, client_authenticated_identity_);
8840 }
8841
TEST_P(XdsServerSecurityTest,TestMtlsWithRootPluginUpdate)8842 TEST_P(XdsServerSecurityTest, TestMtlsWithRootPluginUpdate) {
8843 FakeCertificateProvider::CertDataMap fake1_cert_map = {
8844 {"", {root_cert_, identity_pair_}}};
8845 g_fake1_cert_data_map = &fake1_cert_map;
8846 FakeCertificateProvider::CertDataMap fake2_cert_map = {
8847 {"", {bad_root_cert_, bad_identity_pair_}}};
8848 g_fake2_cert_data_map = &fake2_cert_map;
8849 SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", true);
8850 SendRpc([this]() { return CreateMtlsChannel(); },
8851 server_authenticated_identity_, client_authenticated_identity_);
8852 SetLdsUpdate("fake_plugin2", "", "fake_plugin1", "", true);
8853 SendRpc([this]() { return CreateMtlsChannel(); }, {}, {},
8854 true /* test_expects_failure */);
8855 }
8856
TEST_P(XdsServerSecurityTest,TestMtlsWithIdentityPluginUpdate)8857 TEST_P(XdsServerSecurityTest, TestMtlsWithIdentityPluginUpdate) {
8858 FakeCertificateProvider::CertDataMap fake1_cert_map = {
8859 {"", {root_cert_, identity_pair_}}};
8860 g_fake1_cert_data_map = &fake1_cert_map;
8861 FakeCertificateProvider::CertDataMap fake2_cert_map = {
8862 {"", {root_cert_, identity_pair_2_}}};
8863 g_fake2_cert_data_map = &fake2_cert_map;
8864 SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", true);
8865 SendRpc([this]() { return CreateMtlsChannel(); },
8866 server_authenticated_identity_, client_authenticated_identity_);
8867 SetLdsUpdate("fake_plugin1", "", "fake_plugin2", "", true);
8868 SendRpc([this]() { return CreateMtlsChannel(); },
8869 server_authenticated_identity_2_, client_authenticated_identity_);
8870 }
8871
TEST_P(XdsServerSecurityTest,TestMtlsWithBothPluginsUpdated)8872 TEST_P(XdsServerSecurityTest, TestMtlsWithBothPluginsUpdated) {
8873 FakeCertificateProvider::CertDataMap fake1_cert_map = {
8874 {"", {root_cert_, identity_pair_}}};
8875 g_fake1_cert_data_map = &fake1_cert_map;
8876 FakeCertificateProvider::CertDataMap fake2_cert_map = {
8877 {"good", {root_cert_, identity_pair_2_}},
8878 {"", {bad_root_cert_, bad_identity_pair_}}};
8879 g_fake2_cert_data_map = &fake2_cert_map;
8880 SetLdsUpdate("fake_plugin2", "", "fake_plugin2", "", true);
8881 SendRpc([this]() { return CreateMtlsChannel(); }, {}, {},
8882 true /* test_expects_failure */);
8883 SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", true);
8884 SendRpc([this]() { return CreateMtlsChannel(); },
8885 server_authenticated_identity_, client_authenticated_identity_);
8886 SetLdsUpdate("fake_plugin2", "good", "fake_plugin2", "good", true);
8887 SendRpc([this]() { return CreateMtlsChannel(); },
8888 server_authenticated_identity_2_, client_authenticated_identity_);
8889 }
8890
TEST_P(XdsServerSecurityTest,TestMtlsWithRootCertificateNameUpdate)8891 TEST_P(XdsServerSecurityTest, TestMtlsWithRootCertificateNameUpdate) {
8892 FakeCertificateProvider::CertDataMap fake1_cert_map = {
8893 {"", {root_cert_, identity_pair_}},
8894 {"bad", {bad_root_cert_, bad_identity_pair_}}};
8895 g_fake1_cert_data_map = &fake1_cert_map;
8896 SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", true);
8897 SendRpc([this]() { return CreateMtlsChannel(); },
8898 server_authenticated_identity_, client_authenticated_identity_);
8899 SetLdsUpdate("fake_plugin1", "bad", "fake_plugin1", "", true);
8900 SendRpc([this]() { return CreateMtlsChannel(); }, {}, {},
8901 true /* test_expects_failure */);
8902 }
8903
TEST_P(XdsServerSecurityTest,TestMtlsWithIdentityCertificateNameUpdate)8904 TEST_P(XdsServerSecurityTest, TestMtlsWithIdentityCertificateNameUpdate) {
8905 FakeCertificateProvider::CertDataMap fake1_cert_map = {
8906 {"", {root_cert_, identity_pair_}},
8907 {"good", {root_cert_, identity_pair_2_}}};
8908 g_fake1_cert_data_map = &fake1_cert_map;
8909 SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", true);
8910 SendRpc([this]() { return CreateMtlsChannel(); },
8911 server_authenticated_identity_, client_authenticated_identity_);
8912 SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "good", true);
8913 SendRpc([this]() { return CreateMtlsChannel(); },
8914 server_authenticated_identity_2_, client_authenticated_identity_);
8915 }
8916
TEST_P(XdsServerSecurityTest,TestMtlsWithBothCertificateNamesUpdated)8917 TEST_P(XdsServerSecurityTest, TestMtlsWithBothCertificateNamesUpdated) {
8918 FakeCertificateProvider::CertDataMap fake1_cert_map = {
8919 {"", {root_cert_, identity_pair_}},
8920 {"good", {root_cert_, identity_pair_2_}}};
8921 g_fake1_cert_data_map = &fake1_cert_map;
8922 SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", true);
8923 SendRpc([this]() { return CreateMtlsChannel(); },
8924 server_authenticated_identity_, client_authenticated_identity_);
8925 SetLdsUpdate("fake_plugin1", "good", "fake_plugin1", "good", true);
8926 SendRpc([this]() { return CreateMtlsChannel(); },
8927 server_authenticated_identity_2_, client_authenticated_identity_);
8928 }
8929
TEST_P(XdsServerSecurityTest,TestMtlsNotRequiringButProvidingClientCerts)8930 TEST_P(XdsServerSecurityTest, TestMtlsNotRequiringButProvidingClientCerts) {
8931 FakeCertificateProvider::CertDataMap fake1_cert_map = {
8932 {"", {root_cert_, identity_pair_}}};
8933 g_fake1_cert_data_map = &fake1_cert_map;
8934 SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", false);
8935 SendRpc([this]() { return CreateMtlsChannel(); },
8936 server_authenticated_identity_, client_authenticated_identity_);
8937 }
8938
TEST_P(XdsServerSecurityTest,TestMtlsNotRequiringAndNotProvidingClientCerts)8939 TEST_P(XdsServerSecurityTest, TestMtlsNotRequiringAndNotProvidingClientCerts) {
8940 FakeCertificateProvider::CertDataMap fake1_cert_map = {
8941 {"", {root_cert_, identity_pair_}}};
8942 g_fake1_cert_data_map = &fake1_cert_map;
8943 SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", false);
8944 SendRpc([this]() { return CreateTlsChannel(); },
8945 server_authenticated_identity_, {});
8946 }
8947
TEST_P(XdsServerSecurityTest,TestTls)8948 TEST_P(XdsServerSecurityTest, TestTls) {
8949 FakeCertificateProvider::CertDataMap fake1_cert_map = {
8950 {"", {root_cert_, identity_pair_}}};
8951 g_fake1_cert_data_map = &fake1_cert_map;
8952 SetLdsUpdate("", "", "fake_plugin1", "", false);
8953 SendRpc([this]() { return CreateTlsChannel(); },
8954 server_authenticated_identity_, {});
8955 }
8956
TEST_P(XdsServerSecurityTest,TestTlsWithIdentityPluginUpdate)8957 TEST_P(XdsServerSecurityTest, TestTlsWithIdentityPluginUpdate) {
8958 FakeCertificateProvider::CertDataMap fake1_cert_map = {
8959 {"", {root_cert_, identity_pair_}}};
8960 g_fake1_cert_data_map = &fake1_cert_map;
8961 FakeCertificateProvider::CertDataMap fake2_cert_map = {
8962 {"", {root_cert_, identity_pair_2_}}};
8963 g_fake2_cert_data_map = &fake2_cert_map;
8964 SetLdsUpdate("", "", "fake_plugin1", "", false);
8965 SendRpc([this]() { return CreateTlsChannel(); },
8966 server_authenticated_identity_, {});
8967 SetLdsUpdate("", "", "fake_plugin2", "", false);
8968 SendRpc([this]() { return CreateTlsChannel(); },
8969 server_authenticated_identity_2_, {});
8970 }
8971
TEST_P(XdsServerSecurityTest,TestTlsWithIdentityCertificateNameUpdate)8972 TEST_P(XdsServerSecurityTest, TestTlsWithIdentityCertificateNameUpdate) {
8973 FakeCertificateProvider::CertDataMap fake1_cert_map = {
8974 {"", {root_cert_, identity_pair_}},
8975 {"good", {root_cert_, identity_pair_2_}}};
8976 g_fake1_cert_data_map = &fake1_cert_map;
8977 SetLdsUpdate("", "", "fake_plugin1", "", false);
8978 SendRpc([this]() { return CreateTlsChannel(); },
8979 server_authenticated_identity_, {});
8980 SetLdsUpdate("", "", "fake_plugin1", "good", false);
8981 SendRpc([this]() { return CreateTlsChannel(); },
8982 server_authenticated_identity_2_, {});
8983 }
8984
TEST_P(XdsServerSecurityTest,TestFallback)8985 TEST_P(XdsServerSecurityTest, TestFallback) {
8986 FakeCertificateProvider::CertDataMap fake1_cert_map = {
8987 {"", {root_cert_, identity_pair_}}};
8988 g_fake1_cert_data_map = &fake1_cert_map;
8989 SetLdsUpdate("", "", "", "", false);
8990 SendRpc([this]() { return CreateInsecureChannel(); }, {}, {});
8991 }
8992
TEST_P(XdsServerSecurityTest,TestMtlsToTls)8993 TEST_P(XdsServerSecurityTest, TestMtlsToTls) {
8994 FakeCertificateProvider::CertDataMap fake1_cert_map = {
8995 {"", {root_cert_, identity_pair_}}};
8996 g_fake1_cert_data_map = &fake1_cert_map;
8997 SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", true);
8998 SendRpc([this]() { return CreateTlsChannel(); }, {}, {},
8999 true /* test_expects_failure */);
9000 SetLdsUpdate("", "", "fake_plugin1", "", false);
9001 SendRpc([this]() { return CreateTlsChannel(); },
9002 server_authenticated_identity_, {});
9003 }
9004
TEST_P(XdsServerSecurityTest,TestTlsToMtls)9005 TEST_P(XdsServerSecurityTest, TestTlsToMtls) {
9006 FakeCertificateProvider::CertDataMap fake1_cert_map = {
9007 {"", {root_cert_, identity_pair_}}};
9008 g_fake1_cert_data_map = &fake1_cert_map;
9009 SetLdsUpdate("", "", "fake_plugin1", "", false);
9010 SendRpc([this]() { return CreateTlsChannel(); },
9011 server_authenticated_identity_, {});
9012 SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", true);
9013 SendRpc([this]() { return CreateTlsChannel(); }, {}, {},
9014 true /* test_expects_failure */);
9015 }
9016
TEST_P(XdsServerSecurityTest,TestMtlsToFallback)9017 TEST_P(XdsServerSecurityTest, TestMtlsToFallback) {
9018 FakeCertificateProvider::CertDataMap fake1_cert_map = {
9019 {"", {root_cert_, identity_pair_}}};
9020 g_fake1_cert_data_map = &fake1_cert_map;
9021 SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", false);
9022 SendRpc([this]() { return CreateMtlsChannel(); },
9023 server_authenticated_identity_, client_authenticated_identity_);
9024 SetLdsUpdate("", "", "", "", false);
9025 SendRpc([this]() { return CreateInsecureChannel(); }, {}, {});
9026 }
9027
TEST_P(XdsServerSecurityTest,TestFallbackToMtls)9028 TEST_P(XdsServerSecurityTest, TestFallbackToMtls) {
9029 FakeCertificateProvider::CertDataMap fake1_cert_map = {
9030 {"", {root_cert_, identity_pair_}}};
9031 g_fake1_cert_data_map = &fake1_cert_map;
9032 SetLdsUpdate("", "", "", "", false);
9033 SendRpc([this]() { return CreateInsecureChannel(); }, {}, {});
9034 SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", true);
9035 SendRpc([this]() { return CreateMtlsChannel(); },
9036 server_authenticated_identity_, client_authenticated_identity_);
9037 }
9038
TEST_P(XdsServerSecurityTest,TestTlsToFallback)9039 TEST_P(XdsServerSecurityTest, TestTlsToFallback) {
9040 FakeCertificateProvider::CertDataMap fake1_cert_map = {
9041 {"", {root_cert_, identity_pair_}}};
9042 g_fake1_cert_data_map = &fake1_cert_map;
9043 SetLdsUpdate("", "", "fake_plugin1", "", false);
9044 SendRpc([this]() { return CreateTlsChannel(); },
9045 server_authenticated_identity_, {});
9046 SetLdsUpdate("", "", "", "", false);
9047 SendRpc([this]() { return CreateInsecureChannel(); }, {}, {});
9048 }
9049
TEST_P(XdsServerSecurityTest,TestFallbackToTls)9050 TEST_P(XdsServerSecurityTest, TestFallbackToTls) {
9051 FakeCertificateProvider::CertDataMap fake1_cert_map = {
9052 {"", {root_cert_, identity_pair_}}};
9053 g_fake1_cert_data_map = &fake1_cert_map;
9054 SetLdsUpdate("", "", "", "", false);
9055 SendRpc([this]() { return CreateInsecureChannel(); }, {}, {});
9056 SetLdsUpdate("", "", "fake_plugin1", "", false);
9057 SendRpc([this]() { return CreateTlsChannel(); },
9058 server_authenticated_identity_, {});
9059 }
9060
9061 class XdsEnabledServerStatusNotificationTest : public XdsServerSecurityTest {
9062 protected:
SetValidLdsUpdate()9063 void SetValidLdsUpdate() { SetLdsUpdate("", "", "", "", false); }
9064
SetInvalidLdsUpdate()9065 void SetInvalidLdsUpdate() {
9066 Listener listener = default_server_listener_;
9067 listener.clear_address();
9068 listener.set_name(absl::StrCat(
9069 "grpc/server?xds.resource.listening_address=",
9070 ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()));
9071 balancers_[0]->ads_service()->SetLdsResource(listener);
9072 }
9073
UnsetLdsUpdate()9074 void UnsetLdsUpdate() {
9075 balancers_[0]->ads_service()->UnsetResource(
9076 kLdsTypeUrl, absl::StrCat("grpc/server?xds.resource.listening_address=",
9077 ipv6_only_ ? "[::1]:" : "127.0.0.1:",
9078 backends_[0]->port()));
9079 }
9080 };
9081
TEST_P(XdsEnabledServerStatusNotificationTest,ServingStatus)9082 TEST_P(XdsEnabledServerStatusNotificationTest, ServingStatus) {
9083 SetValidLdsUpdate();
9084 backends_[0]->notifier()->WaitOnServingStatusChange(
9085 absl::StrCat(ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()),
9086 grpc::StatusCode::OK);
9087 SendRpc([this]() { return CreateInsecureChannel(); }, {}, {});
9088 }
9089
TEST_P(XdsEnabledServerStatusNotificationTest,NotServingStatus)9090 TEST_P(XdsEnabledServerStatusNotificationTest, NotServingStatus) {
9091 SetInvalidLdsUpdate();
9092 backends_[0]->notifier()->WaitOnServingStatusChange(
9093 absl::StrCat(ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()),
9094 grpc::StatusCode::UNAVAILABLE);
9095 SendRpc([this]() { return CreateInsecureChannel(); }, {}, {},
9096 true /* test_expects_failure */);
9097 }
9098
TEST_P(XdsEnabledServerStatusNotificationTest,ErrorUpdateWhenAlreadyServing)9099 TEST_P(XdsEnabledServerStatusNotificationTest, ErrorUpdateWhenAlreadyServing) {
9100 SetValidLdsUpdate();
9101 backends_[0]->notifier()->WaitOnServingStatusChange(
9102 absl::StrCat(ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()),
9103 grpc::StatusCode::OK);
9104 SendRpc([this]() { return CreateInsecureChannel(); }, {}, {});
9105 // Invalid update does not lead to a change in the serving status.
9106 SetInvalidLdsUpdate();
9107 do {
9108 SendRpc([this]() { return CreateInsecureChannel(); }, {}, {});
9109 } while (balancers_[0]->ads_service()->lds_response_state().state ==
9110 AdsServiceImpl::ResponseState::SENT);
9111 backends_[0]->notifier()->WaitOnServingStatusChange(
9112 absl::StrCat(ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()),
9113 grpc::StatusCode::OK);
9114 SendRpc([this]() { return CreateInsecureChannel(); }, {}, {});
9115 }
9116
TEST_P(XdsEnabledServerStatusNotificationTest,NotServingStatusToServingStatusTransition)9117 TEST_P(XdsEnabledServerStatusNotificationTest,
9118 NotServingStatusToServingStatusTransition) {
9119 SetInvalidLdsUpdate();
9120 backends_[0]->notifier()->WaitOnServingStatusChange(
9121 absl::StrCat(ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()),
9122 grpc::StatusCode::UNAVAILABLE);
9123 SendRpc([this]() { return CreateInsecureChannel(); }, {}, {},
9124 true /* test_expects_failure */);
9125 // Send a valid LDS update to change to serving status
9126 SetValidLdsUpdate();
9127 backends_[0]->notifier()->WaitOnServingStatusChange(
9128 absl::StrCat(ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()),
9129 grpc::StatusCode::OK);
9130 SendRpc([this]() { return CreateInsecureChannel(); }, {}, {});
9131 }
9132
9133 // This test verifies that the resource getting deleted when already serving
9134 // results in future connections being dropped.
TEST_P(XdsEnabledServerStatusNotificationTest,ServingStatusToNonServingStatusTransition)9135 TEST_P(XdsEnabledServerStatusNotificationTest,
9136 ServingStatusToNonServingStatusTransition) {
9137 SetValidLdsUpdate();
9138 backends_[0]->notifier()->WaitOnServingStatusChange(
9139 absl::StrCat(ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()),
9140 grpc::StatusCode::OK);
9141 SendRpc([this]() { return CreateInsecureChannel(); }, {}, {});
9142 // Deleting the resource should result in a non-serving status.
9143 UnsetLdsUpdate();
9144 backends_[0]->notifier()->WaitOnServingStatusChange(
9145 absl::StrCat(ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()),
9146 grpc::StatusCode::NOT_FOUND);
9147 SendRpc([this]() { return CreateInsecureChannel(); }, {}, {},
9148 true /* test_expects_failure */);
9149 }
9150
TEST_P(XdsEnabledServerStatusNotificationTest,RepeatedServingStatusChanges)9151 TEST_P(XdsEnabledServerStatusNotificationTest, RepeatedServingStatusChanges) {
9152 for (int i = 0; i < 5; i++) {
9153 // Send a valid LDS update to get the server to start listening
9154 SetValidLdsUpdate();
9155 backends_[0]->notifier()->WaitOnServingStatusChange(
9156 absl::StrCat(ipv6_only_ ? "[::1]:" : "127.0.0.1:",
9157 backends_[0]->port()),
9158 grpc::StatusCode::OK);
9159 SendRpc([this]() { return CreateInsecureChannel(); }, {}, {});
9160 // Deleting the resource will make the server start rejecting connections
9161 UnsetLdsUpdate();
9162 backends_[0]->notifier()->WaitOnServingStatusChange(
9163 absl::StrCat(ipv6_only_ ? "[::1]:" : "127.0.0.1:",
9164 backends_[0]->port()),
9165 grpc::StatusCode::NOT_FOUND);
9166 SendRpc([this]() { return CreateInsecureChannel(); }, {}, {},
9167 true /* test_expects_failure */);
9168 }
9169 }
9170
TEST_P(XdsEnabledServerStatusNotificationTest,ExistingRpcsOnResourceDeletion)9171 TEST_P(XdsEnabledServerStatusNotificationTest, ExistingRpcsOnResourceDeletion) {
9172 // Send a valid LDS update to get the server to start listening
9173 SetValidLdsUpdate();
9174 backends_[0]->notifier()->WaitOnServingStatusChange(
9175 absl::StrCat(ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()),
9176 grpc::StatusCode::OK);
9177 constexpr int kNumChannels = 10;
9178 struct StreamingRpc {
9179 std::shared_ptr<Channel> channel;
9180 std::unique_ptr<grpc::testing::EchoTestService::Stub> stub;
9181 ClientContext context;
9182 std::unique_ptr<ClientReaderWriter<EchoRequest, EchoResponse>> stream;
9183 } streaming_rpcs[kNumChannels];
9184 EchoRequest request;
9185 EchoResponse response;
9186 request.set_message("Hello");
9187 for (int i = 0; i < kNumChannels; i++) {
9188 streaming_rpcs[i].channel = CreateInsecureChannel();
9189 streaming_rpcs[i].stub =
9190 grpc::testing::EchoTestService::NewStub(streaming_rpcs[i].channel);
9191 streaming_rpcs[i].context.set_wait_for_ready(true);
9192 streaming_rpcs[i].stream =
9193 streaming_rpcs[i].stub->BidiStream(&streaming_rpcs[i].context);
9194 EXPECT_TRUE(streaming_rpcs[i].stream->Write(request));
9195 streaming_rpcs[i].stream->Read(&response);
9196 EXPECT_EQ(request.message(), response.message());
9197 }
9198 // Deleting the resource will make the server start rejecting connections
9199 UnsetLdsUpdate();
9200 backends_[0]->notifier()->WaitOnServingStatusChange(
9201 absl::StrCat(ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()),
9202 grpc::StatusCode::NOT_FOUND);
9203 SendRpc([this]() { return CreateInsecureChannel(); }, {}, {},
9204 true /* test_expects_failure */);
9205 for (int i = 0; i < kNumChannels; i++) {
9206 EXPECT_TRUE(streaming_rpcs[i].stream->Write(request));
9207 streaming_rpcs[i].stream->Read(&response);
9208 EXPECT_EQ(request.message(), response.message());
9209 EXPECT_TRUE(streaming_rpcs[i].stream->WritesDone());
9210 auto status = streaming_rpcs[i].stream->Finish();
9211 EXPECT_TRUE(status.ok())
9212 << status.error_message() << ", " << status.error_details() << ", "
9213 << streaming_rpcs[i].context.debug_error_string();
9214 // New RPCs on the existing channels should fail.
9215 ClientContext new_context;
9216 new_context.set_deadline(grpc_timeout_milliseconds_to_deadline(1000));
9217 EXPECT_FALSE(
9218 streaming_rpcs[i].stub->Echo(&new_context, request, &response).ok());
9219 }
9220 }
9221
9222 using XdsServerFilterChainMatchTest = XdsServerSecurityTest;
9223
TEST_P(XdsServerFilterChainMatchTest,DefaultFilterChainUsedWhenNoFilterChainMentioned)9224 TEST_P(XdsServerFilterChainMatchTest,
9225 DefaultFilterChainUsedWhenNoFilterChainMentioned) {
9226 SendRpc([this]() { return CreateInsecureChannel(); }, {}, {});
9227 }
9228
TEST_P(XdsServerFilterChainMatchTest,DefaultFilterChainUsedWhenOtherFilterChainsDontMatch)9229 TEST_P(XdsServerFilterChainMatchTest,
9230 DefaultFilterChainUsedWhenOtherFilterChainsDontMatch) {
9231 Listener listener = default_server_listener_;
9232 // Add a filter chain that will never get matched
9233 auto* filter_chain = listener.add_filter_chains();
9234 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9235 ServerHcmAccessor().Unpack(listener));
9236 filter_chain->mutable_filter_chain_match()
9237 ->mutable_destination_port()
9238 ->set_value(8080);
9239 SetServerListenerNameAndRouteConfiguration(0, listener, backends_[0]->port(),
9240 default_server_route_config_);
9241 SendRpc([this]() { return CreateInsecureChannel(); }, {}, {});
9242 }
9243
TEST_P(XdsServerFilterChainMatchTest,FilterChainsWithDestinationPortDontMatch)9244 TEST_P(XdsServerFilterChainMatchTest,
9245 FilterChainsWithDestinationPortDontMatch) {
9246 Listener listener = default_server_listener_;
9247 // Add filter chain with destination port that should never get matched
9248 auto* filter_chain = listener.add_filter_chains();
9249 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9250 ServerHcmAccessor().Unpack(listener));
9251 filter_chain->mutable_filter_chain_match()
9252 ->mutable_destination_port()
9253 ->set_value(8080);
9254 listener.clear_default_filter_chain();
9255 balancers_[0]->ads_service()->SetLdsResource(
9256 PopulateServerListenerNameAndPort(listener, backends_[0]->port()));
9257 // RPC should fail since no matching filter chain was found and no default
9258 // filter chain is configured.
9259 SendRpc([this]() { return CreateInsecureChannel(); }, {}, {},
9260 true /* test_expects_failure */);
9261 }
9262
TEST_P(XdsServerFilterChainMatchTest,FilterChainsWithServerNamesDontMatch)9263 TEST_P(XdsServerFilterChainMatchTest, FilterChainsWithServerNamesDontMatch) {
9264 Listener listener = default_server_listener_;
9265 // Add filter chain with server name that should never get matched
9266 auto* filter_chain = listener.add_filter_chains();
9267 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9268 ServerHcmAccessor().Unpack(listener));
9269 filter_chain->mutable_filter_chain_match()->add_server_names("server_name");
9270 listener.clear_default_filter_chain();
9271 balancers_[0]->ads_service()->SetLdsResource(
9272 PopulateServerListenerNameAndPort(listener, backends_[0]->port()));
9273 // RPC should fail since no matching filter chain was found and no default
9274 // filter chain is configured.
9275 SendRpc([this]() { return CreateInsecureChannel(); }, {}, {},
9276 true /* test_expects_failure */);
9277 }
9278
TEST_P(XdsServerFilterChainMatchTest,FilterChainsWithTransportProtocolsOtherThanRawBufferDontMatch)9279 TEST_P(XdsServerFilterChainMatchTest,
9280 FilterChainsWithTransportProtocolsOtherThanRawBufferDontMatch) {
9281 Listener listener = default_server_listener_;
9282 // Add filter chain with transport protocol "tls" that should never match
9283 auto* filter_chain = listener.add_filter_chains();
9284 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9285 ServerHcmAccessor().Unpack(listener));
9286 filter_chain->mutable_filter_chain_match()->set_transport_protocol("tls");
9287 listener.clear_default_filter_chain();
9288 balancers_[0]->ads_service()->SetLdsResource(
9289 PopulateServerListenerNameAndPort(listener, backends_[0]->port()));
9290 // RPC should fail since no matching filter chain was found and no default
9291 // filter chain is configured.
9292 SendRpc([this]() { return CreateInsecureChannel(); }, {}, {},
9293 true /* test_expects_failure */);
9294 }
9295
TEST_P(XdsServerFilterChainMatchTest,FilterChainsWithApplicationProtocolsDontMatch)9296 TEST_P(XdsServerFilterChainMatchTest,
9297 FilterChainsWithApplicationProtocolsDontMatch) {
9298 Listener listener = default_server_listener_;
9299 // Add filter chain with application protocol that should never get matched
9300 auto* filter_chain = listener.add_filter_chains();
9301 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9302 ServerHcmAccessor().Unpack(listener));
9303 filter_chain->mutable_filter_chain_match()->add_application_protocols("h2");
9304 listener.clear_default_filter_chain();
9305 balancers_[0]->ads_service()->SetLdsResource(
9306 PopulateServerListenerNameAndPort(listener, backends_[0]->port()));
9307 // RPC should fail since no matching filter chain was found and no default
9308 // filter chain is configured.
9309 SendRpc([this]() { return CreateInsecureChannel(); }, {}, {},
9310 true /* test_expects_failure */);
9311 }
9312
TEST_P(XdsServerFilterChainMatchTest,FilterChainsWithTransportProtocolRawBufferIsPreferred)9313 TEST_P(XdsServerFilterChainMatchTest,
9314 FilterChainsWithTransportProtocolRawBufferIsPreferred) {
9315 Listener listener = default_server_listener_;
9316 // Add filter chain with "raw_buffer" transport protocol
9317 auto* filter_chain = listener.add_filter_chains();
9318 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9319 ServerHcmAccessor().Unpack(listener));
9320 filter_chain->mutable_filter_chain_match()->set_transport_protocol(
9321 "raw_buffer");
9322 // Add another filter chain with no transport protocol set but application
9323 // protocol set (fails match)
9324 filter_chain = listener.add_filter_chains();
9325 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9326 ServerHcmAccessor().Unpack(listener));
9327 filter_chain->mutable_filter_chain_match()->add_application_protocols("h2");
9328 listener.clear_default_filter_chain();
9329 balancers_[0]->ads_service()->SetLdsResource(
9330 PopulateServerListenerNameAndPort(listener, backends_[0]->port()));
9331 // A successful RPC proves that filter chains that mention "raw_buffer" as
9332 // the transport protocol are chosen as the best match in the round.
9333 SendRpc([this]() { return CreateInsecureChannel(); }, {}, {});
9334 }
9335
TEST_P(XdsServerFilterChainMatchTest,FilterChainsWithMoreSpecificDestinationPrefixRangesArePreferred)9336 TEST_P(XdsServerFilterChainMatchTest,
9337 FilterChainsWithMoreSpecificDestinationPrefixRangesArePreferred) {
9338 Listener listener = default_server_listener_;
9339 // Add filter chain with prefix range (length 4 and 16) but with server name
9340 // mentioned. (Prefix range is matched first.)
9341 auto* filter_chain = listener.add_filter_chains();
9342 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9343 ServerHcmAccessor().Unpack(listener));
9344 auto* prefix_range =
9345 filter_chain->mutable_filter_chain_match()->add_prefix_ranges();
9346 prefix_range->set_address_prefix(ipv6_only_ ? "::1" : "127.0.0.1");
9347 prefix_range->mutable_prefix_len()->set_value(4);
9348 prefix_range =
9349 filter_chain->mutable_filter_chain_match()->add_prefix_ranges();
9350 prefix_range->set_address_prefix(ipv6_only_ ? "::1" : "127.0.0.1");
9351 prefix_range->mutable_prefix_len()->set_value(16);
9352 filter_chain->mutable_filter_chain_match()->add_server_names("server_name");
9353 // Add filter chain with two prefix ranges (length 8 and 24). Since 24 is
9354 // the highest match, it should be chosen.
9355 filter_chain = listener.add_filter_chains();
9356 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9357 ServerHcmAccessor().Unpack(listener));
9358 prefix_range =
9359 filter_chain->mutable_filter_chain_match()->add_prefix_ranges();
9360 prefix_range->set_address_prefix(ipv6_only_ ? "::1" : "127.0.0.1");
9361 prefix_range->mutable_prefix_len()->set_value(8);
9362 prefix_range =
9363 filter_chain->mutable_filter_chain_match()->add_prefix_ranges();
9364 prefix_range->set_address_prefix(ipv6_only_ ? "::1" : "127.0.0.1");
9365 prefix_range->mutable_prefix_len()->set_value(24);
9366 // Add another filter chain with a non-matching prefix range (with length
9367 // 30)
9368 filter_chain = listener.add_filter_chains();
9369 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9370 ServerHcmAccessor().Unpack(listener));
9371 prefix_range =
9372 filter_chain->mutable_filter_chain_match()->add_prefix_ranges();
9373 prefix_range->set_address_prefix("192.168.1.1");
9374 prefix_range->mutable_prefix_len()->set_value(30);
9375 filter_chain->mutable_filter_chain_match()->add_server_names("server_name");
9376 // Add another filter chain with no prefix range mentioned
9377 filter_chain = listener.add_filter_chains();
9378 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9379 ServerHcmAccessor().Unpack(listener));
9380 filter_chain->mutable_filter_chain_match()->add_server_names("server_name");
9381 listener.clear_default_filter_chain();
9382 balancers_[0]->ads_service()->SetLdsResource(
9383 PopulateServerListenerNameAndPort(listener, backends_[0]->port()));
9384 // A successful RPC proves that the filter chain with the longest matching
9385 // prefix range was the best match.
9386 SendRpc([this]() { return CreateInsecureChannel(); }, {}, {});
9387 }
9388
TEST_P(XdsServerFilterChainMatchTest,FilterChainsThatMentionSourceTypeArePreferred)9389 TEST_P(XdsServerFilterChainMatchTest,
9390 FilterChainsThatMentionSourceTypeArePreferred) {
9391 Listener listener = default_server_listener_;
9392 // Add filter chain with the local source type (best match)
9393 auto* filter_chain = listener.add_filter_chains();
9394 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9395 ServerHcmAccessor().Unpack(listener));
9396 filter_chain->mutable_filter_chain_match()->set_source_type(
9397 FilterChainMatch::SAME_IP_OR_LOOPBACK);
9398 // Add filter chain with the external source type but bad source port.
9399 // Note that backends_[0]->port() will never be a match for the source port
9400 // because it is already being used by a backend.
9401 filter_chain = listener.add_filter_chains();
9402 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9403 ServerHcmAccessor().Unpack(listener));
9404 filter_chain->mutable_filter_chain_match()->set_source_type(
9405 FilterChainMatch::EXTERNAL);
9406 filter_chain->mutable_filter_chain_match()->add_source_ports(
9407 backends_[0]->port());
9408 // Add filter chain with the default source type (ANY) but bad source port.
9409 filter_chain = listener.add_filter_chains();
9410 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9411 ServerHcmAccessor().Unpack(listener));
9412 filter_chain->mutable_filter_chain_match()->add_source_ports(
9413 backends_[0]->port());
9414 listener.clear_default_filter_chain();
9415 balancers_[0]->ads_service()->SetLdsResource(
9416 PopulateServerListenerNameAndPort(listener, backends_[0]->port()));
9417 // A successful RPC proves that the filter chain with the longest matching
9418 // prefix range was the best match.
9419 SendRpc([this]() { return CreateInsecureChannel(); }, {}, {});
9420 }
9421
TEST_P(XdsServerFilterChainMatchTest,FilterChainsWithMoreSpecificSourcePrefixRangesArePreferred)9422 TEST_P(XdsServerFilterChainMatchTest,
9423 FilterChainsWithMoreSpecificSourcePrefixRangesArePreferred) {
9424 Listener listener = default_server_listener_;
9425 // Add filter chain with source prefix range (length 16) but with a bad
9426 // source port mentioned. (Prefix range is matched first.) Note that
9427 // backends_[0]->port() will never be a match for the source port because it
9428 // is already being used by a backend.
9429 auto* filter_chain = listener.add_filter_chains();
9430 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9431 ServerHcmAccessor().Unpack(listener));
9432 auto* source_prefix_range =
9433 filter_chain->mutable_filter_chain_match()->add_source_prefix_ranges();
9434 source_prefix_range->set_address_prefix(ipv6_only_ ? "::1" : "127.0.0.1");
9435 source_prefix_range->mutable_prefix_len()->set_value(4);
9436 source_prefix_range =
9437 filter_chain->mutable_filter_chain_match()->add_source_prefix_ranges();
9438 source_prefix_range->set_address_prefix(ipv6_only_ ? "::1" : "127.0.0.1");
9439 source_prefix_range->mutable_prefix_len()->set_value(16);
9440 filter_chain->mutable_filter_chain_match()->add_source_ports(
9441 backends_[0]->port());
9442 // Add filter chain with two source prefix ranges (length 8 and 24). Since
9443 // 24 is the highest match, it should be chosen.
9444 filter_chain = listener.add_filter_chains();
9445 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9446 ServerHcmAccessor().Unpack(listener));
9447 source_prefix_range =
9448 filter_chain->mutable_filter_chain_match()->add_source_prefix_ranges();
9449 source_prefix_range->set_address_prefix(ipv6_only_ ? "::1" : "127.0.0.1");
9450 source_prefix_range->mutable_prefix_len()->set_value(8);
9451 source_prefix_range =
9452 filter_chain->mutable_filter_chain_match()->add_source_prefix_ranges();
9453 source_prefix_range->set_address_prefix(ipv6_only_ ? "::1" : "127.0.0.1");
9454 source_prefix_range->mutable_prefix_len()->set_value(24);
9455 // Add another filter chain with a non-matching source prefix range (with
9456 // length 30) and bad source port
9457 filter_chain = listener.add_filter_chains();
9458 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9459 ServerHcmAccessor().Unpack(listener));
9460 source_prefix_range =
9461 filter_chain->mutable_filter_chain_match()->add_source_prefix_ranges();
9462 source_prefix_range->set_address_prefix("192.168.1.1");
9463 source_prefix_range->mutable_prefix_len()->set_value(30);
9464 filter_chain->mutable_filter_chain_match()->add_source_ports(
9465 backends_[0]->port());
9466 // Add another filter chain with no source prefix range mentioned and bad
9467 // source port
9468 filter_chain = listener.add_filter_chains();
9469 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9470 ServerHcmAccessor().Unpack(listener));
9471 filter_chain->mutable_filter_chain_match()->add_source_ports(
9472 backends_[0]->port());
9473 listener.clear_default_filter_chain();
9474 balancers_[0]->ads_service()->SetLdsResource(
9475 PopulateServerListenerNameAndPort(listener, backends_[0]->port()));
9476 // A successful RPC proves that the filter chain with the longest matching
9477 // source prefix range was the best match.
9478 SendRpc([this]() { return CreateInsecureChannel(); }, {}, {});
9479 }
9480
TEST_P(XdsServerFilterChainMatchTest,FilterChainsWithMoreSpecificSourcePortArePreferred)9481 TEST_P(XdsServerFilterChainMatchTest,
9482 FilterChainsWithMoreSpecificSourcePortArePreferred) {
9483 Listener listener = default_server_listener_;
9484 auto* filter_chain = listener.add_filter_chains();
9485 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9486 ServerHcmAccessor().Unpack(listener));
9487 // Since we don't know which port will be used by the channel, just add all
9488 // ports except for 0.
9489 for (int i = 1; i < 65536; i++) {
9490 filter_chain->mutable_filter_chain_match()->add_source_ports(i);
9491 }
9492 // Add another filter chain with no source port mentioned with a bad
9493 // DownstreamTlsContext configuration.
9494 filter_chain = listener.add_filter_chains();
9495 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9496 ServerHcmAccessor().Unpack(listener));
9497 auto* transport_socket = filter_chain->mutable_transport_socket();
9498 transport_socket->set_name("envoy.transport_sockets.tls");
9499 DownstreamTlsContext downstream_tls_context;
9500 downstream_tls_context.mutable_common_tls_context()
9501 ->mutable_tls_certificate_provider_instance()
9502 ->set_instance_name("fake_plugin1");
9503 transport_socket->mutable_typed_config()->PackFrom(downstream_tls_context);
9504 listener.clear_default_filter_chain();
9505 balancers_[0]->ads_service()->SetLdsResource(
9506 PopulateServerListenerNameAndPort(listener, backends_[0]->port()));
9507 // A successful RPC proves that the filter chain with matching source port
9508 // was chosen.
9509 SendRpc([this]() { return CreateInsecureChannel(); }, {}, {});
9510 }
9511
TEST_P(XdsServerFilterChainMatchTest,DuplicateMatchNacked)9512 TEST_P(XdsServerFilterChainMatchTest, DuplicateMatchNacked) {
9513 Listener listener = default_server_listener_;
9514 // Add filter chain
9515 auto* filter_chain = listener.add_filter_chains();
9516 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9517 ServerHcmAccessor().Unpack(listener));
9518 // Add a duplicate filter chain
9519 filter_chain = listener.add_filter_chains();
9520 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9521 ServerHcmAccessor().Unpack(listener));
9522 SetServerListenerNameAndRouteConfiguration(0, listener, backends_[0]->port(),
9523 default_server_route_config_);
9524 ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED))
9525 << "timed out waiting for NACK";
9526 EXPECT_THAT(
9527 balancers_[0]->ads_service()->lds_response_state().error_message,
9528 ::testing::HasSubstr(
9529 "Duplicate matching rules detected when adding filter chain: {}"));
9530 }
9531
TEST_P(XdsServerFilterChainMatchTest,DuplicateMatchOnPrefixRangesNacked)9532 TEST_P(XdsServerFilterChainMatchTest, DuplicateMatchOnPrefixRangesNacked) {
9533 Listener listener = default_server_listener_;
9534 // Add filter chain with prefix range
9535 auto* filter_chain = listener.add_filter_chains();
9536 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9537 ServerHcmAccessor().Unpack(listener));
9538 auto* prefix_range =
9539 filter_chain->mutable_filter_chain_match()->add_prefix_ranges();
9540 prefix_range->set_address_prefix(ipv6_only_ ? "::1" : "127.0.0.1");
9541 prefix_range->mutable_prefix_len()->set_value(16);
9542 prefix_range =
9543 filter_chain->mutable_filter_chain_match()->add_prefix_ranges();
9544 prefix_range->set_address_prefix(ipv6_only_ ? "::1" : "127.0.0.1");
9545 prefix_range->mutable_prefix_len()->set_value(24);
9546 // Add a filter chain with a duplicate prefix range entry
9547 filter_chain = listener.add_filter_chains();
9548 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9549 ServerHcmAccessor().Unpack(listener));
9550 prefix_range =
9551 filter_chain->mutable_filter_chain_match()->add_prefix_ranges();
9552 prefix_range->set_address_prefix(ipv6_only_ ? "::1" : "127.0.0.1");
9553 prefix_range->mutable_prefix_len()->set_value(16);
9554 prefix_range =
9555 filter_chain->mutable_filter_chain_match()->add_prefix_ranges();
9556 prefix_range->set_address_prefix(ipv6_only_ ? "::1" : "127.0.0.1");
9557 prefix_range->mutable_prefix_len()->set_value(32);
9558 SetServerListenerNameAndRouteConfiguration(0, listener, backends_[0]->port(),
9559 default_server_route_config_);
9560 ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED))
9561 << "timed out waiting for NACK";
9562 if (ipv6_only_) {
9563 EXPECT_THAT(
9564 balancers_[0]->ads_service()->lds_response_state().error_message,
9565 ::testing::HasSubstr(
9566 "Duplicate matching rules detected when adding filter chain: "
9567 "{prefix_ranges={{address_prefix=[::]:0, prefix_len=16}, "
9568 "{address_prefix=[::]:0, prefix_len=32}}}"));
9569 } else {
9570 EXPECT_THAT(
9571 balancers_[0]->ads_service()->lds_response_state().error_message,
9572 ::testing::HasSubstr(
9573 "Duplicate matching rules detected when adding filter chain: "
9574 "{prefix_ranges={{address_prefix=127.0.0.0:0, prefix_len=16}, "
9575 "{address_prefix=127.0.0.1:0, prefix_len=32}}}"));
9576 }
9577 }
9578
TEST_P(XdsServerFilterChainMatchTest,DuplicateMatchOnTransportProtocolNacked)9579 TEST_P(XdsServerFilterChainMatchTest, DuplicateMatchOnTransportProtocolNacked) {
9580 Listener listener = default_server_listener_;
9581 // Add filter chain with "raw_buffer" transport protocol
9582 auto* filter_chain = listener.add_filter_chains();
9583 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9584 ServerHcmAccessor().Unpack(listener));
9585 filter_chain->mutable_filter_chain_match()->set_transport_protocol(
9586 "raw_buffer");
9587 // Add a duplicate filter chain with the same "raw_buffer" transport
9588 // protocol entry
9589 filter_chain = listener.add_filter_chains();
9590 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9591 ServerHcmAccessor().Unpack(listener));
9592 filter_chain->mutable_filter_chain_match()->set_transport_protocol(
9593 "raw_buffer");
9594 SetServerListenerNameAndRouteConfiguration(0, listener, backends_[0]->port(),
9595 default_server_route_config_);
9596 ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED))
9597 << "timed out waiting for NACK";
9598 EXPECT_THAT(
9599 balancers_[0]->ads_service()->lds_response_state().error_message,
9600 ::testing::HasSubstr("Duplicate matching rules detected when adding "
9601 "filter chain: {transport_protocol=raw_buffer}"));
9602 }
9603
TEST_P(XdsServerFilterChainMatchTest,DuplicateMatchOnLocalSourceTypeNacked)9604 TEST_P(XdsServerFilterChainMatchTest, DuplicateMatchOnLocalSourceTypeNacked) {
9605 Listener listener = default_server_listener_;
9606 // Add filter chain with the local source type
9607 auto* filter_chain = listener.add_filter_chains();
9608 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9609 ServerHcmAccessor().Unpack(listener));
9610 filter_chain->mutable_filter_chain_match()->set_source_type(
9611 FilterChainMatch::SAME_IP_OR_LOOPBACK);
9612 // Add a duplicate filter chain with the same local source type entry
9613 filter_chain = listener.add_filter_chains();
9614 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9615 ServerHcmAccessor().Unpack(listener));
9616 filter_chain->mutable_filter_chain_match()->set_source_type(
9617 FilterChainMatch::SAME_IP_OR_LOOPBACK);
9618 SetServerListenerNameAndRouteConfiguration(0, listener, backends_[0]->port(),
9619 default_server_route_config_);
9620 ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED))
9621 << "timed out waiting for NACK";
9622 EXPECT_THAT(
9623 balancers_[0]->ads_service()->lds_response_state().error_message,
9624 ::testing::HasSubstr("Duplicate matching rules detected when adding "
9625 "filter chain: {source_type=SAME_IP_OR_LOOPBACK}"));
9626 }
9627
TEST_P(XdsServerFilterChainMatchTest,DuplicateMatchOnExternalSourceTypeNacked)9628 TEST_P(XdsServerFilterChainMatchTest,
9629 DuplicateMatchOnExternalSourceTypeNacked) {
9630 Listener listener = default_server_listener_;
9631 // Add filter chain with the external source type
9632 auto* filter_chain = listener.add_filter_chains();
9633 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9634 ServerHcmAccessor().Unpack(listener));
9635 filter_chain->mutable_filter_chain_match()->set_source_type(
9636 FilterChainMatch::EXTERNAL);
9637 // Add a duplicate filter chain with the same external source type entry
9638 filter_chain = listener.add_filter_chains();
9639 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9640 ServerHcmAccessor().Unpack(listener));
9641 filter_chain->mutable_filter_chain_match()->set_source_type(
9642 FilterChainMatch::EXTERNAL);
9643 SetServerListenerNameAndRouteConfiguration(0, listener, backends_[0]->port(),
9644 default_server_route_config_);
9645 ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED))
9646 << "timed out waiting for NACK";
9647 EXPECT_THAT(
9648 balancers_[0]->ads_service()->lds_response_state().error_message,
9649 ::testing::HasSubstr("Duplicate matching rules detected when adding "
9650 "filter chain: {source_type=EXTERNAL}"));
9651 }
9652
TEST_P(XdsServerFilterChainMatchTest,DuplicateMatchOnSourcePrefixRangesNacked)9653 TEST_P(XdsServerFilterChainMatchTest,
9654 DuplicateMatchOnSourcePrefixRangesNacked) {
9655 Listener listener = default_server_listener_;
9656 // Add filter chain with source prefix range
9657 auto* filter_chain = listener.add_filter_chains();
9658 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9659 ServerHcmAccessor().Unpack(listener));
9660 auto* prefix_range =
9661 filter_chain->mutable_filter_chain_match()->add_source_prefix_ranges();
9662 prefix_range->set_address_prefix(ipv6_only_ ? "::1" : "127.0.0.1");
9663 prefix_range->mutable_prefix_len()->set_value(16);
9664 prefix_range =
9665 filter_chain->mutable_filter_chain_match()->add_source_prefix_ranges();
9666 prefix_range->set_address_prefix(ipv6_only_ ? "::1" : "127.0.0.1");
9667 prefix_range->mutable_prefix_len()->set_value(24);
9668 // Add a filter chain with a duplicate source prefix range entry
9669 filter_chain = listener.add_filter_chains();
9670 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9671 ServerHcmAccessor().Unpack(listener));
9672 prefix_range =
9673 filter_chain->mutable_filter_chain_match()->add_source_prefix_ranges();
9674 prefix_range->set_address_prefix(ipv6_only_ ? "::1" : "127.0.0.1");
9675 prefix_range->mutable_prefix_len()->set_value(16);
9676 prefix_range =
9677 filter_chain->mutable_filter_chain_match()->add_source_prefix_ranges();
9678 prefix_range->set_address_prefix(ipv6_only_ ? "::1" : "127.0.0.1");
9679 prefix_range->mutable_prefix_len()->set_value(32);
9680 SetServerListenerNameAndRouteConfiguration(0, listener, backends_[0]->port(),
9681 default_server_route_config_);
9682 ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED))
9683 << "timed out waiting for NACK";
9684 if (ipv6_only_) {
9685 EXPECT_THAT(
9686 balancers_[0]->ads_service()->lds_response_state().error_message,
9687 ::testing::HasSubstr(
9688 "Duplicate matching rules detected when adding filter chain: "
9689 "{source_prefix_ranges={{address_prefix=[::]:0, prefix_len=16}, "
9690 "{address_prefix=[::]:0, prefix_len=32}}}"));
9691 } else {
9692 EXPECT_THAT(
9693 balancers_[0]->ads_service()->lds_response_state().error_message,
9694 ::testing::HasSubstr(
9695 "Duplicate matching rules detected when adding filter chain: "
9696 "{source_prefix_ranges={{address_prefix=127.0.0.0:0, "
9697 "prefix_len=16}, "
9698 "{address_prefix=127.0.0.1:0, prefix_len=32}}}"));
9699 }
9700 }
9701
TEST_P(XdsServerFilterChainMatchTest,DuplicateMatchOnSourcePortNacked)9702 TEST_P(XdsServerFilterChainMatchTest, DuplicateMatchOnSourcePortNacked) {
9703 Listener listener = default_server_listener_;
9704 // Add filter chain with the external source type
9705 auto* filter_chain = listener.add_filter_chains();
9706 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9707 ServerHcmAccessor().Unpack(listener));
9708 filter_chain->mutable_filter_chain_match()->add_source_ports(8080);
9709 // Add a duplicate filter chain with the same source port entry
9710 filter_chain = listener.add_filter_chains();
9711 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9712 ServerHcmAccessor().Unpack(listener));
9713 filter_chain->mutable_filter_chain_match()->add_source_ports(8080);
9714 SetServerListenerNameAndRouteConfiguration(0, listener, backends_[0]->port(),
9715 default_server_route_config_);
9716 ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED))
9717 << "timed out waiting for NACK";
9718 EXPECT_THAT(
9719 balancers_[0]->ads_service()->lds_response_state().error_message,
9720 ::testing::HasSubstr("Duplicate matching rules detected when adding "
9721 "filter chain: {source_ports={8080}}"));
9722 }
9723
9724 class XdsServerRdsTest : public XdsServerSecurityTest {
9725 protected:
SetUpTestSuite()9726 static void SetUpTestSuite() {
9727 gpr_setenv("GRPC_XDS_EXPERIMENTAL_RBAC", "true");
9728 }
9729
TearDownTestSuite()9730 static void TearDownTestSuite() {
9731 gpr_unsetenv("GRPC_XDS_EXPERIMENTAL_RBAC");
9732 }
9733 };
9734
TEST_P(XdsServerRdsTest,NacksInvalidDomainPattern)9735 TEST_P(XdsServerRdsTest, NacksInvalidDomainPattern) {
9736 RouteConfiguration route_config = default_server_route_config_;
9737 route_config.mutable_virtual_hosts()->at(0).add_domains("");
9738 SetServerListenerNameAndRouteConfiguration(
9739 0, default_server_listener_, backends_[0]->port(), route_config);
9740 ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED))
9741 << "timed out waiting for NACK";
9742 EXPECT_THAT(balancers_[0]->ads_service()->lds_response_state().error_message,
9743 ::testing::HasSubstr("Invalid domain pattern \"\""));
9744 }
9745
TEST_P(XdsServerRdsTest,NacksEmptyDomainsList)9746 TEST_P(XdsServerRdsTest, NacksEmptyDomainsList) {
9747 RouteConfiguration route_config = default_server_route_config_;
9748 route_config.mutable_virtual_hosts()->at(0).clear_domains();
9749 SetServerListenerNameAndRouteConfiguration(
9750 0, default_server_listener_, backends_[0]->port(), route_config);
9751 ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED))
9752 << "timed out waiting for NACK";
9753 EXPECT_THAT(balancers_[0]->ads_service()->lds_response_state().error_message,
9754 ::testing::HasSubstr("VirtualHost has no domains"));
9755 }
9756
TEST_P(XdsServerRdsTest,NacksEmptyRoutesList)9757 TEST_P(XdsServerRdsTest, NacksEmptyRoutesList) {
9758 RouteConfiguration route_config = default_server_route_config_;
9759 route_config.mutable_virtual_hosts()->at(0).clear_routes();
9760 SetServerListenerNameAndRouteConfiguration(
9761 0, default_server_listener_, backends_[0]->port(), route_config);
9762 ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED))
9763 << "timed out waiting for NACK";
9764 EXPECT_THAT(balancers_[0]->ads_service()->lds_response_state().error_message,
9765 ::testing::HasSubstr("No route found in the virtual host"));
9766 }
9767
TEST_P(XdsServerRdsTest,NacksEmptyMatch)9768 TEST_P(XdsServerRdsTest, NacksEmptyMatch) {
9769 RouteConfiguration route_config = default_server_route_config_;
9770 route_config.mutable_virtual_hosts()
9771 ->at(0)
9772 .mutable_routes()
9773 ->at(0)
9774 .clear_match();
9775 SetServerListenerNameAndRouteConfiguration(
9776 0, default_server_listener_, backends_[0]->port(), route_config);
9777 ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED))
9778 << "timed out waiting for NACK";
9779 EXPECT_THAT(balancers_[0]->ads_service()->lds_response_state().error_message,
9780 ::testing::HasSubstr("Match can't be null"));
9781 }
9782
9783 using EdsTest = BasicTest;
9784
9785 // Tests that EDS client should send a NACK if the EDS update contains
9786 // sparse priorities.
TEST_P(EdsTest,NacksSparsePriorityList)9787 TEST_P(EdsTest, NacksSparsePriorityList) {
9788 SetNextResolution({});
9789 SetNextResolutionForLbChannelAllBalancers();
9790 EdsResourceArgs args({
9791 {"locality0", CreateEndpointsForBackends(), kDefaultLocalityWeight, 1},
9792 });
9793 balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
9794 ASSERT_TRUE(WaitForEdsNack()) << "timed out waiting for NACK";
9795 const auto response_state =
9796 balancers_[0]->ads_service()->eds_response_state();
9797 EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
9798 EXPECT_THAT(response_state.error_message,
9799 ::testing::HasSubstr("sparse priority list"));
9800 }
9801
9802 // In most of our tests, we use different names for different resource
9803 // types, to make sure that there are no cut-and-paste errors in the code
9804 // that cause us to look at data for the wrong resource type. So we add
9805 // this test to make sure that the EDS resource name defaults to the
9806 // cluster name if not specified in the CDS resource.
TEST_P(EdsTest,EdsServiceNameDefaultsToClusterName)9807 TEST_P(EdsTest, EdsServiceNameDefaultsToClusterName) {
9808 EdsResourceArgs args({
9809 {"locality0", CreateEndpointsForBackends()},
9810 });
9811 balancers_[0]->ads_service()->SetEdsResource(
9812 BuildEdsResource(args, kDefaultClusterName));
9813 Cluster cluster = default_cluster_;
9814 cluster.mutable_eds_cluster_config()->clear_service_name();
9815 balancers_[0]->ads_service()->SetCdsResource(cluster);
9816 SetNextResolution({});
9817 SetNextResolutionForLbChannelAllBalancers();
9818 CheckRpcSendOk();
9819 }
9820
9821 class TimeoutTest : public BasicTest {
9822 protected:
SetUp()9823 void SetUp() override {
9824 xds_resource_does_not_exist_timeout_ms_ = 500;
9825 BasicTest::SetUp();
9826 }
9827 };
9828
9829 // Tests that LDS client times out when no response received.
TEST_P(TimeoutTest,Lds)9830 TEST_P(TimeoutTest, Lds) {
9831 balancers_[0]->ads_service()->IgnoreResourceType(kLdsTypeUrl);
9832 SetNextResolution({});
9833 SetNextResolutionForLbChannelAllBalancers();
9834 CheckRpcSendFailure();
9835 }
9836
TEST_P(TimeoutTest,Rds)9837 TEST_P(TimeoutTest, Rds) {
9838 balancers_[0]->ads_service()->IgnoreResourceType(kRdsTypeUrl);
9839 SetNextResolution({});
9840 SetNextResolutionForLbChannelAllBalancers();
9841 CheckRpcSendFailure();
9842 }
9843
9844 // Tests that CDS client times out when no response received.
TEST_P(TimeoutTest,Cds)9845 TEST_P(TimeoutTest, Cds) {
9846 balancers_[0]->ads_service()->IgnoreResourceType(kCdsTypeUrl);
9847 SetNextResolution({});
9848 SetNextResolutionForLbChannelAllBalancers();
9849 CheckRpcSendFailure();
9850 }
9851
TEST_P(TimeoutTest,Eds)9852 TEST_P(TimeoutTest, Eds) {
9853 balancers_[0]->ads_service()->IgnoreResourceType(kEdsTypeUrl);
9854 SetNextResolution({});
9855 SetNextResolutionForLbChannelAllBalancers();
9856 CheckRpcSendFailure();
9857 }
9858
9859 using LocalityMapTest = BasicTest;
9860
9861 // Tests that the localities in a locality map are picked according to their
9862 // weights.
TEST_P(LocalityMapTest,WeightedRoundRobin)9863 TEST_P(LocalityMapTest, WeightedRoundRobin) {
9864 SetNextResolution({});
9865 SetNextResolutionForLbChannelAllBalancers();
9866 const int kLocalityWeight0 = 2;
9867 const int kLocalityWeight1 = 8;
9868 const int kTotalLocalityWeight = kLocalityWeight0 + kLocalityWeight1;
9869 const double kLocalityWeightRate0 =
9870 static_cast<double>(kLocalityWeight0) / kTotalLocalityWeight;
9871 const double kLocalityWeightRate1 =
9872 static_cast<double>(kLocalityWeight1) / kTotalLocalityWeight;
9873 const double kErrorTolerance = 0.05;
9874 const size_t kNumRpcs =
9875 ComputeIdealNumRpcs(kLocalityWeightRate0, kErrorTolerance);
9876 // ADS response contains 2 localities, each of which contains 1 backend.
9877 EdsResourceArgs args({
9878 {"locality0", CreateEndpointsForBackends(0, 1), kLocalityWeight0},
9879 {"locality1", CreateEndpointsForBackends(1, 2), kLocalityWeight1},
9880 });
9881 balancers_[0]->ads_service()->SetEdsResource(
9882 BuildEdsResource(args, DefaultEdsServiceName()));
9883 // Wait for both backends to be ready.
9884 WaitForAllBackends(0, 2);
9885 // Send kNumRpcs RPCs.
9886 CheckRpcSendOk(kNumRpcs);
9887 // The locality picking rates should be roughly equal to the expectation.
9888 const double locality_picked_rate_0 =
9889 static_cast<double>(backends_[0]->backend_service()->request_count()) /
9890 kNumRpcs;
9891 const double locality_picked_rate_1 =
9892 static_cast<double>(backends_[1]->backend_service()->request_count()) /
9893 kNumRpcs;
9894 EXPECT_THAT(locality_picked_rate_0,
9895 ::testing::DoubleNear(kLocalityWeightRate0, kErrorTolerance));
9896 EXPECT_THAT(locality_picked_rate_1,
9897 ::testing::DoubleNear(kLocalityWeightRate1, kErrorTolerance));
9898 }
9899
9900 // Tests that we correctly handle a locality containing no endpoints.
TEST_P(LocalityMapTest,LocalityContainingNoEndpoints)9901 TEST_P(LocalityMapTest, LocalityContainingNoEndpoints) {
9902 SetNextResolution({});
9903 SetNextResolutionForLbChannelAllBalancers();
9904 const size_t kNumRpcs = 5000;
9905 // EDS response contains 2 localities, one with no endpoints.
9906 EdsResourceArgs args({
9907 {"locality0", CreateEndpointsForBackends()},
9908 {"locality1", {}},
9909 });
9910 balancers_[0]->ads_service()->SetEdsResource(
9911 BuildEdsResource(args, DefaultEdsServiceName()));
9912 // Wait for both backends to be ready.
9913 WaitForAllBackends();
9914 // Send kNumRpcs RPCs.
9915 CheckRpcSendOk(kNumRpcs);
9916 // All traffic should go to the reachable locality.
9917 EXPECT_EQ(backends_[0]->backend_service()->request_count(),
9918 kNumRpcs / backends_.size());
9919 EXPECT_EQ(backends_[1]->backend_service()->request_count(),
9920 kNumRpcs / backends_.size());
9921 EXPECT_EQ(backends_[2]->backend_service()->request_count(),
9922 kNumRpcs / backends_.size());
9923 EXPECT_EQ(backends_[3]->backend_service()->request_count(),
9924 kNumRpcs / backends_.size());
9925 }
9926
9927 // EDS update with no localities.
TEST_P(LocalityMapTest,NoLocalities)9928 TEST_P(LocalityMapTest, NoLocalities) {
9929 SetNextResolution({});
9930 SetNextResolutionForLbChannelAllBalancers();
9931 balancers_[0]->ads_service()->SetEdsResource(
9932 BuildEdsResource({}, DefaultEdsServiceName()));
9933 Status status = SendRpc();
9934 EXPECT_FALSE(status.ok());
9935 EXPECT_EQ(status.error_code(), StatusCode::UNAVAILABLE);
9936 }
9937
9938 // Tests that the locality map can work properly even when it contains a large
9939 // number of localities.
TEST_P(LocalityMapTest,StressTest)9940 TEST_P(LocalityMapTest, StressTest) {
9941 SetNextResolution({});
9942 SetNextResolutionForLbChannelAllBalancers();
9943 const size_t kNumLocalities = 100;
9944 const uint32_t kRpcTimeoutMs = 5000;
9945 // The first ADS response contains kNumLocalities localities, each of which
9946 // contains backend 0.
9947 EdsResourceArgs args;
9948 for (size_t i = 0; i < kNumLocalities; ++i) {
9949 std::string name = absl::StrCat("locality", i);
9950 EdsResourceArgs::Locality locality(name, CreateEndpointsForBackends(0, 1));
9951 args.locality_list.emplace_back(std::move(locality));
9952 }
9953 balancers_[0]->ads_service()->SetEdsResource(
9954 BuildEdsResource(args, DefaultEdsServiceName()));
9955 // The second ADS response contains 1 locality, which contains backend 1.
9956 args = EdsResourceArgs({
9957 {"locality0", CreateEndpointsForBackends(1, 2)},
9958 });
9959 std::thread delayed_resource_setter(
9960 std::bind(&BasicTest::SetEdsResourceWithDelay, this, 0,
9961 BuildEdsResource(args, DefaultEdsServiceName()), 60 * 1000));
9962 // Wait until backend 0 is ready, before which kNumLocalities localities are
9963 // received and handled by the xds policy.
9964 WaitForBackend(0, WaitForBackendOptions().set_reset_counters(false),
9965 RpcOptions().set_timeout_ms(kRpcTimeoutMs));
9966 EXPECT_EQ(0U, backends_[1]->backend_service()->request_count());
9967 // Wait until backend 1 is ready, before which kNumLocalities localities are
9968 // removed by the xds policy.
9969 WaitForBackend(1);
9970 delayed_resource_setter.join();
9971 }
9972
9973 // Tests that the localities in a locality map are picked correctly after
9974 // update (addition, modification, deletion).
TEST_P(LocalityMapTest,UpdateMap)9975 TEST_P(LocalityMapTest, UpdateMap) {
9976 SetNextResolution({});
9977 SetNextResolutionForLbChannelAllBalancers();
9978 const size_t kNumRpcs = 3000;
9979 // The locality weight for the first 3 localities.
9980 const std::vector<int> kLocalityWeights0 = {2, 3, 4};
9981 const double kTotalLocalityWeight0 =
9982 std::accumulate(kLocalityWeights0.begin(), kLocalityWeights0.end(), 0);
9983 std::vector<double> locality_weight_rate_0;
9984 locality_weight_rate_0.reserve(kLocalityWeights0.size());
9985 for (int weight : kLocalityWeights0) {
9986 locality_weight_rate_0.push_back(weight / kTotalLocalityWeight0);
9987 }
9988 // Delete the first locality, keep the second locality, change the third
9989 // locality's weight from 4 to 2, and add a new locality with weight 6.
9990 const std::vector<int> kLocalityWeights1 = {3, 2, 6};
9991 const double kTotalLocalityWeight1 =
9992 std::accumulate(kLocalityWeights1.begin(), kLocalityWeights1.end(), 0);
9993 std::vector<double> locality_weight_rate_1 = {
9994 0 /* placeholder for locality 0 */};
9995 for (int weight : kLocalityWeights1) {
9996 locality_weight_rate_1.push_back(weight / kTotalLocalityWeight1);
9997 }
9998 EdsResourceArgs args({
9999 {"locality0", CreateEndpointsForBackends(0, 1), 2},
10000 {"locality1", CreateEndpointsForBackends(1, 2), 3},
10001 {"locality2", CreateEndpointsForBackends(2, 3), 4},
10002 });
10003 balancers_[0]->ads_service()->SetEdsResource(
10004 BuildEdsResource(args, DefaultEdsServiceName()));
10005 // Wait for the first 3 backends to be ready.
10006 WaitForAllBackends(0, 3);
10007 gpr_log(GPR_INFO, "========= BEFORE FIRST BATCH ==========");
10008 // Send kNumRpcs RPCs.
10009 CheckRpcSendOk(kNumRpcs);
10010 gpr_log(GPR_INFO, "========= DONE WITH FIRST BATCH ==========");
10011 // The picking rates of the first 3 backends should be roughly equal to the
10012 // expectation.
10013 std::vector<double> locality_picked_rates;
10014 for (size_t i = 0; i < 3; ++i) {
10015 locality_picked_rates.push_back(
10016 static_cast<double>(backends_[i]->backend_service()->request_count()) /
10017 kNumRpcs);
10018 }
10019 const double kErrorTolerance = 0.2;
10020 for (size_t i = 0; i < 3; ++i) {
10021 gpr_log(GPR_INFO, "Locality %" PRIuPTR " rate %f", i,
10022 locality_picked_rates[i]);
10023 EXPECT_THAT(
10024 locality_picked_rates[i],
10025 ::testing::AllOf(
10026 ::testing::Ge(locality_weight_rate_0[i] * (1 - kErrorTolerance)),
10027 ::testing::Le(locality_weight_rate_0[i] * (1 + kErrorTolerance))));
10028 }
10029 args = EdsResourceArgs({
10030 {"locality1", CreateEndpointsForBackends(1, 2), 3},
10031 {"locality2", CreateEndpointsForBackends(2, 3), 2},
10032 {"locality3", CreateEndpointsForBackends(3, 4), 6},
10033 });
10034 balancers_[0]->ads_service()->SetEdsResource(
10035 BuildEdsResource(args, DefaultEdsServiceName()));
10036 // Backend 3 hasn't received any request.
10037 EXPECT_EQ(0U, backends_[3]->backend_service()->request_count());
10038 // Wait until the locality update has been processed, as signaled by backend
10039 // 3 receiving a request.
10040 WaitForAllBackends(3, 4);
10041 gpr_log(GPR_INFO, "========= BEFORE SECOND BATCH ==========");
10042 // Send kNumRpcs RPCs.
10043 CheckRpcSendOk(kNumRpcs);
10044 gpr_log(GPR_INFO, "========= DONE WITH SECOND BATCH ==========");
10045 // Backend 0 no longer receives any request.
10046 EXPECT_EQ(0U, backends_[0]->backend_service()->request_count());
10047 // The picking rates of the last 3 backends should be roughly equal to the
10048 // expectation.
10049 locality_picked_rates = {0 /* placeholder for backend 0 */};
10050 for (size_t i = 1; i < 4; ++i) {
10051 locality_picked_rates.push_back(
10052 static_cast<double>(backends_[i]->backend_service()->request_count()) /
10053 kNumRpcs);
10054 }
10055 for (size_t i = 1; i < 4; ++i) {
10056 gpr_log(GPR_INFO, "Locality %" PRIuPTR " rate %f", i,
10057 locality_picked_rates[i]);
10058 EXPECT_THAT(
10059 locality_picked_rates[i],
10060 ::testing::AllOf(
10061 ::testing::Ge(locality_weight_rate_1[i] * (1 - kErrorTolerance)),
10062 ::testing::Le(locality_weight_rate_1[i] * (1 + kErrorTolerance))));
10063 }
10064 }
10065
10066 // Tests that we don't fail RPCs when replacing all of the localities in
10067 // a given priority.
TEST_P(LocalityMapTest,ReplaceAllLocalitiesInPriority)10068 TEST_P(LocalityMapTest, ReplaceAllLocalitiesInPriority) {
10069 SetNextResolution({});
10070 SetNextResolutionForLbChannelAllBalancers();
10071 EdsResourceArgs args({
10072 {"locality0", CreateEndpointsForBackends(0, 1)},
10073 });
10074 balancers_[0]->ads_service()->SetEdsResource(
10075 BuildEdsResource(args, DefaultEdsServiceName()));
10076 args = EdsResourceArgs({
10077 {"locality1", CreateEndpointsForBackends(1, 2)},
10078 });
10079 std::thread delayed_resource_setter(
10080 std::bind(&BasicTest::SetEdsResourceWithDelay, this, 0,
10081 BuildEdsResource(args, DefaultEdsServiceName()), 5000));
10082 // Wait for the first backend to be ready.
10083 WaitForBackend(0);
10084 // Keep sending RPCs until we switch over to backend 1, which tells us
10085 // that we received the update. No RPCs should fail during this
10086 // transition.
10087 WaitForBackend(1);
10088 delayed_resource_setter.join();
10089 }
10090
10091 class FailoverTest : public BasicTest {
10092 public:
SetUp()10093 void SetUp() override {
10094 BasicTest::SetUp();
10095 ResetStub(500);
10096 }
10097 };
10098
10099 // Localities with the highest priority are used when multiple priority exist.
TEST_P(FailoverTest,ChooseHighestPriority)10100 TEST_P(FailoverTest, ChooseHighestPriority) {
10101 SetNextResolution({});
10102 SetNextResolutionForLbChannelAllBalancers();
10103 EdsResourceArgs args({
10104 {"locality0", CreateEndpointsForBackends(0, 1), kDefaultLocalityWeight,
10105 1},
10106 {"locality1", CreateEndpointsForBackends(1, 2), kDefaultLocalityWeight,
10107 2},
10108 {"locality2", CreateEndpointsForBackends(2, 3), kDefaultLocalityWeight,
10109 3},
10110 {"locality3", CreateEndpointsForBackends(3, 4), kDefaultLocalityWeight,
10111 0},
10112 });
10113 balancers_[0]->ads_service()->SetEdsResource(
10114 BuildEdsResource(args, DefaultEdsServiceName()));
10115 WaitForBackend(3, WaitForBackendOptions().set_reset_counters(false));
10116 for (size_t i = 0; i < 3; ++i) {
10117 EXPECT_EQ(0U, backends_[i]->backend_service()->request_count());
10118 }
10119 }
10120
10121 // Does not choose priority with no endpoints.
TEST_P(FailoverTest,DoesNotUsePriorityWithNoEndpoints)10122 TEST_P(FailoverTest, DoesNotUsePriorityWithNoEndpoints) {
10123 SetNextResolution({});
10124 SetNextResolutionForLbChannelAllBalancers();
10125 EdsResourceArgs args({
10126 {"locality0", CreateEndpointsForBackends(0, 1), kDefaultLocalityWeight,
10127 1},
10128 {"locality1", CreateEndpointsForBackends(1, 2), kDefaultLocalityWeight,
10129 2},
10130 {"locality2", CreateEndpointsForBackends(2, 3), kDefaultLocalityWeight,
10131 3},
10132 {"locality3", {}, kDefaultLocalityWeight, 0},
10133 });
10134 balancers_[0]->ads_service()->SetEdsResource(
10135 BuildEdsResource(args, DefaultEdsServiceName()));
10136 WaitForBackend(0, WaitForBackendOptions().set_reset_counters(false));
10137 for (size_t i = 1; i < 3; ++i) {
10138 EXPECT_EQ(0U, backends_[i]->backend_service()->request_count());
10139 }
10140 }
10141
10142 // Does not choose locality with no endpoints.
TEST_P(FailoverTest,DoesNotUseLocalityWithNoEndpoints)10143 TEST_P(FailoverTest, DoesNotUseLocalityWithNoEndpoints) {
10144 SetNextResolution({});
10145 SetNextResolutionForLbChannelAllBalancers();
10146 EdsResourceArgs args({
10147 {"locality0", {}, kDefaultLocalityWeight, 0},
10148 {"locality1", CreateEndpointsForBackends(), kDefaultLocalityWeight, 0},
10149 });
10150 balancers_[0]->ads_service()->SetEdsResource(
10151 BuildEdsResource(args, DefaultEdsServiceName()));
10152 // Wait for all backends to be used.
10153 std::tuple<int, int, int> counts = WaitForAllBackends();
10154 // Make sure no RPCs failed in the transition.
10155 EXPECT_EQ(0, std::get<1>(counts));
10156 }
10157
10158 // If the higher priority localities are not reachable, failover to the
10159 // highest priority among the rest.
TEST_P(FailoverTest,Failover)10160 TEST_P(FailoverTest, Failover) {
10161 SetNextResolution({});
10162 SetNextResolutionForLbChannelAllBalancers();
10163 EdsResourceArgs args({
10164 {"locality0", CreateEndpointsForBackends(0, 1), kDefaultLocalityWeight,
10165 1},
10166 {"locality1", CreateEndpointsForBackends(1, 2), kDefaultLocalityWeight,
10167 2},
10168 {"locality2", CreateEndpointsForBackends(2, 3), kDefaultLocalityWeight,
10169 3},
10170 {"locality3", CreateEndpointsForBackends(3, 4), kDefaultLocalityWeight,
10171 0},
10172 });
10173 ShutdownBackend(3);
10174 ShutdownBackend(0);
10175 balancers_[0]->ads_service()->SetEdsResource(
10176 BuildEdsResource(args, DefaultEdsServiceName()));
10177 WaitForBackend(1, WaitForBackendOptions().set_reset_counters(false));
10178 for (size_t i = 0; i < 4; ++i) {
10179 if (i == 1) continue;
10180 EXPECT_EQ(0U, backends_[i]->backend_service()->request_count());
10181 }
10182 }
10183
10184 // If a locality with higher priority than the current one becomes ready,
10185 // switch to it.
TEST_P(FailoverTest,SwitchBackToHigherPriority)10186 TEST_P(FailoverTest, SwitchBackToHigherPriority) {
10187 SetNextResolution({});
10188 SetNextResolutionForLbChannelAllBalancers();
10189 const size_t kNumRpcs = 100;
10190 EdsResourceArgs args({
10191 {"locality0", CreateEndpointsForBackends(0, 1), kDefaultLocalityWeight,
10192 1},
10193 {"locality1", CreateEndpointsForBackends(1, 2), kDefaultLocalityWeight,
10194 2},
10195 {"locality2", CreateEndpointsForBackends(2, 3), kDefaultLocalityWeight,
10196 3},
10197 {"locality3", CreateEndpointsForBackends(3, 4), kDefaultLocalityWeight,
10198 0},
10199 });
10200 balancers_[0]->ads_service()->SetEdsResource(
10201 BuildEdsResource(args, DefaultEdsServiceName()));
10202 WaitForBackend(3);
10203 ShutdownBackend(3);
10204 ShutdownBackend(0);
10205 WaitForBackend(
10206 1, WaitForBackendOptions().set_reset_counters(false).set_allow_failures(
10207 true));
10208 for (size_t i = 0; i < 4; ++i) {
10209 if (i == 1) continue;
10210 EXPECT_EQ(0U, backends_[i]->backend_service()->request_count());
10211 }
10212 StartBackend(0);
10213 WaitForBackend(0);
10214 CheckRpcSendOk(kNumRpcs);
10215 EXPECT_EQ(kNumRpcs, backends_[0]->backend_service()->request_count());
10216 }
10217
10218 // The first update only contains unavailable priorities. The second update
10219 // contains available priorities.
TEST_P(FailoverTest,UpdateInitialUnavailable)10220 TEST_P(FailoverTest, UpdateInitialUnavailable) {
10221 SetNextResolution({});
10222 SetNextResolutionForLbChannelAllBalancers();
10223 EdsResourceArgs args({
10224 {"locality0", CreateEndpointsForBackends(0, 1), kDefaultLocalityWeight,
10225 0},
10226 {"locality1", CreateEndpointsForBackends(1, 2), kDefaultLocalityWeight,
10227 1},
10228 });
10229 balancers_[0]->ads_service()->SetEdsResource(
10230 BuildEdsResource(args, DefaultEdsServiceName()));
10231 args = EdsResourceArgs({
10232 {"locality0", CreateEndpointsForBackends(0, 1), kDefaultLocalityWeight,
10233 0},
10234 {"locality1", CreateEndpointsForBackends(1, 2), kDefaultLocalityWeight,
10235 1},
10236 {"locality2", CreateEndpointsForBackends(2, 3), kDefaultLocalityWeight,
10237 2},
10238 {"locality3", CreateEndpointsForBackends(3, 4), kDefaultLocalityWeight,
10239 3},
10240 });
10241 ShutdownBackend(0);
10242 ShutdownBackend(1);
10243 std::thread delayed_resource_setter(
10244 std::bind(&BasicTest::SetEdsResourceWithDelay, this, 0,
10245 BuildEdsResource(args, DefaultEdsServiceName()), 1000));
10246 gpr_timespec deadline = gpr_time_add(gpr_now(GPR_CLOCK_REALTIME),
10247 gpr_time_from_millis(500, GPR_TIMESPAN));
10248 // Send 0.5 second worth of RPCs.
10249 do {
10250 CheckRpcSendFailure();
10251 } while (gpr_time_cmp(gpr_now(GPR_CLOCK_REALTIME), deadline) < 0);
10252 WaitForBackend(
10253 2, WaitForBackendOptions().set_reset_counters(false).set_allow_failures(
10254 true));
10255 for (size_t i = 0; i < 4; ++i) {
10256 if (i == 2) continue;
10257 EXPECT_EQ(0U, backends_[i]->backend_service()->request_count());
10258 }
10259 delayed_resource_setter.join();
10260 }
10261
10262 // Tests that after the localities' priorities are updated, we still choose
10263 // the highest READY priority with the updated localities.
TEST_P(FailoverTest,UpdatePriority)10264 TEST_P(FailoverTest, UpdatePriority) {
10265 SetNextResolution({});
10266 SetNextResolutionForLbChannelAllBalancers();
10267 const size_t kNumRpcs = 100;
10268 EdsResourceArgs args({
10269 {"locality0", CreateEndpointsForBackends(0, 1), kDefaultLocalityWeight,
10270 1},
10271 {"locality1", CreateEndpointsForBackends(1, 2), kDefaultLocalityWeight,
10272 2},
10273 {"locality2", CreateEndpointsForBackends(2, 3), kDefaultLocalityWeight,
10274 3},
10275 {"locality3", CreateEndpointsForBackends(3, 4), kDefaultLocalityWeight,
10276 0},
10277 });
10278 balancers_[0]->ads_service()->SetEdsResource(
10279 BuildEdsResource(args, DefaultEdsServiceName()));
10280 args = EdsResourceArgs({
10281 {"locality0", CreateEndpointsForBackends(0, 1), kDefaultLocalityWeight,
10282 2},
10283 {"locality1", CreateEndpointsForBackends(1, 2), kDefaultLocalityWeight,
10284 0},
10285 {"locality2", CreateEndpointsForBackends(2, 3), kDefaultLocalityWeight,
10286 1},
10287 {"locality3", CreateEndpointsForBackends(3, 4), kDefaultLocalityWeight,
10288 3},
10289 });
10290 std::thread delayed_resource_setter(
10291 std::bind(&BasicTest::SetEdsResourceWithDelay, this, 0,
10292 BuildEdsResource(args, DefaultEdsServiceName()), 1000));
10293 WaitForBackend(3, WaitForBackendOptions().set_reset_counters(false));
10294 for (size_t i = 0; i < 3; ++i) {
10295 EXPECT_EQ(0U, backends_[i]->backend_service()->request_count());
10296 }
10297 WaitForBackend(1);
10298 CheckRpcSendOk(kNumRpcs);
10299 EXPECT_EQ(kNumRpcs, backends_[1]->backend_service()->request_count());
10300 delayed_resource_setter.join();
10301 }
10302
10303 // Moves all localities in the current priority to a higher priority.
TEST_P(FailoverTest,MoveAllLocalitiesInCurrentPriorityToHigherPriority)10304 TEST_P(FailoverTest, MoveAllLocalitiesInCurrentPriorityToHigherPriority) {
10305 SetNextResolution({});
10306 SetNextResolutionForLbChannelAllBalancers();
10307 // First update:
10308 // - Priority 0 is locality 0, containing backend 0, which is down.
10309 // - Priority 1 is locality 1, containing backends 1 and 2, which are up.
10310 ShutdownBackend(0);
10311 EdsResourceArgs args({
10312 {"locality0", CreateEndpointsForBackends(0, 1), kDefaultLocalityWeight,
10313 0},
10314 {"locality1", CreateEndpointsForBackends(1, 3), kDefaultLocalityWeight,
10315 1},
10316 });
10317 balancers_[0]->ads_service()->SetEdsResource(
10318 BuildEdsResource(args, DefaultEdsServiceName()));
10319 // Second update:
10320 // - Priority 0 contains both localities 0 and 1.
10321 // - Priority 1 is not present.
10322 // - We add backend 3 to locality 1, just so we have a way to know
10323 // when the update has been seen by the client.
10324 args = EdsResourceArgs({
10325 {"locality0", CreateEndpointsForBackends(0, 1), kDefaultLocalityWeight,
10326 0},
10327 {"locality1", CreateEndpointsForBackends(1, 4), kDefaultLocalityWeight,
10328 0},
10329 });
10330 std::thread delayed_resource_setter(
10331 std::bind(&BasicTest::SetEdsResourceWithDelay, this, 0,
10332 BuildEdsResource(args, DefaultEdsServiceName()), 1000));
10333 // When we get the first update, all backends in priority 0 are down,
10334 // so we will create priority 1. Backends 1 and 2 should have traffic,
10335 // but backend 3 should not.
10336 WaitForAllBackends(1, 3, WaitForBackendOptions().set_reset_counters(false));
10337 EXPECT_EQ(0UL, backends_[3]->backend_service()->request_count());
10338 // When backend 3 gets traffic, we know the second update has been seen.
10339 WaitForBackend(3);
10340 // The ADS service of balancer 0 got at least 1 response.
10341 EXPECT_GT(balancers_[0]->ads_service()->eds_response_state().state,
10342 AdsServiceImpl::ResponseState::NOT_SENT);
10343 delayed_resource_setter.join();
10344 }
10345
10346 using DropTest = BasicTest;
10347
10348 // Tests that RPCs are dropped according to the drop config.
TEST_P(DropTest,Vanilla)10349 TEST_P(DropTest, Vanilla) {
10350 SetNextResolution({});
10351 SetNextResolutionForLbChannelAllBalancers();
10352 const uint32_t kDropPerMillionForLb = 100000;
10353 const uint32_t kDropPerMillionForThrottle = 200000;
10354 const double kDropRateForLb = kDropPerMillionForLb / 1000000.0;
10355 const double kDropRateForThrottle = kDropPerMillionForThrottle / 1000000.0;
10356 const double kDropRateForLbAndThrottle =
10357 kDropRateForLb + (1 - kDropRateForLb) * kDropRateForThrottle;
10358 const double kErrorTolerance = 0.05;
10359 const size_t kNumRpcs =
10360 ComputeIdealNumRpcs(kDropRateForLbAndThrottle, kErrorTolerance);
10361 // The ADS response contains two drop categories.
10362 EdsResourceArgs args({
10363 {"locality0", CreateEndpointsForBackends()},
10364 });
10365 args.drop_categories = {{kLbDropType, kDropPerMillionForLb},
10366 {kThrottleDropType, kDropPerMillionForThrottle}};
10367 balancers_[0]->ads_service()->SetEdsResource(
10368 BuildEdsResource(args, DefaultEdsServiceName()));
10369 WaitForAllBackends();
10370 // Send kNumRpcs RPCs and count the drops.
10371 size_t num_drops = 0;
10372 for (size_t i = 0; i < kNumRpcs; ++i) {
10373 EchoResponse response;
10374 const Status status = SendRpc(RpcOptions(), &response);
10375 if (!status.ok() &&
10376 absl::StartsWith(status.error_message(), "EDS-configured drop: ")) {
10377 ++num_drops;
10378 } else {
10379 EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
10380 << " message=" << status.error_message();
10381 EXPECT_EQ(response.message(), kRequestMessage);
10382 }
10383 }
10384 // The drop rate should be roughly equal to the expectation.
10385 const double seen_drop_rate = static_cast<double>(num_drops) / kNumRpcs;
10386 EXPECT_THAT(seen_drop_rate, ::testing::DoubleNear(kDropRateForLbAndThrottle,
10387 kErrorTolerance));
10388 }
10389
10390 // Tests that drop config is converted correctly from per hundred.
TEST_P(DropTest,DropPerHundred)10391 TEST_P(DropTest, DropPerHundred) {
10392 SetNextResolution({});
10393 SetNextResolutionForLbChannelAllBalancers();
10394 const uint32_t kDropPerHundredForLb = 10;
10395 const double kDropRateForLb = kDropPerHundredForLb / 100.0;
10396 const double kErrorTolerance = 0.05;
10397 const size_t kNumRpcs = ComputeIdealNumRpcs(kDropRateForLb, kErrorTolerance);
10398 // The ADS response contains one drop category.
10399 EdsResourceArgs args({
10400 {"locality0", CreateEndpointsForBackends()},
10401 });
10402 args.drop_categories = {{kLbDropType, kDropPerHundredForLb}};
10403 args.drop_denominator = FractionalPercent::HUNDRED;
10404 balancers_[0]->ads_service()->SetEdsResource(
10405 BuildEdsResource(args, DefaultEdsServiceName()));
10406 WaitForAllBackends();
10407 // Send kNumRpcs RPCs and count the drops.
10408 size_t num_drops = 0;
10409 for (size_t i = 0; i < kNumRpcs; ++i) {
10410 EchoResponse response;
10411 const Status status = SendRpc(RpcOptions(), &response);
10412 if (!status.ok() &&
10413 absl::StartsWith(status.error_message(), "EDS-configured drop: ")) {
10414 ++num_drops;
10415 } else {
10416 EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
10417 << " message=" << status.error_message();
10418 EXPECT_EQ(response.message(), kRequestMessage);
10419 }
10420 }
10421 // The drop rate should be roughly equal to the expectation.
10422 const double seen_drop_rate = static_cast<double>(num_drops) / kNumRpcs;
10423 EXPECT_THAT(seen_drop_rate,
10424 ::testing::DoubleNear(kDropRateForLb, kErrorTolerance));
10425 }
10426
10427 // Tests that drop config is converted correctly from per ten thousand.
TEST_P(DropTest,DropPerTenThousand)10428 TEST_P(DropTest, DropPerTenThousand) {
10429 SetNextResolution({});
10430 SetNextResolutionForLbChannelAllBalancers();
10431 const uint32_t kDropPerTenThousandForLb = 1000;
10432 const double kDropRateForLb = kDropPerTenThousandForLb / 10000.0;
10433 const double kErrorTolerance = 0.05;
10434 const size_t kNumRpcs = ComputeIdealNumRpcs(kDropRateForLb, kErrorTolerance);
10435 // The ADS response contains one drop category.
10436 EdsResourceArgs args({
10437 {"locality0", CreateEndpointsForBackends()},
10438 });
10439 args.drop_categories = {{kLbDropType, kDropPerTenThousandForLb}};
10440 args.drop_denominator = FractionalPercent::TEN_THOUSAND;
10441 balancers_[0]->ads_service()->SetEdsResource(
10442 BuildEdsResource(args, DefaultEdsServiceName()));
10443 WaitForAllBackends();
10444 // Send kNumRpcs RPCs and count the drops.
10445 size_t num_drops = 0;
10446 for (size_t i = 0; i < kNumRpcs; ++i) {
10447 EchoResponse response;
10448 const Status status = SendRpc(RpcOptions(), &response);
10449 if (!status.ok() &&
10450 absl::StartsWith(status.error_message(), "EDS-configured drop: ")) {
10451 ++num_drops;
10452 } else {
10453 EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
10454 << " message=" << status.error_message();
10455 EXPECT_EQ(response.message(), kRequestMessage);
10456 }
10457 }
10458 // The drop rate should be roughly equal to the expectation.
10459 const double seen_drop_rate = static_cast<double>(num_drops) / kNumRpcs;
10460 EXPECT_THAT(seen_drop_rate,
10461 ::testing::DoubleNear(kDropRateForLb, kErrorTolerance));
10462 }
10463
10464 // Tests that drop is working correctly after update.
TEST_P(DropTest,Update)10465 TEST_P(DropTest, Update) {
10466 SetNextResolution({});
10467 SetNextResolutionForLbChannelAllBalancers();
10468 const uint32_t kDropPerMillionForLb = 100000;
10469 const uint32_t kDropPerMillionForThrottle = 200000;
10470 const double kErrorTolerance = 0.05;
10471 const double kDropRateForLb = kDropPerMillionForLb / 1000000.0;
10472 const double kDropRateForThrottle = kDropPerMillionForThrottle / 1000000.0;
10473 const double kDropRateForLbAndThrottle =
10474 kDropRateForLb + (1 - kDropRateForLb) * kDropRateForThrottle;
10475 const size_t kNumRpcsLbOnly =
10476 ComputeIdealNumRpcs(kDropRateForLb, kErrorTolerance);
10477 const size_t kNumRpcsBoth =
10478 ComputeIdealNumRpcs(kDropRateForLbAndThrottle, kErrorTolerance);
10479 // The first ADS response contains one drop category.
10480 EdsResourceArgs args({
10481 {"locality0", CreateEndpointsForBackends()},
10482 });
10483 args.drop_categories = {{kLbDropType, kDropPerMillionForLb}};
10484 balancers_[0]->ads_service()->SetEdsResource(
10485 BuildEdsResource(args, DefaultEdsServiceName()));
10486 WaitForAllBackends();
10487 // Send kNumRpcsLbOnly RPCs and count the drops.
10488 size_t num_drops = 0;
10489 gpr_log(GPR_INFO, "========= BEFORE FIRST BATCH ==========");
10490 for (size_t i = 0; i < kNumRpcsLbOnly; ++i) {
10491 EchoResponse response;
10492 const Status status = SendRpc(RpcOptions(), &response);
10493 if (!status.ok() &&
10494 absl::StartsWith(status.error_message(), "EDS-configured drop: ")) {
10495 ++num_drops;
10496 } else {
10497 EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
10498 << " message=" << status.error_message();
10499 EXPECT_EQ(response.message(), kRequestMessage);
10500 }
10501 }
10502 gpr_log(GPR_INFO, "========= DONE WITH FIRST BATCH ==========");
10503 // The drop rate should be roughly equal to the expectation.
10504 double seen_drop_rate = static_cast<double>(num_drops) / kNumRpcsLbOnly;
10505 gpr_log(GPR_INFO, "First batch drop rate %f", seen_drop_rate);
10506 EXPECT_THAT(seen_drop_rate,
10507 ::testing::DoubleNear(kDropRateForLb, kErrorTolerance));
10508 // The second ADS response contains two drop categories, send an update EDS
10509 // response.
10510 args.drop_categories = {{kLbDropType, kDropPerMillionForLb},
10511 {kThrottleDropType, kDropPerMillionForThrottle}};
10512 balancers_[0]->ads_service()->SetEdsResource(
10513 BuildEdsResource(args, DefaultEdsServiceName()));
10514 // Wait until the drop rate increases to the middle of the two configs,
10515 // which implies that the update has been in effect.
10516 const double kDropRateThreshold =
10517 (kDropRateForLb + kDropRateForLbAndThrottle) / 2;
10518 size_t num_rpcs = kNumRpcsBoth;
10519 while (seen_drop_rate < kDropRateThreshold) {
10520 EchoResponse response;
10521 const Status status = SendRpc(RpcOptions(), &response);
10522 ++num_rpcs;
10523 if (!status.ok() &&
10524 absl::StartsWith(status.error_message(), "EDS-configured drop: ")) {
10525 ++num_drops;
10526 } else {
10527 EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
10528 << " message=" << status.error_message();
10529 EXPECT_EQ(response.message(), kRequestMessage);
10530 }
10531 seen_drop_rate = static_cast<double>(num_drops) / num_rpcs;
10532 }
10533 // Send kNumRpcsBoth RPCs and count the drops.
10534 num_drops = 0;
10535 gpr_log(GPR_INFO, "========= BEFORE SECOND BATCH ==========");
10536 for (size_t i = 0; i < kNumRpcsBoth; ++i) {
10537 EchoResponse response;
10538 const Status status = SendRpc(RpcOptions(), &response);
10539 if (!status.ok() &&
10540 absl::StartsWith(status.error_message(), "EDS-configured drop: ")) {
10541 ++num_drops;
10542 } else {
10543 EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
10544 << " message=" << status.error_message();
10545 EXPECT_EQ(response.message(), kRequestMessage);
10546 }
10547 }
10548 gpr_log(GPR_INFO, "========= DONE WITH SECOND BATCH ==========");
10549 // The new drop rate should be roughly equal to the expectation.
10550 seen_drop_rate = static_cast<double>(num_drops) / kNumRpcsBoth;
10551 gpr_log(GPR_INFO, "Second batch drop rate %f", seen_drop_rate);
10552 EXPECT_THAT(seen_drop_rate, ::testing::DoubleNear(kDropRateForLbAndThrottle,
10553 kErrorTolerance));
10554 }
10555
10556 // Tests that all the RPCs are dropped if any drop category drops 100%.
TEST_P(DropTest,DropAll)10557 TEST_P(DropTest, DropAll) {
10558 SetNextResolution({});
10559 SetNextResolutionForLbChannelAllBalancers();
10560 const size_t kNumRpcs = 1000;
10561 const uint32_t kDropPerMillionForLb = 100000;
10562 const uint32_t kDropPerMillionForThrottle = 1000000;
10563 // The ADS response contains two drop categories.
10564 EdsResourceArgs args;
10565 args.drop_categories = {{kLbDropType, kDropPerMillionForLb},
10566 {kThrottleDropType, kDropPerMillionForThrottle}};
10567 balancers_[0]->ads_service()->SetEdsResource(
10568 BuildEdsResource(args, DefaultEdsServiceName()));
10569 // Send kNumRpcs RPCs and all of them are dropped.
10570 for (size_t i = 0; i < kNumRpcs; ++i) {
10571 EchoResponse response;
10572 const Status status = SendRpc(RpcOptions(), &response);
10573 EXPECT_EQ(status.error_code(), StatusCode::UNAVAILABLE);
10574 EXPECT_THAT(status.error_message(),
10575 ::testing::StartsWith("EDS-configured drop: "));
10576 }
10577 }
10578
10579 class BalancerUpdateTest : public XdsEnd2endTest {
10580 public:
BalancerUpdateTest()10581 BalancerUpdateTest() : XdsEnd2endTest(4, 3) {}
10582 };
10583
10584 // Tests that the old LB call is still used after the balancer address update
10585 // as long as that call is still alive.
TEST_P(BalancerUpdateTest,UpdateBalancersButKeepUsingOriginalBalancer)10586 TEST_P(BalancerUpdateTest, UpdateBalancersButKeepUsingOriginalBalancer) {
10587 SetNextResolution({});
10588 SetNextResolutionForLbChannelAllBalancers();
10589 EdsResourceArgs args({{"locality0", CreateEndpointsForBackends(0, 1)}});
10590 balancers_[0]->ads_service()->SetEdsResource(
10591 BuildEdsResource(args, DefaultEdsServiceName()));
10592 args = EdsResourceArgs({{"locality0", CreateEndpointsForBackends(1, 2)}});
10593 balancers_[1]->ads_service()->SetEdsResource(
10594 BuildEdsResource(args, DefaultEdsServiceName()));
10595 // Wait until the first backend is ready.
10596 WaitForBackend(0);
10597 // Send 10 requests.
10598 gpr_log(GPR_INFO, "========= BEFORE FIRST BATCH ==========");
10599 CheckRpcSendOk(10);
10600 gpr_log(GPR_INFO, "========= DONE WITH FIRST BATCH ==========");
10601 // All 10 requests should have gone to the first backend.
10602 EXPECT_EQ(10U, backends_[0]->backend_service()->request_count());
10603 // The ADS service of balancer 0 sent at least 1 response.
10604 EXPECT_GT(balancers_[0]->ads_service()->eds_response_state().state,
10605 AdsServiceImpl::ResponseState::NOT_SENT);
10606 EXPECT_EQ(balancers_[1]->ads_service()->eds_response_state().state,
10607 AdsServiceImpl::ResponseState::NOT_SENT)
10608 << "Error Message:"
10609 << balancers_[1]->ads_service()->eds_response_state().error_message;
10610 EXPECT_EQ(balancers_[2]->ads_service()->eds_response_state().state,
10611 AdsServiceImpl::ResponseState::NOT_SENT)
10612 << "Error Message:"
10613 << balancers_[2]->ads_service()->eds_response_state().error_message;
10614 gpr_log(GPR_INFO, "========= ABOUT TO UPDATE 1 ==========");
10615 SetNextResolutionForLbChannel({balancers_[1]->port()});
10616 gpr_log(GPR_INFO, "========= UPDATE 1 DONE ==========");
10617 EXPECT_EQ(0U, backends_[1]->backend_service()->request_count());
10618 gpr_timespec deadline = gpr_time_add(
10619 gpr_now(GPR_CLOCK_REALTIME), gpr_time_from_millis(10000, GPR_TIMESPAN));
10620 // Send 10 seconds worth of RPCs
10621 do {
10622 CheckRpcSendOk();
10623 } while (gpr_time_cmp(gpr_now(GPR_CLOCK_REALTIME), deadline) < 0);
10624 // The current LB call is still working, so xds continued using it to the
10625 // first balancer, which doesn't assign the second backend.
10626 EXPECT_EQ(0U, backends_[1]->backend_service()->request_count());
10627 // The ADS service of balancer 0 sent at least 1 response.
10628 EXPECT_GT(balancers_[0]->ads_service()->eds_response_state().state,
10629 AdsServiceImpl::ResponseState::NOT_SENT);
10630 EXPECT_EQ(balancers_[1]->ads_service()->eds_response_state().state,
10631 AdsServiceImpl::ResponseState::NOT_SENT)
10632 << "Error Message:"
10633 << balancers_[1]->ads_service()->eds_response_state().error_message;
10634 EXPECT_EQ(balancers_[2]->ads_service()->eds_response_state().state,
10635 AdsServiceImpl::ResponseState::NOT_SENT)
10636 << "Error Message:"
10637 << balancers_[2]->ads_service()->eds_response_state().error_message;
10638 }
10639
10640 // Tests that the old LB call is still used after multiple balancer address
10641 // updates as long as that call is still alive. Send an update with the same
10642 // set of LBs as the one in SetUp() in order to verify that the LB channel
10643 // inside xds keeps the initial connection (which by definition is also
10644 // present in the update).
TEST_P(BalancerUpdateTest,Repeated)10645 TEST_P(BalancerUpdateTest, Repeated) {
10646 SetNextResolution({});
10647 SetNextResolutionForLbChannelAllBalancers();
10648 EdsResourceArgs args({{"locality0", CreateEndpointsForBackends(0, 1)}});
10649 balancers_[0]->ads_service()->SetEdsResource(
10650 BuildEdsResource(args, DefaultEdsServiceName()));
10651 args = EdsResourceArgs({{"locality0", CreateEndpointsForBackends(1, 2)}});
10652 balancers_[1]->ads_service()->SetEdsResource(
10653 BuildEdsResource(args, DefaultEdsServiceName()));
10654 // Wait until the first backend is ready.
10655 WaitForBackend(0);
10656 // Send 10 requests.
10657 gpr_log(GPR_INFO, "========= BEFORE FIRST BATCH ==========");
10658 CheckRpcSendOk(10);
10659 gpr_log(GPR_INFO, "========= DONE WITH FIRST BATCH ==========");
10660 // All 10 requests should have gone to the first backend.
10661 EXPECT_EQ(10U, backends_[0]->backend_service()->request_count());
10662 // The ADS service of balancer 0 sent at least 1 response.
10663 EXPECT_GT(balancers_[0]->ads_service()->eds_response_state().state,
10664 AdsServiceImpl::ResponseState::NOT_SENT);
10665 EXPECT_EQ(balancers_[1]->ads_service()->eds_response_state().state,
10666 AdsServiceImpl::ResponseState::NOT_SENT)
10667 << "Error Message:"
10668 << balancers_[1]->ads_service()->eds_response_state().error_message;
10669 EXPECT_EQ(balancers_[2]->ads_service()->eds_response_state().state,
10670 AdsServiceImpl::ResponseState::NOT_SENT)
10671 << "Error Message:"
10672 << balancers_[2]->ads_service()->eds_response_state().error_message;
10673 std::vector<int> ports;
10674 ports.emplace_back(balancers_[0]->port());
10675 ports.emplace_back(balancers_[1]->port());
10676 ports.emplace_back(balancers_[2]->port());
10677 gpr_log(GPR_INFO, "========= ABOUT TO UPDATE 1 ==========");
10678 SetNextResolutionForLbChannel(ports);
10679 gpr_log(GPR_INFO, "========= UPDATE 1 DONE ==========");
10680 EXPECT_EQ(0U, backends_[1]->backend_service()->request_count());
10681 gpr_timespec deadline = gpr_time_add(
10682 gpr_now(GPR_CLOCK_REALTIME), gpr_time_from_millis(10000, GPR_TIMESPAN));
10683 // Send 10 seconds worth of RPCs
10684 do {
10685 CheckRpcSendOk();
10686 } while (gpr_time_cmp(gpr_now(GPR_CLOCK_REALTIME), deadline) < 0);
10687 // xds continued using the original LB call to the first balancer, which
10688 // doesn't assign the second backend.
10689 EXPECT_EQ(0U, backends_[1]->backend_service()->request_count());
10690 ports.clear();
10691 ports.emplace_back(balancers_[0]->port());
10692 ports.emplace_back(balancers_[1]->port());
10693 gpr_log(GPR_INFO, "========= ABOUT TO UPDATE 2 ==========");
10694 SetNextResolutionForLbChannel(ports);
10695 gpr_log(GPR_INFO, "========= UPDATE 2 DONE ==========");
10696 EXPECT_EQ(0U, backends_[1]->backend_service()->request_count());
10697 deadline = gpr_time_add(gpr_now(GPR_CLOCK_REALTIME),
10698 gpr_time_from_millis(10000, GPR_TIMESPAN));
10699 // Send 10 seconds worth of RPCs
10700 do {
10701 CheckRpcSendOk();
10702 } while (gpr_time_cmp(gpr_now(GPR_CLOCK_REALTIME), deadline) < 0);
10703 // xds continued using the original LB call to the first balancer, which
10704 // doesn't assign the second backend.
10705 EXPECT_EQ(0U, backends_[1]->backend_service()->request_count());
10706 }
10707
10708 // Tests that if the balancer is down, the RPCs will still be sent to the
10709 // backends according to the last balancer response, until a new balancer is
10710 // reachable.
TEST_P(BalancerUpdateTest,DeadUpdate)10711 TEST_P(BalancerUpdateTest, DeadUpdate) {
10712 SetNextResolution({});
10713 SetNextResolutionForLbChannel({balancers_[0]->port()});
10714 EdsResourceArgs args({{"locality0", CreateEndpointsForBackends(0, 1)}});
10715 balancers_[0]->ads_service()->SetEdsResource(
10716 BuildEdsResource(args, DefaultEdsServiceName()));
10717 args = EdsResourceArgs({{"locality0", CreateEndpointsForBackends(1, 2)}});
10718 balancers_[1]->ads_service()->SetEdsResource(
10719 BuildEdsResource(args, DefaultEdsServiceName()));
10720 // Start servers and send 10 RPCs per server.
10721 gpr_log(GPR_INFO, "========= BEFORE FIRST BATCH ==========");
10722 CheckRpcSendOk(10);
10723 gpr_log(GPR_INFO, "========= DONE WITH FIRST BATCH ==========");
10724 // All 10 requests should have gone to the first backend.
10725 EXPECT_EQ(10U, backends_[0]->backend_service()->request_count());
10726 // The ADS service of balancer 0 sent at least 1 response.
10727 EXPECT_GT(balancers_[0]->ads_service()->eds_response_state().state,
10728 AdsServiceImpl::ResponseState::NOT_SENT);
10729 EXPECT_EQ(balancers_[1]->ads_service()->eds_response_state().state,
10730 AdsServiceImpl::ResponseState::NOT_SENT)
10731 << "Error Message:"
10732 << balancers_[1]->ads_service()->eds_response_state().error_message;
10733 EXPECT_EQ(balancers_[2]->ads_service()->eds_response_state().state,
10734 AdsServiceImpl::ResponseState::NOT_SENT)
10735 << "Error Message:"
10736 << balancers_[2]->ads_service()->eds_response_state().error_message;
10737 // Kill balancer 0
10738 gpr_log(GPR_INFO, "********** ABOUT TO KILL BALANCER 0 *************");
10739 balancers_[0]->Shutdown();
10740 gpr_log(GPR_INFO, "********** KILLED BALANCER 0 *************");
10741 // This is serviced by the existing child policy.
10742 gpr_log(GPR_INFO, "========= BEFORE SECOND BATCH ==========");
10743 CheckRpcSendOk(10);
10744 gpr_log(GPR_INFO, "========= DONE WITH SECOND BATCH ==========");
10745 // All 10 requests should again have gone to the first backend.
10746 EXPECT_EQ(20U, backends_[0]->backend_service()->request_count());
10747 EXPECT_EQ(0U, backends_[1]->backend_service()->request_count());
10748 // The ADS service of no balancers sent anything
10749 EXPECT_EQ(balancers_[0]->ads_service()->eds_response_state().state,
10750 AdsServiceImpl::ResponseState::NOT_SENT)
10751 << "Error Message:"
10752 << balancers_[0]->ads_service()->eds_response_state().error_message;
10753 EXPECT_EQ(balancers_[1]->ads_service()->eds_response_state().state,
10754 AdsServiceImpl::ResponseState::NOT_SENT)
10755 << "Error Message:"
10756 << balancers_[1]->ads_service()->eds_response_state().error_message;
10757 EXPECT_EQ(balancers_[2]->ads_service()->eds_response_state().state,
10758 AdsServiceImpl::ResponseState::NOT_SENT)
10759 << "Error Message:"
10760 << balancers_[2]->ads_service()->eds_response_state().error_message;
10761 gpr_log(GPR_INFO, "========= ABOUT TO UPDATE 1 ==========");
10762 SetNextResolutionForLbChannel({balancers_[1]->port()});
10763 gpr_log(GPR_INFO, "========= UPDATE 1 DONE ==========");
10764 // Wait until update has been processed, as signaled by the second backend
10765 // receiving a request. In the meantime, the client continues to be serviced
10766 // (by the first backend) without interruption.
10767 EXPECT_EQ(0U, backends_[1]->backend_service()->request_count());
10768 WaitForBackend(1);
10769 // This is serviced by the updated RR policy
10770 backends_[1]->backend_service()->ResetCounters();
10771 gpr_log(GPR_INFO, "========= BEFORE THIRD BATCH ==========");
10772 CheckRpcSendOk(10);
10773 gpr_log(GPR_INFO, "========= DONE WITH THIRD BATCH ==========");
10774 // All 10 requests should have gone to the second backend.
10775 EXPECT_EQ(10U, backends_[1]->backend_service()->request_count());
10776 // The ADS service of balancer 1 sent at least 1 response.
10777 EXPECT_EQ(balancers_[0]->ads_service()->eds_response_state().state,
10778 AdsServiceImpl::ResponseState::NOT_SENT)
10779 << "Error Message:"
10780 << balancers_[0]->ads_service()->eds_response_state().error_message;
10781 EXPECT_GT(balancers_[1]->ads_service()->eds_response_state().state,
10782 AdsServiceImpl::ResponseState::NOT_SENT);
10783 EXPECT_EQ(balancers_[2]->ads_service()->eds_response_state().state,
10784 AdsServiceImpl::ResponseState::NOT_SENT)
10785 << "Error Message:"
10786 << balancers_[2]->ads_service()->eds_response_state().error_message;
10787 }
10788
10789 class ClientLoadReportingTest : public XdsEnd2endTest {
10790 public:
ClientLoadReportingTest()10791 ClientLoadReportingTest() : XdsEnd2endTest(4, 1, 3) {}
10792 };
10793
10794 // Tests that the load report received at the balancer is correct.
TEST_P(ClientLoadReportingTest,Vanilla)10795 TEST_P(ClientLoadReportingTest, Vanilla) {
10796 if (GetParam().use_fake_resolver()) {
10797 balancers_[0]->lrs_service()->set_cluster_names({kServerName});
10798 }
10799 SetNextResolution({});
10800 SetNextResolutionForLbChannel({balancers_[0]->port()});
10801 const size_t kNumRpcsPerAddress = 10;
10802 const size_t kNumFailuresPerAddress = 3;
10803 // TODO(juanlishen): Partition the backends after multiple localities is
10804 // tested.
10805 EdsResourceArgs args({
10806 {"locality0", CreateEndpointsForBackends()},
10807 });
10808 balancers_[0]->ads_service()->SetEdsResource(
10809 BuildEdsResource(args, DefaultEdsServiceName()));
10810 // Wait until all backends are ready.
10811 int num_ok = 0;
10812 int num_failure = 0;
10813 int num_drops = 0;
10814 std::tie(num_ok, num_failure, num_drops) = WaitForAllBackends();
10815 // Send kNumRpcsPerAddress RPCs per server.
10816 CheckRpcSendOk(kNumRpcsPerAddress * num_backends_);
10817 CheckRpcSendFailure(CheckRpcSendFailureOptions()
10818 .set_times(kNumFailuresPerAddress * num_backends_)
10819 .set_rpc_options(RpcOptions().set_server_fail(true)));
10820 // Check that each backend got the right number of requests.
10821 for (size_t i = 0; i < backends_.size(); ++i) {
10822 EXPECT_EQ(kNumRpcsPerAddress + kNumFailuresPerAddress,
10823 backends_[i]->backend_service()->request_count());
10824 }
10825 // The load report received at the balancer should be correct.
10826 std::vector<ClientStats> load_report =
10827 balancers_[0]->lrs_service()->WaitForLoadReport();
10828 ASSERT_EQ(load_report.size(), 1UL);
10829 ClientStats& client_stats = load_report.front();
10830 EXPECT_EQ(kNumRpcsPerAddress * num_backends_ + num_ok,
10831 client_stats.total_successful_requests());
10832 EXPECT_EQ(0U, client_stats.total_requests_in_progress());
10833 EXPECT_EQ((kNumRpcsPerAddress + kNumFailuresPerAddress) * num_backends_ +
10834 num_ok + num_failure,
10835 client_stats.total_issued_requests());
10836 EXPECT_EQ(kNumFailuresPerAddress * num_backends_ + num_failure,
10837 client_stats.total_error_requests());
10838 EXPECT_EQ(0U, client_stats.total_dropped_requests());
10839 // The LRS service got a single request, and sent a single response.
10840 EXPECT_EQ(1U, balancers_[0]->lrs_service()->request_count());
10841 EXPECT_EQ(1U, balancers_[0]->lrs_service()->response_count());
10842 }
10843
10844 // Tests send_all_clusters.
TEST_P(ClientLoadReportingTest,SendAllClusters)10845 TEST_P(ClientLoadReportingTest, SendAllClusters) {
10846 balancers_[0]->lrs_service()->set_send_all_clusters(true);
10847 SetNextResolution({});
10848 SetNextResolutionForLbChannel({balancers_[0]->port()});
10849 const size_t kNumRpcsPerAddress = 10;
10850 const size_t kNumFailuresPerAddress = 3;
10851 // TODO(juanlishen): Partition the backends after multiple localities is
10852 // tested.
10853 EdsResourceArgs args({
10854 {"locality0", CreateEndpointsForBackends()},
10855 });
10856 balancers_[0]->ads_service()->SetEdsResource(
10857 BuildEdsResource(args, DefaultEdsServiceName()));
10858 // Wait until all backends are ready.
10859 int num_ok = 0;
10860 int num_failure = 0;
10861 int num_drops = 0;
10862 std::tie(num_ok, num_failure, num_drops) = WaitForAllBackends();
10863 // Send kNumRpcsPerAddress RPCs per server.
10864 CheckRpcSendOk(kNumRpcsPerAddress * num_backends_);
10865 CheckRpcSendFailure(CheckRpcSendFailureOptions()
10866 .set_times(kNumFailuresPerAddress * num_backends_)
10867 .set_rpc_options(RpcOptions().set_server_fail(true)));
10868 // Check that each backend got the right number of requests.
10869 for (size_t i = 0; i < backends_.size(); ++i) {
10870 EXPECT_EQ(kNumRpcsPerAddress + kNumFailuresPerAddress,
10871 backends_[i]->backend_service()->request_count());
10872 }
10873 // The load report received at the balancer should be correct.
10874 std::vector<ClientStats> load_report =
10875 balancers_[0]->lrs_service()->WaitForLoadReport();
10876 ASSERT_EQ(load_report.size(), 1UL);
10877 ClientStats& client_stats = load_report.front();
10878 EXPECT_EQ(kNumRpcsPerAddress * num_backends_ + num_ok,
10879 client_stats.total_successful_requests());
10880 EXPECT_EQ(0U, client_stats.total_requests_in_progress());
10881 EXPECT_EQ((kNumRpcsPerAddress + kNumFailuresPerAddress) * num_backends_ +
10882 num_ok + num_failure,
10883 client_stats.total_issued_requests());
10884 EXPECT_EQ(kNumFailuresPerAddress * num_backends_ + num_failure,
10885 client_stats.total_error_requests());
10886 EXPECT_EQ(0U, client_stats.total_dropped_requests());
10887 // The LRS service got a single request, and sent a single response.
10888 EXPECT_EQ(1U, balancers_[0]->lrs_service()->request_count());
10889 EXPECT_EQ(1U, balancers_[0]->lrs_service()->response_count());
10890 }
10891
10892 // Tests that we don't include stats for clusters that are not requested
10893 // by the LRS server.
TEST_P(ClientLoadReportingTest,HonorsClustersRequestedByLrsServer)10894 TEST_P(ClientLoadReportingTest, HonorsClustersRequestedByLrsServer) {
10895 balancers_[0]->lrs_service()->set_cluster_names({"bogus"});
10896 SetNextResolution({});
10897 SetNextResolutionForLbChannel({balancers_[0]->port()});
10898 const size_t kNumRpcsPerAddress = 100;
10899 EdsResourceArgs args({
10900 {"locality0", CreateEndpointsForBackends()},
10901 });
10902 balancers_[0]->ads_service()->SetEdsResource(
10903 BuildEdsResource(args, DefaultEdsServiceName()));
10904 // Wait until all backends are ready.
10905 int num_ok = 0;
10906 int num_failure = 0;
10907 int num_drops = 0;
10908 std::tie(num_ok, num_failure, num_drops) = WaitForAllBackends();
10909 // Send kNumRpcsPerAddress RPCs per server.
10910 CheckRpcSendOk(kNumRpcsPerAddress * num_backends_);
10911 // Each backend should have gotten 100 requests.
10912 for (size_t i = 0; i < backends_.size(); ++i) {
10913 EXPECT_EQ(kNumRpcsPerAddress,
10914 backends_[i]->backend_service()->request_count());
10915 }
10916 // The LRS service got a single request, and sent a single response.
10917 EXPECT_EQ(1U, balancers_[0]->lrs_service()->request_count());
10918 EXPECT_EQ(1U, balancers_[0]->lrs_service()->response_count());
10919 // The load report received at the balancer should be correct.
10920 std::vector<ClientStats> load_report =
10921 balancers_[0]->lrs_service()->WaitForLoadReport();
10922 ASSERT_EQ(load_report.size(), 0UL);
10923 }
10924
10925 // Tests that if the balancer restarts, the client load report contains the
10926 // stats before and after the restart correctly.
TEST_P(ClientLoadReportingTest,BalancerRestart)10927 TEST_P(ClientLoadReportingTest, BalancerRestart) {
10928 if (GetParam().use_fake_resolver()) {
10929 balancers_[0]->lrs_service()->set_cluster_names({kServerName});
10930 }
10931 SetNextResolution({});
10932 SetNextResolutionForLbChannel({balancers_[0]->port()});
10933 const size_t kNumBackendsFirstPass = backends_.size() / 2;
10934 const size_t kNumBackendsSecondPass =
10935 backends_.size() - kNumBackendsFirstPass;
10936 EdsResourceArgs args({
10937 {"locality0", CreateEndpointsForBackends(0, kNumBackendsFirstPass)},
10938 });
10939 balancers_[0]->ads_service()->SetEdsResource(
10940 BuildEdsResource(args, DefaultEdsServiceName()));
10941 // Wait until all backends returned by the balancer are ready.
10942 int num_ok = 0;
10943 int num_failure = 0;
10944 int num_drops = 0;
10945 std::tie(num_ok, num_failure, num_drops) =
10946 WaitForAllBackends(/* start_index */ 0,
10947 /* stop_index */ kNumBackendsFirstPass);
10948 std::vector<ClientStats> load_report =
10949 balancers_[0]->lrs_service()->WaitForLoadReport();
10950 ASSERT_EQ(load_report.size(), 1UL);
10951 ClientStats client_stats = std::move(load_report.front());
10952 EXPECT_EQ(static_cast<size_t>(num_ok),
10953 client_stats.total_successful_requests());
10954 EXPECT_EQ(0U, client_stats.total_requests_in_progress());
10955 EXPECT_EQ(0U, client_stats.total_error_requests());
10956 EXPECT_EQ(0U, client_stats.total_dropped_requests());
10957 // Shut down the balancer.
10958 balancers_[0]->Shutdown();
10959 // We should continue using the last EDS response we received from the
10960 // balancer before it was shut down.
10961 // Note: We need to use WaitForAllBackends() here instead of just
10962 // CheckRpcSendOk(kNumBackendsFirstPass), because when the balancer
10963 // shuts down, the XdsClient will generate an error to the
10964 // ServiceConfigWatcher, which will cause the xds resolver to send a
10965 // no-op update to the LB policy. When this update gets down to the
10966 // round_robin child policy for the locality, it will generate a new
10967 // subchannel list, which resets the start index randomly. So we need
10968 // to be a little more permissive here to avoid spurious failures.
10969 ResetBackendCounters();
10970 int num_started = std::get<0>(WaitForAllBackends(
10971 /* start_index */ 0, /* stop_index */ kNumBackendsFirstPass));
10972 // Now restart the balancer, this time pointing to the new backends.
10973 balancers_[0]->Start();
10974 args = EdsResourceArgs({
10975 {"locality0", CreateEndpointsForBackends(kNumBackendsFirstPass)},
10976 });
10977 balancers_[0]->ads_service()->SetEdsResource(
10978 BuildEdsResource(args, DefaultEdsServiceName()));
10979 // Wait for queries to start going to one of the new backends.
10980 // This tells us that we're now using the new serverlist.
10981 std::tie(num_ok, num_failure, num_drops) =
10982 WaitForAllBackends(/* start_index */ kNumBackendsFirstPass);
10983 num_started += num_ok + num_failure + num_drops;
10984 // Send one RPC per backend.
10985 CheckRpcSendOk(kNumBackendsSecondPass);
10986 num_started += kNumBackendsSecondPass;
10987 // Check client stats.
10988 load_report = balancers_[0]->lrs_service()->WaitForLoadReport();
10989 ASSERT_EQ(load_report.size(), 1UL);
10990 client_stats = std::move(load_report.front());
10991 EXPECT_EQ(num_started, client_stats.total_successful_requests());
10992 EXPECT_EQ(0U, client_stats.total_requests_in_progress());
10993 EXPECT_EQ(0U, client_stats.total_error_requests());
10994 EXPECT_EQ(0U, client_stats.total_dropped_requests());
10995 }
10996
10997 class ClientLoadReportingWithDropTest : public XdsEnd2endTest {
10998 public:
ClientLoadReportingWithDropTest()10999 ClientLoadReportingWithDropTest() : XdsEnd2endTest(4, 1, 20) {}
11000 };
11001
11002 // Tests that the drop stats are correctly reported by client load reporting.
TEST_P(ClientLoadReportingWithDropTest,Vanilla)11003 TEST_P(ClientLoadReportingWithDropTest, Vanilla) {
11004 if (GetParam().use_fake_resolver()) {
11005 balancers_[0]->lrs_service()->set_cluster_names({kServerName});
11006 }
11007 SetNextResolution({});
11008 SetNextResolutionForLbChannelAllBalancers();
11009 const uint32_t kDropPerMillionForLb = 100000;
11010 const uint32_t kDropPerMillionForThrottle = 200000;
11011 const double kErrorTolerance = 0.05;
11012 const double kDropRateForLb = kDropPerMillionForLb / 1000000.0;
11013 const double kDropRateForThrottle = kDropPerMillionForThrottle / 1000000.0;
11014 const double kDropRateForLbAndThrottle =
11015 kDropRateForLb + (1 - kDropRateForLb) * kDropRateForThrottle;
11016 const size_t kNumRpcs =
11017 ComputeIdealNumRpcs(kDropRateForLbAndThrottle, kErrorTolerance);
11018 // The ADS response contains two drop categories.
11019 EdsResourceArgs args({
11020 {"locality0", CreateEndpointsForBackends()},
11021 });
11022 args.drop_categories = {{kLbDropType, kDropPerMillionForLb},
11023 {kThrottleDropType, kDropPerMillionForThrottle}};
11024 balancers_[0]->ads_service()->SetEdsResource(
11025 BuildEdsResource(args, DefaultEdsServiceName()));
11026 int num_ok = 0;
11027 int num_failure = 0;
11028 int num_drops = 0;
11029 std::tie(num_ok, num_failure, num_drops) = WaitForAllBackends();
11030 const size_t num_warmup = num_ok + num_failure + num_drops;
11031 // Send kNumRpcs RPCs and count the drops.
11032 for (size_t i = 0; i < kNumRpcs; ++i) {
11033 EchoResponse response;
11034 const Status status = SendRpc(RpcOptions(), &response);
11035 if (!status.ok() &&
11036 absl::StartsWith(status.error_message(), "EDS-configured drop: ")) {
11037 ++num_drops;
11038 } else {
11039 EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
11040 << " message=" << status.error_message();
11041 EXPECT_EQ(response.message(), kRequestMessage);
11042 }
11043 }
11044 // The drop rate should be roughly equal to the expectation.
11045 const double seen_drop_rate = static_cast<double>(num_drops) / kNumRpcs;
11046 EXPECT_THAT(seen_drop_rate, ::testing::DoubleNear(kDropRateForLbAndThrottle,
11047 kErrorTolerance));
11048 // Check client stats.
11049 const size_t total_rpc = num_warmup + kNumRpcs;
11050 ClientStats client_stats;
11051 do {
11052 std::vector<ClientStats> load_reports =
11053 balancers_[0]->lrs_service()->WaitForLoadReport();
11054 for (const auto& load_report : load_reports) {
11055 client_stats += load_report;
11056 }
11057 } while (client_stats.total_issued_requests() +
11058 client_stats.total_dropped_requests() <
11059 total_rpc);
11060 EXPECT_EQ(num_drops, client_stats.total_dropped_requests());
11061 EXPECT_THAT(static_cast<double>(client_stats.dropped_requests(kLbDropType)) /
11062 total_rpc,
11063 ::testing::DoubleNear(kDropRateForLb, kErrorTolerance));
11064 EXPECT_THAT(
11065 static_cast<double>(client_stats.dropped_requests(kThrottleDropType)) /
11066 (total_rpc * (1 - kDropRateForLb)),
11067 ::testing::DoubleNear(kDropRateForThrottle, kErrorTolerance));
11068 }
11069
11070 class FaultInjectionTest : public XdsEnd2endTest {
11071 public:
FaultInjectionTest()11072 FaultInjectionTest() : XdsEnd2endTest(1, 1) {}
11073
11074 // Builds a Listener with Fault Injection filter config. If the http_fault
11075 // is nullptr, then assign an empty filter config. This filter config is
11076 // required to enable the fault injection features.
BuildListenerWithFaultInjection(const HTTPFault & http_fault=HTTPFault ())11077 static Listener BuildListenerWithFaultInjection(
11078 const HTTPFault& http_fault = HTTPFault()) {
11079 HttpConnectionManager http_connection_manager;
11080 Listener listener;
11081 listener.set_name(kServerName);
11082 HttpFilter* fault_filter = http_connection_manager.add_http_filters();
11083 fault_filter->set_name("envoy.fault");
11084 fault_filter->mutable_typed_config()->PackFrom(http_fault);
11085 HttpFilter* router_filter = http_connection_manager.add_http_filters();
11086 router_filter->set_name("router");
11087 router_filter->mutable_typed_config()->PackFrom(
11088 envoy::extensions::filters::http::router::v3::Router());
11089 listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
11090 http_connection_manager);
11091 return listener;
11092 }
11093
BuildRouteConfigurationWithFaultInjection(const HTTPFault & http_fault)11094 RouteConfiguration BuildRouteConfigurationWithFaultInjection(
11095 const HTTPFault& http_fault) {
11096 // Package as Any
11097 google::protobuf::Any filter_config;
11098 filter_config.PackFrom(http_fault);
11099 // Plug into the RouteConfiguration
11100 RouteConfiguration new_route_config = default_route_config_;
11101 auto* config_map = new_route_config.mutable_virtual_hosts(0)
11102 ->mutable_routes(0)
11103 ->mutable_typed_per_filter_config();
11104 (*config_map)["envoy.fault"] = std::move(filter_config);
11105 return new_route_config;
11106 }
11107
SetFilterConfig(HTTPFault & http_fault)11108 void SetFilterConfig(HTTPFault& http_fault) {
11109 switch (GetParam().filter_config_setup()) {
11110 case TestType::FilterConfigSetup::kRouteOverride: {
11111 Listener listener = BuildListenerWithFaultInjection();
11112 RouteConfiguration route =
11113 BuildRouteConfigurationWithFaultInjection(http_fault);
11114 SetListenerAndRouteConfiguration(0, listener, route);
11115 break;
11116 }
11117 case TestType::FilterConfigSetup::kHTTPConnectionManagerOriginal: {
11118 Listener listener = BuildListenerWithFaultInjection(http_fault);
11119 SetListenerAndRouteConfiguration(0, listener, default_route_config_);
11120 }
11121 };
11122 }
11123 };
11124
11125 // Test to ensure the most basic fault injection config works.
TEST_P(FaultInjectionTest,XdsFaultInjectionAlwaysAbort)11126 TEST_P(FaultInjectionTest, XdsFaultInjectionAlwaysAbort) {
11127 const uint32_t kAbortPercentagePerHundred = 100;
11128 SetNextResolution({});
11129 SetNextResolutionForLbChannelAllBalancers();
11130 // Construct the fault injection filter config
11131 HTTPFault http_fault;
11132 auto* abort_percentage = http_fault.mutable_abort()->mutable_percentage();
11133 abort_percentage->set_numerator(kAbortPercentagePerHundred);
11134 abort_percentage->set_denominator(FractionalPercent::HUNDRED);
11135 http_fault.mutable_abort()->set_grpc_status(
11136 static_cast<uint32_t>(StatusCode::ABORTED));
11137 // Config fault injection via different setup
11138 SetFilterConfig(http_fault);
11139 // Fire several RPCs, and expect all of them to be aborted.
11140 CheckRpcSendFailure(
11141 CheckRpcSendFailureOptions()
11142 .set_times(5)
11143 .set_rpc_options(RpcOptions().set_wait_for_ready(true))
11144 .set_expected_error_code(StatusCode::ABORTED));
11145 }
11146
11147 // Without the listener config, the fault injection won't be enabled.
TEST_P(FaultInjectionTest,XdsFaultInjectionWithoutListenerFilter)11148 TEST_P(FaultInjectionTest, XdsFaultInjectionWithoutListenerFilter) {
11149 const uint32_t kAbortPercentagePerHundred = 100;
11150 SetNextResolution({});
11151 SetNextResolutionForLbChannelAllBalancers();
11152 // Create an EDS resource
11153 EdsResourceArgs args({
11154 {"locality0", CreateEndpointsForBackends()},
11155 });
11156 balancers_[0]->ads_service()->SetEdsResource(
11157 BuildEdsResource(args, DefaultEdsServiceName()));
11158 // Construct the fault injection filter config
11159 HTTPFault http_fault;
11160 auto* abort_percentage = http_fault.mutable_abort()->mutable_percentage();
11161 abort_percentage->set_numerator(kAbortPercentagePerHundred);
11162 abort_percentage->set_denominator(FractionalPercent::HUNDRED);
11163 http_fault.mutable_abort()->set_grpc_status(
11164 static_cast<uint32_t>(StatusCode::ABORTED));
11165 // Turn on fault injection
11166 RouteConfiguration route =
11167 BuildRouteConfigurationWithFaultInjection(http_fault);
11168 SetListenerAndRouteConfiguration(0, default_listener_, route);
11169 // Fire several RPCs, and expect all of them to be pass.
11170 CheckRpcSendOk(5, RpcOptions().set_wait_for_ready(true));
11171 }
11172
TEST_P(FaultInjectionTest,XdsFaultInjectionPercentageAbort)11173 TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageAbort) {
11174 const uint32_t kAbortPercentagePerHundred = 50;
11175 const double kAbortRate = kAbortPercentagePerHundred / 100.0;
11176 const double kErrorTolerance = 0.05;
11177 const size_t kNumRpcs = ComputeIdealNumRpcs(kAbortRate, kErrorTolerance);
11178 SetNextResolution({});
11179 SetNextResolutionForLbChannelAllBalancers();
11180 // Create an EDS resource
11181 EdsResourceArgs args({
11182 {"locality0", CreateEndpointsForBackends()},
11183 });
11184 balancers_[0]->ads_service()->SetEdsResource(
11185 BuildEdsResource(args, DefaultEdsServiceName()));
11186 // Construct the fault injection filter config
11187 HTTPFault http_fault;
11188 auto* abort_percentage = http_fault.mutable_abort()->mutable_percentage();
11189 abort_percentage->set_numerator(kAbortPercentagePerHundred);
11190 abort_percentage->set_denominator(FractionalPercent::HUNDRED);
11191 http_fault.mutable_abort()->set_grpc_status(
11192 static_cast<uint32_t>(StatusCode::ABORTED));
11193 // Config fault injection via different setup
11194 SetFilterConfig(http_fault);
11195 // Send kNumRpcs RPCs and count the aborts.
11196 int num_total = 0, num_ok = 0, num_failure = 0, num_aborted = 0;
11197 for (size_t i = 0; i < kNumRpcs; ++i) {
11198 SendRpcAndCount(&num_total, &num_ok, &num_failure, &num_aborted,
11199 RpcOptions(), "Fault injected");
11200 }
11201 EXPECT_EQ(kNumRpcs, num_total);
11202 EXPECT_EQ(0, num_failure);
11203 // The abort rate should be roughly equal to the expectation.
11204 const double seen_abort_rate = static_cast<double>(num_aborted) / kNumRpcs;
11205 EXPECT_THAT(seen_abort_rate,
11206 ::testing::DoubleNear(kAbortRate, kErrorTolerance));
11207 }
11208
TEST_P(FaultInjectionTest,XdsFaultInjectionPercentageAbortViaHeaders)11209 TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageAbortViaHeaders) {
11210 const uint32_t kAbortPercentageCap = 100;
11211 const uint32_t kAbortPercentage = 50;
11212 const double kAbortRate = kAbortPercentage / 100.0;
11213 const double kErrorTolerance = 0.05;
11214 const size_t kNumRpcs = ComputeIdealNumRpcs(kAbortRate, kErrorTolerance);
11215 SetNextResolution({});
11216 SetNextResolutionForLbChannelAllBalancers();
11217 // Create an EDS resource
11218 EdsResourceArgs args({
11219 {"locality0", CreateEndpointsForBackends()},
11220 });
11221 balancers_[0]->ads_service()->SetEdsResource(
11222 BuildEdsResource(args, DefaultEdsServiceName()));
11223 // Construct the fault injection filter config
11224 HTTPFault http_fault;
11225 http_fault.mutable_abort()->mutable_header_abort();
11226 http_fault.mutable_abort()->mutable_percentage()->set_numerator(
11227 kAbortPercentageCap);
11228 // Config fault injection via different setup
11229 SetFilterConfig(http_fault);
11230 // Send kNumRpcs RPCs and count the aborts.
11231 std::vector<std::pair<std::string, std::string>> metadata = {
11232 {"x-envoy-fault-abort-grpc-request", "10"},
11233 {"x-envoy-fault-abort-percentage", std::to_string(kAbortPercentage)},
11234 };
11235 int num_total = 0, num_ok = 0, num_failure = 0, num_aborted = 0;
11236 RpcOptions options = RpcOptions().set_metadata(metadata);
11237 for (size_t i = 0; i < kNumRpcs; ++i) {
11238 SendRpcAndCount(&num_total, &num_ok, &num_failure, &num_aborted, options,
11239 "Fault injected");
11240 }
11241 EXPECT_EQ(kNumRpcs, num_total);
11242 EXPECT_EQ(0, num_failure);
11243 // The abort rate should be roughly equal to the expectation.
11244 const double seen_abort_rate = static_cast<double>(num_aborted) / kNumRpcs;
11245 EXPECT_THAT(seen_abort_rate,
11246 ::testing::DoubleNear(kAbortRate, kErrorTolerance));
11247 }
11248
TEST_P(FaultInjectionTest,XdsFaultInjectionPercentageDelay)11249 TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageDelay) {
11250 const uint32_t kRpcTimeoutMilliseconds = grpc_test_slowdown_factor() * 3000;
11251 const uint32_t kFixedDelaySeconds = 100;
11252 const uint32_t kDelayPercentagePerHundred = 50;
11253 const double kDelayRate = kDelayPercentagePerHundred / 100.0;
11254 const double kErrorTolerance = 0.05;
11255 const size_t kNumRpcs = ComputeIdealNumRpcs(kDelayRate, kErrorTolerance);
11256 const size_t kMaxConcurrentRequests = kNumRpcs;
11257 SetNextResolution({});
11258 SetNextResolutionForLbChannelAllBalancers();
11259 // Create an EDS resource
11260 EdsResourceArgs args({
11261 {"locality0", CreateEndpointsForBackends()},
11262 });
11263 balancers_[0]->ads_service()->SetEdsResource(
11264 BuildEdsResource(args, DefaultEdsServiceName()));
11265 // Loosen the max concurrent request limit
11266 Cluster cluster = default_cluster_;
11267 auto* threshold = cluster.mutable_circuit_breakers()->add_thresholds();
11268 threshold->set_priority(RoutingPriority::DEFAULT);
11269 threshold->mutable_max_requests()->set_value(kMaxConcurrentRequests);
11270 balancers_[0]->ads_service()->SetCdsResource(cluster);
11271 // Construct the fault injection filter config
11272 HTTPFault http_fault;
11273 auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage();
11274 delay_percentage->set_numerator(kDelayPercentagePerHundred);
11275 delay_percentage->set_denominator(FractionalPercent::HUNDRED);
11276 auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay();
11277 fixed_delay->set_seconds(kFixedDelaySeconds);
11278 // Config fault injection via different setup
11279 SetFilterConfig(http_fault);
11280 // Send kNumRpcs RPCs and count the delays.
11281 RpcOptions rpc_options = RpcOptions()
11282 .set_timeout_ms(kRpcTimeoutMilliseconds)
11283 .set_skip_cancelled_check(true);
11284 std::vector<ConcurrentRpc> rpcs =
11285 SendConcurrentRpcs(stub_.get(), kNumRpcs, rpc_options);
11286 size_t num_delayed = 0;
11287 for (auto& rpc : rpcs) {
11288 if (rpc.status.error_code() == StatusCode::OK) continue;
11289 EXPECT_EQ(StatusCode::DEADLINE_EXCEEDED, rpc.status.error_code());
11290 ++num_delayed;
11291 }
11292 // The delay rate should be roughly equal to the expectation.
11293 const double seen_delay_rate = static_cast<double>(num_delayed) / kNumRpcs;
11294 EXPECT_THAT(seen_delay_rate,
11295 ::testing::DoubleNear(kDelayRate, kErrorTolerance));
11296 }
11297
TEST_P(FaultInjectionTest,XdsFaultInjectionPercentageDelayViaHeaders)11298 TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageDelayViaHeaders) {
11299 const uint32_t kFixedDelayMilliseconds = 100000;
11300 const uint32_t kRpcTimeoutMilliseconds = grpc_test_slowdown_factor() * 3000;
11301 const uint32_t kDelayPercentageCap = 100;
11302 const uint32_t kDelayPercentage = 50;
11303 const double kDelayRate = kDelayPercentage / 100.0;
11304 const double kErrorTolerance = 0.05;
11305 const size_t kNumRpcs = ComputeIdealNumRpcs(kDelayRate, kErrorTolerance);
11306 const size_t kMaxConcurrentRequests = kNumRpcs;
11307 SetNextResolution({});
11308 SetNextResolutionForLbChannelAllBalancers();
11309 // Create an EDS resource
11310 EdsResourceArgs args({
11311 {"locality0", CreateEndpointsForBackends()},
11312 });
11313 balancers_[0]->ads_service()->SetEdsResource(
11314 BuildEdsResource(args, DefaultEdsServiceName()));
11315 // Loosen the max concurrent request limit
11316 Cluster cluster = default_cluster_;
11317 auto* threshold = cluster.mutable_circuit_breakers()->add_thresholds();
11318 threshold->set_priority(RoutingPriority::DEFAULT);
11319 threshold->mutable_max_requests()->set_value(kMaxConcurrentRequests);
11320 balancers_[0]->ads_service()->SetCdsResource(cluster);
11321 // Construct the fault injection filter config
11322 HTTPFault http_fault;
11323 http_fault.mutable_delay()->mutable_header_delay();
11324 http_fault.mutable_delay()->mutable_percentage()->set_numerator(
11325 kDelayPercentageCap);
11326 // Config fault injection via different setup
11327 SetFilterConfig(http_fault);
11328 // Send kNumRpcs RPCs and count the delays.
11329 std::vector<std::pair<std::string, std::string>> metadata = {
11330 {"x-envoy-fault-delay-request", std::to_string(kFixedDelayMilliseconds)},
11331 {"x-envoy-fault-delay-request-percentage",
11332 std::to_string(kDelayPercentage)},
11333 };
11334 RpcOptions rpc_options = RpcOptions()
11335 .set_metadata(metadata)
11336 .set_timeout_ms(kRpcTimeoutMilliseconds)
11337 .set_skip_cancelled_check(true);
11338 std::vector<ConcurrentRpc> rpcs =
11339 SendConcurrentRpcs(stub_.get(), kNumRpcs, rpc_options);
11340 size_t num_delayed = 0;
11341 for (auto& rpc : rpcs) {
11342 if (rpc.status.error_code() == StatusCode::OK) continue;
11343 EXPECT_EQ(StatusCode::DEADLINE_EXCEEDED, rpc.status.error_code());
11344 ++num_delayed;
11345 }
11346 // The delay rate should be roughly equal to the expectation.
11347 const double seen_delay_rate = static_cast<double>(num_delayed) / kNumRpcs;
11348 EXPECT_THAT(seen_delay_rate,
11349 ::testing::DoubleNear(kDelayRate, kErrorTolerance));
11350 }
11351
TEST_P(FaultInjectionTest,XdsFaultInjectionAbortAfterDelayForStreamCall)11352 TEST_P(FaultInjectionTest, XdsFaultInjectionAbortAfterDelayForStreamCall) {
11353 const uint32_t kFixedDelaySeconds = 1;
11354 const uint32_t kRpcTimeoutMilliseconds = 100 * 1000; // 100s should not reach
11355 SetNextResolution({});
11356 SetNextResolutionForLbChannelAllBalancers();
11357 // Create an EDS resource
11358 EdsResourceArgs args({
11359 {"locality0", CreateEndpointsForBackends()},
11360 });
11361 balancers_[0]->ads_service()->SetEdsResource(
11362 BuildEdsResource(args, DefaultEdsServiceName()));
11363 // Construct the fault injection filter config
11364 HTTPFault http_fault;
11365 auto* abort_percentage = http_fault.mutable_abort()->mutable_percentage();
11366 abort_percentage->set_numerator(100); // Always inject ABORT!
11367 abort_percentage->set_denominator(FractionalPercent::HUNDRED);
11368 http_fault.mutable_abort()->set_grpc_status(
11369 static_cast<uint32_t>(StatusCode::ABORTED));
11370 auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage();
11371 delay_percentage->set_numerator(100); // Always inject DELAY!
11372 delay_percentage->set_denominator(FractionalPercent::HUNDRED);
11373 auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay();
11374 fixed_delay->set_seconds(kFixedDelaySeconds);
11375 // Config fault injection via different setup
11376 SetFilterConfig(http_fault);
11377 // Send a stream RPC and check its status code
11378 ClientContext context;
11379 context.set_deadline(
11380 grpc_timeout_milliseconds_to_deadline(kRpcTimeoutMilliseconds));
11381 auto stream = stub_->BidiStream(&context);
11382 stream->WritesDone();
11383 auto status = stream->Finish();
11384 EXPECT_EQ(StatusCode::ABORTED, status.error_code())
11385 << status.error_message() << ", " << status.error_details() << ", "
11386 << context.debug_error_string();
11387 }
11388
TEST_P(FaultInjectionTest,XdsFaultInjectionAlwaysDelayPercentageAbort)11389 TEST_P(FaultInjectionTest, XdsFaultInjectionAlwaysDelayPercentageAbort) {
11390 const uint32_t kAbortPercentagePerHundred = 50;
11391 const double kAbortRate = kAbortPercentagePerHundred / 100.0;
11392 const uint32_t kFixedDelaySeconds = 1;
11393 const uint32_t kRpcTimeoutMilliseconds = 100 * 1000; // 100s should not reach
11394 const uint32_t kConnectionTimeoutMilliseconds =
11395 10 * 1000; // 10s should not reach
11396 const double kErrorTolerance = 0.05;
11397 const size_t kNumRpcs = ComputeIdealNumRpcs(kAbortRate, kErrorTolerance);
11398 const size_t kMaxConcurrentRequests = kNumRpcs;
11399 SetNextResolution({});
11400 SetNextResolutionForLbChannelAllBalancers();
11401 // Create an EDS resource
11402 EdsResourceArgs args({
11403 {"locality0", CreateEndpointsForBackends()},
11404 });
11405 balancers_[0]->ads_service()->SetEdsResource(
11406 BuildEdsResource(args, DefaultEdsServiceName()));
11407 // Loosen the max concurrent request limit
11408 Cluster cluster = default_cluster_;
11409 auto* threshold = cluster.mutable_circuit_breakers()->add_thresholds();
11410 threshold->set_priority(RoutingPriority::DEFAULT);
11411 threshold->mutable_max_requests()->set_value(kMaxConcurrentRequests);
11412 balancers_[0]->ads_service()->SetCdsResource(cluster);
11413 // Construct the fault injection filter config
11414 HTTPFault http_fault;
11415 auto* abort_percentage = http_fault.mutable_abort()->mutable_percentage();
11416 abort_percentage->set_numerator(kAbortPercentagePerHundred);
11417 abort_percentage->set_denominator(FractionalPercent::HUNDRED);
11418 http_fault.mutable_abort()->set_grpc_status(
11419 static_cast<uint32_t>(StatusCode::ABORTED));
11420 auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage();
11421 delay_percentage->set_numerator(1000000); // Always inject DELAY!
11422 delay_percentage->set_denominator(FractionalPercent::MILLION);
11423 auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay();
11424 fixed_delay->set_seconds(kFixedDelaySeconds);
11425 // Config fault injection via different setup
11426 SetFilterConfig(http_fault);
11427 // Allow the channel to connect to one backends, so the herd of queued RPCs
11428 // won't be executed on the same ExecCtx object and using the cached Now()
11429 // value, which causes millisecond level delay error.
11430 channel_->WaitForConnected(
11431 grpc_timeout_milliseconds_to_deadline(kConnectionTimeoutMilliseconds));
11432 // Send kNumRpcs RPCs and count the aborts.
11433 int num_aborted = 0;
11434 RpcOptions rpc_options = RpcOptions().set_timeout_ms(kRpcTimeoutMilliseconds);
11435 std::vector<ConcurrentRpc> rpcs =
11436 SendConcurrentRpcs(stub_.get(), kNumRpcs, rpc_options);
11437 for (auto& rpc : rpcs) {
11438 EXPECT_GE(rpc.elapsed_time, kFixedDelaySeconds * 1000);
11439 if (rpc.status.error_code() == StatusCode::OK) continue;
11440 EXPECT_EQ("Fault injected", rpc.status.error_message());
11441 ++num_aborted;
11442 }
11443 // The abort rate should be roughly equal to the expectation.
11444 const double seen_abort_rate = static_cast<double>(num_aborted) / kNumRpcs;
11445 EXPECT_THAT(seen_abort_rate,
11446 ::testing::DoubleNear(kAbortRate, kErrorTolerance));
11447 }
11448
11449 // This test and the above test apply different denominators to delay and
11450 // abort. This ensures that we are using the right denominator for each
11451 // injected fault in our code.
TEST_P(FaultInjectionTest,XdsFaultInjectionAlwaysDelayPercentageAbortSwitchDenominator)11452 TEST_P(FaultInjectionTest,
11453 XdsFaultInjectionAlwaysDelayPercentageAbortSwitchDenominator) {
11454 const uint32_t kAbortPercentagePerMillion = 500000;
11455 const double kAbortRate = kAbortPercentagePerMillion / 1000000.0;
11456 const uint32_t kFixedDelaySeconds = 1; // 1s
11457 const uint32_t kRpcTimeoutMilliseconds = 100 * 1000; // 100s should not reach
11458 const uint32_t kConnectionTimeoutMilliseconds =
11459 10 * 1000; // 10s should not reach
11460 const double kErrorTolerance = 0.05;
11461 const size_t kNumRpcs = ComputeIdealNumRpcs(kAbortRate, kErrorTolerance);
11462 const size_t kMaxConcurrentRequests = kNumRpcs;
11463 SetNextResolution({});
11464 SetNextResolutionForLbChannelAllBalancers();
11465 // Create an EDS resource
11466 EdsResourceArgs args({
11467 {"locality0", CreateEndpointsForBackends()},
11468 });
11469 balancers_[0]->ads_service()->SetEdsResource(
11470 BuildEdsResource(args, DefaultEdsServiceName()));
11471 // Loosen the max concurrent request limit
11472 Cluster cluster = default_cluster_;
11473 auto* threshold = cluster.mutable_circuit_breakers()->add_thresholds();
11474 threshold->set_priority(RoutingPriority::DEFAULT);
11475 threshold->mutable_max_requests()->set_value(kMaxConcurrentRequests);
11476 balancers_[0]->ads_service()->SetCdsResource(cluster);
11477 // Construct the fault injection filter config
11478 HTTPFault http_fault;
11479 auto* abort_percentage = http_fault.mutable_abort()->mutable_percentage();
11480 abort_percentage->set_numerator(kAbortPercentagePerMillion);
11481 abort_percentage->set_denominator(FractionalPercent::MILLION);
11482 http_fault.mutable_abort()->set_grpc_status(
11483 static_cast<uint32_t>(StatusCode::ABORTED));
11484 auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage();
11485 delay_percentage->set_numerator(100); // Always inject DELAY!
11486 delay_percentage->set_denominator(FractionalPercent::HUNDRED);
11487 auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay();
11488 fixed_delay->set_seconds(kFixedDelaySeconds);
11489 // Config fault injection via different setup
11490 SetFilterConfig(http_fault);
11491 // Allow the channel to connect to one backends, so the herd of queued RPCs
11492 // won't be executed on the same ExecCtx object and using the cached Now()
11493 // value, which causes millisecond level delay error.
11494 channel_->WaitForConnected(
11495 grpc_timeout_milliseconds_to_deadline(kConnectionTimeoutMilliseconds));
11496 // Send kNumRpcs RPCs and count the aborts.
11497 int num_aborted = 0;
11498 RpcOptions rpc_options = RpcOptions().set_timeout_ms(kRpcTimeoutMilliseconds);
11499 std::vector<ConcurrentRpc> rpcs =
11500 SendConcurrentRpcs(stub_.get(), kNumRpcs, rpc_options);
11501 for (auto& rpc : rpcs) {
11502 EXPECT_GE(rpc.elapsed_time, kFixedDelaySeconds * 1000);
11503 if (rpc.status.error_code() == StatusCode::OK) continue;
11504 EXPECT_EQ("Fault injected", rpc.status.error_message());
11505 ++num_aborted;
11506 }
11507 // The abort rate should be roughly equal to the expectation.
11508 const double seen_abort_rate = static_cast<double>(num_aborted) / kNumRpcs;
11509 EXPECT_THAT(seen_abort_rate,
11510 ::testing::DoubleNear(kAbortRate, kErrorTolerance));
11511 }
11512
TEST_P(FaultInjectionTest,XdsFaultInjectionMaxFault)11513 TEST_P(FaultInjectionTest, XdsFaultInjectionMaxFault) {
11514 const uint32_t kMaxFault = 10;
11515 const uint32_t kNumRpcs = 30; // kNumRpcs should be bigger than kMaxFault
11516 const uint32_t kRpcTimeoutMs = 4000; // 4 seconds
11517 const uint32_t kLongDelaySeconds = 100; // 100 seconds
11518 const uint32_t kAlwaysDelayPercentage = 100;
11519 SetNextResolution({});
11520 SetNextResolutionForLbChannelAllBalancers();
11521 // Create an EDS resource
11522 EdsResourceArgs args({
11523 {"locality0", CreateEndpointsForBackends()},
11524 });
11525 balancers_[0]->ads_service()->SetEdsResource(
11526 BuildEdsResource(args, DefaultEdsServiceName()));
11527 // Construct the fault injection filter config
11528 HTTPFault http_fault;
11529 auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage();
11530 delay_percentage->set_numerator(
11531 kAlwaysDelayPercentage); // Always inject DELAY!
11532 delay_percentage->set_denominator(FractionalPercent::HUNDRED);
11533 auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay();
11534 fixed_delay->set_seconds(kLongDelaySeconds);
11535 http_fault.mutable_max_active_faults()->set_value(kMaxFault);
11536 // Config fault injection via different setup
11537 SetFilterConfig(http_fault);
11538 // Sends a batch of long running RPCs with long timeout to consume all
11539 // active faults quota.
11540 int num_delayed = 0;
11541 RpcOptions rpc_options = RpcOptions().set_timeout_ms(kRpcTimeoutMs);
11542 std::vector<ConcurrentRpc> rpcs =
11543 SendConcurrentRpcs(stub_.get(), kNumRpcs, rpc_options);
11544 for (auto& rpc : rpcs) {
11545 if (rpc.status.error_code() == StatusCode::OK) continue;
11546 EXPECT_EQ(StatusCode::DEADLINE_EXCEEDED, rpc.status.error_code());
11547 ++num_delayed;
11548 }
11549 // Only kMaxFault number of RPC should be fault injected..
11550 EXPECT_EQ(kMaxFault, num_delayed);
11551 }
11552
TEST_P(FaultInjectionTest,XdsFaultInjectionBidiStreamDelayOk)11553 TEST_P(FaultInjectionTest, XdsFaultInjectionBidiStreamDelayOk) {
11554 // kRpcTimeoutMilliseconds is 10s should never be reached.
11555 const uint32_t kRpcTimeoutMilliseconds = grpc_test_slowdown_factor() * 10000;
11556 const uint32_t kFixedDelaySeconds = 1;
11557 const uint32_t kDelayPercentagePerHundred = 100;
11558 SetNextResolution({});
11559 SetNextResolutionForLbChannelAllBalancers();
11560 // Create an EDS resource
11561 EdsResourceArgs args({
11562 {"locality0", CreateEndpointsForBackends()},
11563 });
11564 balancers_[0]->ads_service()->SetEdsResource(
11565 BuildEdsResource(args, DefaultEdsServiceName()));
11566 // Construct the fault injection filter config
11567 HTTPFault http_fault;
11568 auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage();
11569 delay_percentage->set_numerator(kDelayPercentagePerHundred);
11570 delay_percentage->set_denominator(FractionalPercent::HUNDRED);
11571 auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay();
11572 fixed_delay->set_seconds(kFixedDelaySeconds);
11573 // Config fault injection via different setup
11574 SetFilterConfig(http_fault);
11575 ClientContext context;
11576 context.set_deadline(
11577 grpc_timeout_milliseconds_to_deadline(kRpcTimeoutMilliseconds));
11578 auto stream = stub_->BidiStream(&context);
11579 stream->WritesDone();
11580 auto status = stream->Finish();
11581 EXPECT_TRUE(status.ok()) << status.error_message() << ", "
11582 << status.error_details() << ", "
11583 << context.debug_error_string();
11584 }
11585
11586 // This case catches a bug in the retry code that was triggered by a bad
11587 // interaction with the FI code. See https://github.com/grpc/grpc/pull/27217
11588 // for description.
TEST_P(FaultInjectionTest,XdsFaultInjectionBidiStreamDelayError)11589 TEST_P(FaultInjectionTest, XdsFaultInjectionBidiStreamDelayError) {
11590 const uint32_t kRpcTimeoutMilliseconds = grpc_test_slowdown_factor() * 500;
11591 const uint32_t kFixedDelaySeconds = 100;
11592 const uint32_t kDelayPercentagePerHundred = 100;
11593 SetNextResolution({});
11594 SetNextResolutionForLbChannelAllBalancers();
11595 // Create an EDS resource
11596 EdsResourceArgs args({
11597 {"locality0", CreateEndpointsForBackends()},
11598 });
11599 balancers_[0]->ads_service()->SetEdsResource(
11600 BuildEdsResource(args, DefaultEdsServiceName()));
11601 // Construct the fault injection filter config
11602 HTTPFault http_fault;
11603 auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage();
11604 delay_percentage->set_numerator(kDelayPercentagePerHundred);
11605 delay_percentage->set_denominator(FractionalPercent::HUNDRED);
11606 auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay();
11607 fixed_delay->set_seconds(kFixedDelaySeconds);
11608 // Config fault injection via different setup
11609 SetFilterConfig(http_fault);
11610 ClientContext context;
11611 context.set_deadline(
11612 grpc_timeout_milliseconds_to_deadline(kRpcTimeoutMilliseconds));
11613 auto stream = stub_->BidiStream(&context);
11614 stream->WritesDone();
11615 auto status = stream->Finish();
11616 EXPECT_EQ(StatusCode::DEADLINE_EXCEEDED, status.error_code())
11617 << status.error_message() << ", " << status.error_details() << ", "
11618 << context.debug_error_string();
11619 }
11620
11621 class BootstrapSourceTest : public XdsEnd2endTest {
11622 public:
BootstrapSourceTest()11623 BootstrapSourceTest() : XdsEnd2endTest(4, 1) {}
11624 };
11625
TEST_P(BootstrapSourceTest,Vanilla)11626 TEST_P(BootstrapSourceTest, Vanilla) {
11627 SetNextResolution({});
11628 SetNextResolutionForLbChannelAllBalancers();
11629 EdsResourceArgs args({
11630 {"locality0", CreateEndpointsForBackends()},
11631 });
11632 balancers_[0]->ads_service()->SetEdsResource(
11633 BuildEdsResource(args, DefaultEdsServiceName()));
11634 WaitForAllBackends();
11635 }
11636
11637 #ifndef DISABLED_XDS_PROTO_IN_CC
11638 class ClientStatusDiscoveryServiceTest : public XdsEnd2endTest {
11639 public:
ClientStatusDiscoveryServiceTest()11640 ClientStatusDiscoveryServiceTest() : XdsEnd2endTest(1, 1) {}
11641
SetUp()11642 void SetUp() override {
11643 XdsEnd2endTest::SetUp();
11644 admin_server_thread_ = absl::make_unique<AdminServerThread>(this);
11645 admin_server_thread_->Start();
11646 std::string admin_server_address = absl::StrCat(
11647 ipv6_only_ ? "[::1]:" : "127.0.0.1:", admin_server_thread_->port());
11648 admin_channel_ = grpc::CreateChannel(
11649 admin_server_address,
11650 std::make_shared<SecureChannelCredentials>(
11651 grpc_fake_transport_security_credentials_create()));
11652 csds_stub_ =
11653 envoy::service::status::v3::ClientStatusDiscoveryService::NewStub(
11654 admin_channel_);
11655 if (GetParam().use_csds_streaming()) {
11656 stream_ = csds_stub_->StreamClientStatus(&stream_context_);
11657 }
11658 }
11659
TearDown()11660 void TearDown() override {
11661 if (stream_ != nullptr) {
11662 EXPECT_TRUE(stream_->WritesDone());
11663 Status status = stream_->Finish();
11664 EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
11665 << " message=" << status.error_message();
11666 }
11667 admin_server_thread_->Shutdown();
11668 XdsEnd2endTest::TearDown();
11669 }
11670
FetchCsdsResponse()11671 envoy::service::status::v3::ClientStatusResponse FetchCsdsResponse() {
11672 envoy::service::status::v3::ClientStatusResponse response;
11673 if (!GetParam().use_csds_streaming()) {
11674 // Fetch through unary pulls
11675 ClientContext context;
11676 Status status = csds_stub_->FetchClientStatus(
11677 &context, envoy::service::status::v3::ClientStatusRequest(),
11678 &response);
11679 EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
11680 << " message=" << status.error_message();
11681 } else {
11682 // Fetch through streaming pulls
11683 EXPECT_TRUE(
11684 stream_->Write(envoy::service::status::v3::ClientStatusRequest()));
11685 EXPECT_TRUE(stream_->Read(&response));
11686 }
11687 return response;
11688 }
11689
11690 private:
11691 std::unique_ptr<AdminServerThread> admin_server_thread_;
11692 std::shared_ptr<Channel> admin_channel_;
11693 std::unique_ptr<
11694 envoy::service::status::v3::ClientStatusDiscoveryService::Stub>
11695 csds_stub_;
11696 ClientContext stream_context_;
11697 std::unique_ptr<
11698 ClientReaderWriter<envoy::service::status::v3::ClientStatusRequest,
11699 envoy::service::status::v3::ClientStatusResponse>>
11700 stream_;
11701 };
11702
11703 MATCHER_P4(EqNode, id, user_agent_name, user_agent_version, client_features,
11704 "equals Node") {
11705 bool ok = true;
11706 ok &= ::testing::ExplainMatchResult(id, arg.id(), result_listener);
11707 ok &= ::testing::ExplainMatchResult(user_agent_name, arg.user_agent_name(),
11708 result_listener);
11709 ok &= ::testing::ExplainMatchResult(
11710 user_agent_version, arg.user_agent_version(), result_listener);
11711 ok &= ::testing::ExplainMatchResult(client_features, arg.client_features(),
11712 result_listener);
11713 return ok;
11714 }
11715
11716 MATCHER_P6(EqGenericXdsConfig, type_url, name, version_info, xds_config,
11717 client_status, error_state, "equals GenericXdsConfig") {
11718 bool ok = true;
11719 ok &=
11720 ::testing::ExplainMatchResult(type_url, arg.type_url(), result_listener);
11721 ok &= ::testing::ExplainMatchResult(name, arg.name(), result_listener);
11722 ok &= ::testing::ExplainMatchResult(version_info, arg.version_info(),
11723 result_listener);
11724 ok &= ::testing::ExplainMatchResult(xds_config, arg.xds_config(),
11725 result_listener);
11726 ok &= ::testing::ExplainMatchResult(client_status, arg.client_status(),
11727 result_listener);
11728 ok &= ::testing::ExplainMatchResult(error_state, arg.error_state(),
11729 result_listener);
11730 return ok;
11731 }
11732
11733 MATCHER_P2(EqListener, name, api_listener, "equals Listener") {
11734 bool ok = true;
11735 ok &= ::testing::ExplainMatchResult(name, arg.name(), result_listener);
11736 ok &= ::testing::ExplainMatchResult(
11737 api_listener, arg.api_listener().api_listener(), result_listener);
11738 return ok;
11739 }
11740
11741 MATCHER_P(EqHttpConnectionManagerNotRds, route_config,
11742 "equals HttpConnectionManager") {
11743 bool ok = true;
11744 ok &= ::testing::ExplainMatchResult(route_config, arg.route_config(),
11745 result_listener);
11746 return ok;
11747 }
11748
11749 MATCHER_P(EqRouteConfigurationName, name, "equals RouteConfiguration") {
11750 bool ok = true;
11751 ok &= ::testing::ExplainMatchResult(name, arg.name(), result_listener);
11752 return ok;
11753 }
11754
11755 MATCHER_P2(EqRouteConfiguration, name, cluster_name,
11756 "equals RouteConfiguration") {
11757 bool ok = true;
11758 ok &= ::testing::ExplainMatchResult(name, arg.name(), result_listener);
11759 ok &= ::testing::ExplainMatchResult(
11760 ::testing::ElementsAre(::testing::Property(
11761 &envoy::config::route::v3::VirtualHost::routes,
11762 ::testing::ElementsAre(::testing::Property(
11763 &envoy::config::route::v3::Route::route,
11764 ::testing::Property(
11765 &envoy::config::route::v3::RouteAction::cluster,
11766 cluster_name))))),
11767 arg.virtual_hosts(), result_listener);
11768 return ok;
11769 }
11770
11771 MATCHER_P(EqCluster, name, "equals Cluster") {
11772 bool ok = true;
11773 ok &= ::testing::ExplainMatchResult(name, arg.name(), result_listener);
11774 return ok;
11775 }
11776
11777 MATCHER_P(EqEndpoint, port, "equals Endpoint") {
11778 bool ok = true;
11779 ok &= ::testing::ExplainMatchResult(
11780 port, arg.address().socket_address().port_value(), result_listener);
11781 return ok;
11782 }
11783
11784 MATCHER_P2(EqLocalityLbEndpoints, port, weight, "equals LocalityLbEndpoints") {
11785 bool ok = true;
11786 ok &= ::testing::ExplainMatchResult(
11787 ::testing::ElementsAre(::testing::Property(
11788 &envoy::config::endpoint::v3::LbEndpoint::endpoint,
11789 EqEndpoint(port))),
11790 arg.lb_endpoints(), result_listener);
11791 ok &= ::testing::ExplainMatchResult(
11792 weight, arg.load_balancing_weight().value(), result_listener);
11793 return ok;
11794 }
11795
11796 MATCHER_P(EqClusterLoadAssignmentName, cluster_name,
11797 "equals ClusterLoadAssignment") {
11798 bool ok = true;
11799 ok &= ::testing::ExplainMatchResult(cluster_name, arg.cluster_name(),
11800 result_listener);
11801 return ok;
11802 }
11803
11804 MATCHER_P3(EqClusterLoadAssignment, cluster_name, port, weight,
11805 "equals ClusterLoadAssignment") {
11806 bool ok = true;
11807 ok &= ::testing::ExplainMatchResult(cluster_name, arg.cluster_name(),
11808 result_listener);
11809 ok &= ::testing::ExplainMatchResult(
11810 ::testing::ElementsAre(EqLocalityLbEndpoints(port, weight)),
11811 arg.endpoints(), result_listener);
11812 return ok;
11813 }
11814
11815 MATCHER_P2(EqUpdateFailureState, details, version_info,
11816 "equals UpdateFailureState") {
11817 bool ok = true;
11818 ok &= ::testing::ExplainMatchResult(details, arg.details(), result_listener);
11819 ok &= ::testing::ExplainMatchResult(version_info, arg.version_info(),
11820 result_listener);
11821 return ok;
11822 }
11823
11824 MATCHER_P(UnpackListener, matcher, "is a Listener") {
11825 Listener config;
11826 if (!::testing::ExplainMatchResult(true, arg.UnpackTo(&config),
11827 result_listener)) {
11828 return false;
11829 }
11830 return ::testing::ExplainMatchResult(matcher, config, result_listener);
11831 }
11832
11833 MATCHER_P(UnpackRouteConfiguration, matcher, "is a RouteConfiguration") {
11834 RouteConfiguration config;
11835 if (!::testing::ExplainMatchResult(true, arg.UnpackTo(&config),
11836 result_listener)) {
11837 return false;
11838 }
11839 return ::testing::ExplainMatchResult(matcher, config, result_listener);
11840 }
11841
11842 MATCHER_P(UnpackHttpConnectionManager, matcher, "is a HttpConnectionManager") {
11843 HttpConnectionManager config;
11844 if (!::testing::ExplainMatchResult(true, arg.UnpackTo(&config),
11845 result_listener)) {
11846 return false;
11847 }
11848 return ::testing::ExplainMatchResult(matcher, config, result_listener);
11849 }
11850
11851 MATCHER_P(UnpackCluster, matcher, "is a Cluster") {
11852 Cluster config;
11853 if (!::testing::ExplainMatchResult(true, arg.UnpackTo(&config),
11854 result_listener)) {
11855 return false;
11856 }
11857 return ::testing::ExplainMatchResult(matcher, config, result_listener);
11858 }
11859
11860 MATCHER_P(UnpackClusterLoadAssignment, matcher, "is a ClusterLoadAssignment") {
11861 ClusterLoadAssignment config;
11862 if (!::testing::ExplainMatchResult(true, arg.UnpackTo(&config),
11863 result_listener)) {
11864 return false;
11865 }
11866 return ::testing::ExplainMatchResult(matcher, config, result_listener);
11867 }
11868
11869 MATCHER(IsRdsEnabledHCM, "is a RDS enabled HttpConnectionManager") {
11870 return ::testing::ExplainMatchResult(
11871 UnpackHttpConnectionManager(
11872 ::testing::Property(&HttpConnectionManager::has_rds, true)),
11873 arg, result_listener);
11874 }
11875
11876 MATCHER_P2(EqNoRdsHCM, route_configuration_name, cluster_name,
11877 "equals RDS disabled HttpConnectionManager") {
11878 return ::testing::ExplainMatchResult(
11879 UnpackHttpConnectionManager(EqHttpConnectionManagerNotRds(
11880 EqRouteConfiguration(route_configuration_name, cluster_name))),
11881 arg, result_listener);
11882 }
11883
TEST_P(ClientStatusDiscoveryServiceTest,XdsConfigDumpVanilla)11884 TEST_P(ClientStatusDiscoveryServiceTest, XdsConfigDumpVanilla) {
11885 const size_t kNumRpcs = 5;
11886 SetNextResolution({});
11887 SetNextResolutionForLbChannelAllBalancers();
11888 EdsResourceArgs args({{"locality0", CreateEndpointsForBackends(0, 1)}});
11889 balancers_[0]->ads_service()->SetEdsResource(
11890 BuildEdsResource(args, DefaultEdsServiceName()));
11891 // Send several RPCs to ensure the xDS setup works
11892 CheckRpcSendOk(kNumRpcs);
11893 // Fetches the client config
11894 auto csds_response = FetchCsdsResponse();
11895 gpr_log(GPR_INFO, "xDS config dump: %s", csds_response.DebugString().c_str());
11896 EXPECT_EQ(1, csds_response.config_size());
11897 const auto& client_config = csds_response.config(0);
11898 // Validate the Node information
11899 EXPECT_THAT(client_config.node(),
11900 EqNode("xds_end2end_test", ::testing::HasSubstr("C-core"),
11901 ::testing::HasSubstr(grpc_version_string()),
11902 ::testing::ElementsAre(
11903 "envoy.lb.does_not_support_overprovisioning")));
11904 // Listener matcher depends on whether RDS is enabled.
11905 ::testing::Matcher<google::protobuf::Any> api_listener_matcher;
11906 if (GetParam().enable_rds_testing()) {
11907 api_listener_matcher = IsRdsEnabledHCM();
11908 } else {
11909 api_listener_matcher =
11910 EqNoRdsHCM(kDefaultRouteConfigurationName, kDefaultClusterName);
11911 }
11912 // Construct list of all matchers.
11913 std::vector<::testing::Matcher<
11914 envoy::service::status::v3::ClientConfig_GenericXdsConfig>>
11915 matchers = {
11916 // Listener
11917 EqGenericXdsConfig(
11918 kLdsTypeUrl, kServerName, /*version_info=*/"1",
11919 UnpackListener(EqListener(kServerName, api_listener_matcher)),
11920 ClientResourceStatus::ACKED, /*error_state=*/::testing::_),
11921 // Cluster
11922 EqGenericXdsConfig(
11923 kCdsTypeUrl, kDefaultClusterName, /*version_info=*/"1",
11924 UnpackCluster(EqCluster(kDefaultClusterName)),
11925 ClientResourceStatus::ACKED, /*error_state=*/::testing::_),
11926 // ClusterLoadAssignment
11927 EqGenericXdsConfig(
11928 kEdsTypeUrl, kDefaultEdsServiceName, /*version_info=*/"1",
11929 UnpackClusterLoadAssignment(EqClusterLoadAssignment(
11930 kDefaultEdsServiceName, backends_[0]->port(),
11931 kDefaultLocalityWeight)),
11932 ClientResourceStatus::ACKED, /*error_state=*/::testing::_),
11933 };
11934 // If RDS is enabled, add matcher for RDS resource.
11935 if (GetParam().enable_rds_testing()) {
11936 matchers.push_back(EqGenericXdsConfig(
11937 kRdsTypeUrl, kDefaultRouteConfigurationName, /*version_info=*/"1",
11938 UnpackRouteConfiguration(EqRouteConfiguration(
11939 kDefaultRouteConfigurationName, kDefaultClusterName)),
11940 ClientResourceStatus::ACKED, /*error_state=*/::testing::_));
11941 }
11942 // Validate the dumped xDS configs
11943 EXPECT_THAT(client_config.generic_xds_configs(),
11944 ::testing::UnorderedElementsAreArray(matchers))
11945 << "Actual: " << client_config.DebugString();
11946 }
11947
TEST_P(ClientStatusDiscoveryServiceTest,XdsConfigDumpEmpty)11948 TEST_P(ClientStatusDiscoveryServiceTest, XdsConfigDumpEmpty) {
11949 // The CSDS service should not fail if XdsClient is not initialized or there
11950 // is no working xDS configs.
11951 FetchCsdsResponse();
11952 }
11953
TEST_P(ClientStatusDiscoveryServiceTest,XdsConfigDumpListenerError)11954 TEST_P(ClientStatusDiscoveryServiceTest, XdsConfigDumpListenerError) {
11955 int kFetchConfigRetries = 3;
11956 int kFetchIntervalMilliseconds = 200;
11957 SetNextResolution({});
11958 SetNextResolutionForLbChannelAllBalancers();
11959 EdsResourceArgs args({{"locality0", CreateEndpointsForBackends(0, 1)}});
11960 balancers_[0]->ads_service()->SetEdsResource(
11961 BuildEdsResource(args, DefaultEdsServiceName()));
11962 // Ensure the xDS resolver has working configs.
11963 CheckRpcSendOk();
11964 // Bad Listener should be rejected.
11965 Listener listener;
11966 listener.set_name(kServerName);
11967 balancers_[0]->ads_service()->SetLdsResource(listener);
11968 // The old xDS configs should still be effective.
11969 CheckRpcSendOk();
11970 ::testing::Matcher<google::protobuf::Any> api_listener_matcher;
11971 if (GetParam().enable_rds_testing()) {
11972 api_listener_matcher = IsRdsEnabledHCM();
11973 } else {
11974 api_listener_matcher =
11975 EqNoRdsHCM(kDefaultRouteConfigurationName, kDefaultClusterName);
11976 }
11977 for (int i = 0; i < kFetchConfigRetries; ++i) {
11978 auto csds_response = FetchCsdsResponse();
11979 // Check if error state is propagated
11980 bool ok = ::testing::Value(
11981 csds_response.config(0).generic_xds_configs(),
11982 ::testing::Contains(EqGenericXdsConfig(
11983 kLdsTypeUrl, kServerName, /*version_info=*/"1",
11984 UnpackListener(EqListener(kServerName, api_listener_matcher)),
11985 ClientResourceStatus::NACKED,
11986 EqUpdateFailureState(
11987 ::testing::HasSubstr(
11988 "Listener has neither address nor ApiListener"),
11989 "2"))));
11990 if (ok) return; // TEST PASSED!
11991 gpr_sleep_until(
11992 grpc_timeout_milliseconds_to_deadline(kFetchIntervalMilliseconds));
11993 }
11994 FAIL() << "error_state not seen in CSDS responses";
11995 }
11996
TEST_P(ClientStatusDiscoveryServiceTest,XdsConfigDumpRouteError)11997 TEST_P(ClientStatusDiscoveryServiceTest, XdsConfigDumpRouteError) {
11998 int kFetchConfigRetries = 3;
11999 int kFetchIntervalMilliseconds = 200;
12000 SetNextResolution({});
12001 SetNextResolutionForLbChannelAllBalancers();
12002 EdsResourceArgs args({{"locality0", CreateEndpointsForBackends(0, 1)}});
12003 balancers_[0]->ads_service()->SetEdsResource(
12004 BuildEdsResource(args, DefaultEdsServiceName()));
12005 // Ensure the xDS resolver has working configs.
12006 CheckRpcSendOk();
12007 // Bad route config will be rejected.
12008 RouteConfiguration route_config;
12009 route_config.set_name(kDefaultRouteConfigurationName);
12010 route_config.add_virtual_hosts();
12011 SetRouteConfiguration(0, route_config);
12012 // The old xDS configs should still be effective.
12013 SetNextResolution({});
12014 SetNextResolutionForLbChannelAllBalancers();
12015 CheckRpcSendOk();
12016 for (int i = 0; i < kFetchConfigRetries; ++i) {
12017 auto csds_response = FetchCsdsResponse();
12018 bool ok = false;
12019 if (GetParam().enable_rds_testing()) {
12020 ok = ::testing::Value(
12021 csds_response.config(0).generic_xds_configs(),
12022 ::testing::Contains(EqGenericXdsConfig(
12023 kRdsTypeUrl, kDefaultRouteConfigurationName, /*version_info=*/"1",
12024 UnpackRouteConfiguration(EqRouteConfiguration(
12025 kDefaultRouteConfigurationName, kDefaultClusterName)),
12026 ClientResourceStatus::NACKED,
12027 EqUpdateFailureState(
12028 ::testing::HasSubstr("VirtualHost has no domains"), "2"))));
12029 } else {
12030 ok = ::testing::Value(
12031 csds_response.config(0).generic_xds_configs(),
12032 ::testing::Contains(EqGenericXdsConfig(
12033 kLdsTypeUrl, kServerName, /*version_info=*/"1",
12034 UnpackListener(EqListener(
12035 kServerName, EqNoRdsHCM(kDefaultRouteConfigurationName,
12036 kDefaultClusterName))),
12037 ClientResourceStatus::NACKED,
12038 EqUpdateFailureState(
12039 ::testing::HasSubstr("VirtualHost has no domains"), "2"))));
12040 }
12041 if (ok) return; // TEST PASSED!
12042 gpr_sleep_until(
12043 grpc_timeout_milliseconds_to_deadline(kFetchIntervalMilliseconds));
12044 }
12045 FAIL() << "error_state not seen in CSDS responses";
12046 }
12047
TEST_P(ClientStatusDiscoveryServiceTest,XdsConfigDumpClusterError)12048 TEST_P(ClientStatusDiscoveryServiceTest, XdsConfigDumpClusterError) {
12049 int kFetchConfigRetries = 3;
12050 int kFetchIntervalMilliseconds = 200;
12051 SetNextResolution({});
12052 SetNextResolutionForLbChannelAllBalancers();
12053 EdsResourceArgs args({{"locality0", CreateEndpointsForBackends(0, 1)}});
12054 balancers_[0]->ads_service()->SetEdsResource(
12055 BuildEdsResource(args, DefaultEdsServiceName()));
12056 // Ensure the xDS resolver has working configs.
12057 CheckRpcSendOk();
12058 // Listener without any route, will be rejected.
12059 Cluster cluster;
12060 cluster.set_name(kDefaultClusterName);
12061 balancers_[0]->ads_service()->SetCdsResource(cluster);
12062 // The old xDS configs should still be effective.
12063 SetNextResolution({});
12064 SetNextResolutionForLbChannelAllBalancers();
12065 CheckRpcSendOk();
12066 for (int i = 0; i < kFetchConfigRetries; ++i) {
12067 auto csds_response = FetchCsdsResponse();
12068 // Check if error state is propagated
12069 bool ok = ::testing::Value(
12070 csds_response.config(0).generic_xds_configs(),
12071 ::testing::Contains(EqGenericXdsConfig(
12072 kCdsTypeUrl, kDefaultClusterName, /*version_info=*/"1",
12073 UnpackCluster(EqCluster(kDefaultClusterName)),
12074 ClientResourceStatus::NACKED,
12075 EqUpdateFailureState(
12076 ::testing::HasSubstr("DiscoveryType not found"), "2"))));
12077 if (ok) return; // TEST PASSED!
12078 gpr_sleep_until(
12079 grpc_timeout_milliseconds_to_deadline(kFetchIntervalMilliseconds));
12080 }
12081 FAIL() << "error_state not seen in CSDS responses";
12082 }
12083
TEST_P(ClientStatusDiscoveryServiceTest,XdsConfigDumpEndpointError)12084 TEST_P(ClientStatusDiscoveryServiceTest, XdsConfigDumpEndpointError) {
12085 int kFetchConfigRetries = 3;
12086 int kFetchIntervalMilliseconds = 200;
12087 SetNextResolution({});
12088 SetNextResolutionForLbChannelAllBalancers();
12089 EdsResourceArgs args({{"locality0", CreateEndpointsForBackends(0, 1)}});
12090 balancers_[0]->ads_service()->SetEdsResource(
12091 BuildEdsResource(args, DefaultEdsServiceName()));
12092 // Ensure the xDS resolver has working configs.
12093 CheckRpcSendOk();
12094 // Bad endpoint config will be rejected.
12095 ClusterLoadAssignment cluster_load_assignment;
12096 cluster_load_assignment.set_cluster_name(kDefaultEdsServiceName);
12097 auto* endpoints = cluster_load_assignment.add_endpoints();
12098 endpoints->mutable_load_balancing_weight()->set_value(1);
12099 auto* endpoint = endpoints->add_lb_endpoints()->mutable_endpoint();
12100 endpoint->mutable_address()->mutable_socket_address()->set_port_value(1 << 1);
12101 balancers_[0]->ads_service()->SetEdsResource(cluster_load_assignment);
12102 // The old xDS configs should still be effective.
12103 SetNextResolution({});
12104 SetNextResolutionForLbChannelAllBalancers();
12105 CheckRpcSendOk();
12106 for (int i = 0; i < kFetchConfigRetries; ++i) {
12107 auto csds_response = FetchCsdsResponse();
12108 // Check if error state is propagated
12109 bool ok = ::testing::Value(
12110 csds_response.config(0).generic_xds_configs(),
12111 ::testing::Contains(EqGenericXdsConfig(
12112 kEdsTypeUrl, kDefaultEdsServiceName, /*version_info=*/"1",
12113 UnpackClusterLoadAssignment(EqClusterLoadAssignment(
12114 kDefaultEdsServiceName, backends_[0]->port(),
12115 kDefaultLocalityWeight)),
12116 ClientResourceStatus::NACKED,
12117 EqUpdateFailureState(::testing::HasSubstr("Empty locality"),
12118 "2"))));
12119 if (ok) return; // TEST PASSED!
12120 gpr_sleep_until(
12121 grpc_timeout_milliseconds_to_deadline(kFetchIntervalMilliseconds));
12122 }
12123 FAIL() << "error_state not seen in CSDS responses";
12124 }
12125
TEST_P(ClientStatusDiscoveryServiceTest,XdsConfigDumpListenerRequested)12126 TEST_P(ClientStatusDiscoveryServiceTest, XdsConfigDumpListenerRequested) {
12127 int kTimeoutMillisecond = 1000;
12128 balancers_[0]->ads_service()->UnsetResource(kLdsTypeUrl, kServerName);
12129 CheckRpcSendFailure(
12130 CheckRpcSendFailureOptions()
12131 .set_rpc_options(RpcOptions().set_timeout_ms(kTimeoutMillisecond))
12132 .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
12133 auto csds_response = FetchCsdsResponse();
12134 EXPECT_THAT(
12135 csds_response.config(0).generic_xds_configs(),
12136 ::testing::Contains(EqGenericXdsConfig(
12137 kLdsTypeUrl, kServerName, /*version_info=*/::testing::_, ::testing::_,
12138 ClientResourceStatus::REQUESTED, /*error_state=*/::testing::_)));
12139 }
12140
TEST_P(ClientStatusDiscoveryServiceTest,XdsConfigDumpClusterRequested)12141 TEST_P(ClientStatusDiscoveryServiceTest, XdsConfigDumpClusterRequested) {
12142 int kTimeoutMillisecond = 1000;
12143 std::string kClusterName1 = "cluster-1";
12144 std::string kClusterName2 = "cluster-2";
12145 SetNextResolution({});
12146 SetNextResolutionForLbChannelAllBalancers();
12147 // Create a route config requesting two non-existing clusters
12148 RouteConfiguration route_config;
12149 route_config.set_name(kDefaultRouteConfigurationName);
12150 auto* vh = route_config.add_virtual_hosts();
12151 // The VirtualHost must match the domain name, otherwise will cause resolver
12152 // transient failure.
12153 vh->add_domains("*");
12154 auto* routes1 = vh->add_routes();
12155 routes1->mutable_match()->set_prefix("");
12156 routes1->mutable_route()->set_cluster(kClusterName1);
12157 auto* routes2 = vh->add_routes();
12158 routes2->mutable_match()->set_prefix("");
12159 routes2->mutable_route()->set_cluster(kClusterName2);
12160 SetRouteConfiguration(0, route_config);
12161 // Try to get the configs plumb through
12162 CheckRpcSendFailure(
12163 CheckRpcSendFailureOptions()
12164 .set_rpc_options(RpcOptions().set_timeout_ms(kTimeoutMillisecond))
12165 .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
12166 auto csds_response = FetchCsdsResponse();
12167 EXPECT_THAT(csds_response.config(0).generic_xds_configs(),
12168 ::testing::AllOf(
12169 ::testing::Contains(EqGenericXdsConfig(
12170 kCdsTypeUrl, kClusterName1, /*version_info=*/::testing::_,
12171 ::testing::_, ClientResourceStatus::REQUESTED,
12172 /*error_state=*/::testing::_)),
12173 ::testing::Contains(EqGenericXdsConfig(
12174 kCdsTypeUrl, kClusterName2, /*version_info=*/::testing::_,
12175 ::testing::_, ClientResourceStatus::REQUESTED,
12176 /*error_state=*/::testing::_))));
12177 }
12178
12179 class CsdsShortAdsTimeoutTest : public ClientStatusDiscoveryServiceTest {
SetUp()12180 void SetUp() override {
12181 // Shorten the ADS subscription timeout to speed up the test run.
12182 xds_resource_does_not_exist_timeout_ms_ = 2000;
12183 ClientStatusDiscoveryServiceTest::SetUp();
12184 }
12185 };
12186
TEST_P(CsdsShortAdsTimeoutTest,XdsConfigDumpListenerDoesNotExist)12187 TEST_P(CsdsShortAdsTimeoutTest, XdsConfigDumpListenerDoesNotExist) {
12188 int kTimeoutMillisecond = 1000000; // 1000s wait for the transient failure.
12189 balancers_[0]->ads_service()->UnsetResource(kLdsTypeUrl, kServerName);
12190 CheckRpcSendFailure(
12191 CheckRpcSendFailureOptions()
12192 .set_rpc_options(RpcOptions().set_timeout_ms(kTimeoutMillisecond))
12193 .set_expected_error_code(grpc::UNAVAILABLE));
12194 auto csds_response = FetchCsdsResponse();
12195 EXPECT_THAT(
12196 csds_response.config(0).generic_xds_configs(),
12197 ::testing::Contains(EqGenericXdsConfig(
12198 kLdsTypeUrl, kServerName, /*version_info=*/::testing::_, ::testing::_,
12199 ClientResourceStatus::DOES_NOT_EXIST, /*error_state=*/::testing::_)));
12200 }
12201
TEST_P(CsdsShortAdsTimeoutTest,XdsConfigDumpRouteConfigDoesNotExist)12202 TEST_P(CsdsShortAdsTimeoutTest, XdsConfigDumpRouteConfigDoesNotExist) {
12203 if (!GetParam().enable_rds_testing()) return;
12204 int kTimeoutMillisecond = 1000000; // 1000s wait for the transient failure.
12205 SetNextResolution({});
12206 SetNextResolutionForLbChannelAllBalancers();
12207 balancers_[0]->ads_service()->UnsetResource(kRdsTypeUrl,
12208 kDefaultRouteConfigurationName);
12209 CheckRpcSendFailure(
12210 CheckRpcSendFailureOptions()
12211 .set_rpc_options(RpcOptions().set_timeout_ms(kTimeoutMillisecond))
12212 .set_expected_error_code(grpc::UNAVAILABLE));
12213 auto csds_response = FetchCsdsResponse();
12214 EXPECT_THAT(
12215 csds_response.config(0).generic_xds_configs(),
12216 ::testing::Contains(EqGenericXdsConfig(
12217 kRdsTypeUrl, kDefaultRouteConfigurationName,
12218 /*version_info=*/::testing::_, ::testing::_,
12219 ClientResourceStatus::DOES_NOT_EXIST, /*error_state=*/::testing::_)));
12220 }
12221
TEST_P(CsdsShortAdsTimeoutTest,XdsConfigDumpClusterDoesNotExist)12222 TEST_P(CsdsShortAdsTimeoutTest, XdsConfigDumpClusterDoesNotExist) {
12223 int kTimeoutMillisecond = 1000000; // 1000s wait for the transient failure.
12224 SetNextResolution({});
12225 SetNextResolutionForLbChannelAllBalancers();
12226 balancers_[0]->ads_service()->UnsetResource(kCdsTypeUrl, kDefaultClusterName);
12227 CheckRpcSendFailure(
12228 CheckRpcSendFailureOptions()
12229 .set_rpc_options(RpcOptions().set_timeout_ms(kTimeoutMillisecond))
12230 .set_expected_error_code(grpc::UNAVAILABLE));
12231 auto csds_response = FetchCsdsResponse();
12232 EXPECT_THAT(
12233 csds_response.config(0).generic_xds_configs(),
12234 ::testing::Contains(EqGenericXdsConfig(
12235 kCdsTypeUrl, kDefaultClusterName, /*version_info=*/::testing::_,
12236 ::testing::_, ClientResourceStatus::DOES_NOT_EXIST,
12237 /*error_state=*/::testing::_)));
12238 }
12239
TEST_P(CsdsShortAdsTimeoutTest,XdsConfigDumpEndpointDoesNotExist)12240 TEST_P(CsdsShortAdsTimeoutTest, XdsConfigDumpEndpointDoesNotExist) {
12241 int kTimeoutMillisecond = 1000000; // 1000s wait for the transient failure.
12242 SetNextResolution({});
12243 SetNextResolutionForLbChannelAllBalancers();
12244 balancers_[0]->ads_service()->UnsetResource(kEdsTypeUrl,
12245 kDefaultEdsServiceName);
12246 CheckRpcSendFailure(
12247 CheckRpcSendFailureOptions()
12248 .set_rpc_options(RpcOptions().set_timeout_ms(kTimeoutMillisecond))
12249 .set_expected_error_code(grpc::UNAVAILABLE));
12250 auto csds_response = FetchCsdsResponse();
12251 EXPECT_THAT(csds_response.config(0).generic_xds_configs(),
12252 ::testing::Contains(EqGenericXdsConfig(
12253 kEdsTypeUrl, kDefaultEdsServiceName,
12254 /*version_info=*/::testing::_, ::testing::_,
12255 ClientResourceStatus::DOES_NOT_EXIST,
12256 /*error_state=*/::testing::_)));
12257 }
12258
12259 #endif // DISABLED_XDS_PROTO_IN_CC
12260
TestTypeName(const::testing::TestParamInfo<TestType> & info)12261 std::string TestTypeName(const ::testing::TestParamInfo<TestType>& info) {
12262 return info.param.AsString();
12263 }
12264
12265 // Run with all combinations of xds/fake resolver and enabling load reporting.
12266 INSTANTIATE_TEST_SUITE_P(
12267 XdsTest, BasicTest,
12268 ::testing::Values(
12269 TestType(), TestType().set_enable_load_reporting(),
12270 TestType().set_use_fake_resolver(),
12271 TestType().set_use_fake_resolver().set_enable_load_reporting()),
12272 &TestTypeName);
12273
12274 // Run with both fake resolver and xds resolver.
12275 // Don't run with load reporting or v2 or RDS, since they are irrelevant to
12276 // the tests.
12277 INSTANTIATE_TEST_SUITE_P(XdsTest, SecureNamingTest,
12278 ::testing::Values(TestType(),
12279 TestType().set_use_fake_resolver()),
12280 &TestTypeName);
12281
12282 // LDS depends on XdsResolver.
12283 INSTANTIATE_TEST_SUITE_P(XdsTest, LdsTest, ::testing::Values(TestType()),
12284 &TestTypeName);
12285 INSTANTIATE_TEST_SUITE_P(XdsTest, LdsV2Test,
12286 ::testing::Values(TestType().set_use_v2()),
12287 &TestTypeName);
12288
12289 // LDS/RDS commmon tests depend on XdsResolver.
12290 INSTANTIATE_TEST_SUITE_P(
12291 XdsTest, LdsRdsTest,
12292 ::testing::Values(TestType(), TestType().set_enable_rds_testing(),
12293 // Also test with xDS v2.
12294 TestType().set_enable_rds_testing().set_use_v2()),
12295 &TestTypeName);
12296
12297 // CDS depends on XdsResolver.
12298 INSTANTIATE_TEST_SUITE_P(
12299 XdsTest, CdsTest,
12300 ::testing::Values(TestType(), TestType().set_enable_load_reporting()),
12301 &TestTypeName);
12302
12303 // CDS depends on XdsResolver.
12304 // Security depends on v3.
12305 // Not enabling load reporting or RDS, since those are irrelevant to these
12306 // tests.
12307 INSTANTIATE_TEST_SUITE_P(
12308 XdsTest, XdsSecurityTest,
12309 ::testing::Values(TestType().set_use_xds_credentials()), &TestTypeName);
12310
12311 // We are only testing the server here.
12312 // Run with bootstrap from env var, so that we use a global XdsClient
12313 // instance. Otherwise, we would need to use a separate fake resolver
12314 // result generator on the client and server sides.
12315 INSTANTIATE_TEST_SUITE_P(XdsTest, XdsEnabledServerTest,
12316 ::testing::Values(TestType().set_bootstrap_source(
12317 TestType::kBootstrapFromEnvVar)),
12318 &TestTypeName);
12319
12320 // We are only testing the server here.
12321 INSTANTIATE_TEST_SUITE_P(XdsTest, XdsServerSecurityTest,
12322 ::testing::Values(TestType()
12323 .set_use_fake_resolver()
12324 .set_use_xds_credentials()),
12325 &TestTypeName);
12326
12327 // We are only testing the server here.
12328 INSTANTIATE_TEST_SUITE_P(XdsTest, XdsEnabledServerStatusNotificationTest,
12329 ::testing::Values(TestType()
12330 .set_use_fake_resolver()
12331 .set_use_xds_credentials()),
12332 &TestTypeName);
12333
12334 // We are only testing the server here.
12335 INSTANTIATE_TEST_SUITE_P(XdsTest, XdsServerFilterChainMatchTest,
12336 ::testing::Values(TestType()
12337 .set_use_fake_resolver()
12338 .set_use_xds_credentials()),
12339 &TestTypeName);
12340
12341 // We are only testing the server here.
12342 // TODO(yashykt): Also add a test type with set_enable_rds_testing() once we
12343 // start fetching non-inline resources.
12344 INSTANTIATE_TEST_SUITE_P(XdsTest, XdsServerRdsTest,
12345 ::testing::Values(TestType()
12346 .set_use_fake_resolver()
12347 .set_use_xds_credentials()),
12348 &TestTypeName);
12349
12350 // EDS could be tested with or without XdsResolver, but the tests would
12351 // be the same either way, so we test it only with XdsResolver.
12352 INSTANTIATE_TEST_SUITE_P(
12353 XdsTest, EdsTest,
12354 ::testing::Values(TestType(), TestType().set_enable_load_reporting()),
12355 &TestTypeName);
12356
12357 // Test initial resource timeouts for each resource type.
12358 // Do this only for XdsResolver with RDS enabled, so that we can test
12359 // all resource types.
12360 // Run with V3 only, since the functionality is no different in V2.
12361 INSTANTIATE_TEST_SUITE_P(XdsTest, TimeoutTest,
12362 ::testing::Values(TestType().set_enable_rds_testing()),
12363 &TestTypeName);
12364
12365 // XdsResolverOnlyTest depends on XdsResolver.
12366 INSTANTIATE_TEST_SUITE_P(
12367 XdsTest, XdsResolverOnlyTest,
12368 ::testing::Values(TestType(), TestType().set_enable_load_reporting()),
12369 &TestTypeName);
12370
12371 // Runs with bootstrap from env var, so that there's a global XdsClient.
12372 INSTANTIATE_TEST_SUITE_P(
12373 XdsTest, GlobalXdsClientTest,
12374 ::testing::Values(
12375 TestType().set_bootstrap_source(TestType::kBootstrapFromEnvVar),
12376 TestType()
12377 .set_bootstrap_source(TestType::kBootstrapFromEnvVar)
12378 .set_enable_load_reporting()),
12379 &TestTypeName);
12380
12381 // XdsResolverLoadReprtingOnlyTest depends on XdsResolver and load reporting.
12382 INSTANTIATE_TEST_SUITE_P(
12383 XdsTest, XdsResolverLoadReportingOnlyTest,
12384 ::testing::Values(TestType().set_enable_load_reporting()), &TestTypeName);
12385
12386 INSTANTIATE_TEST_SUITE_P(
12387 XdsTest, LocalityMapTest,
12388 ::testing::Values(
12389 TestType(), TestType().set_enable_load_reporting(),
12390 TestType().set_use_fake_resolver(),
12391 TestType().set_use_fake_resolver().set_enable_load_reporting()),
12392 &TestTypeName);
12393
12394 INSTANTIATE_TEST_SUITE_P(
12395 XdsTest, FailoverTest,
12396 ::testing::Values(
12397 TestType(), TestType().set_enable_load_reporting(),
12398 TestType().set_use_fake_resolver(),
12399 TestType().set_use_fake_resolver().set_enable_load_reporting()),
12400 &TestTypeName);
12401
12402 INSTANTIATE_TEST_SUITE_P(
12403 XdsTest, DropTest,
12404 ::testing::Values(
12405 TestType(), TestType().set_enable_load_reporting(),
12406 TestType().set_use_fake_resolver(),
12407 TestType().set_use_fake_resolver().set_enable_load_reporting()),
12408 &TestTypeName);
12409
12410 INSTANTIATE_TEST_SUITE_P(
12411 XdsTest, BalancerUpdateTest,
12412 ::testing::Values(
12413 TestType().set_use_fake_resolver(),
12414 TestType().set_use_fake_resolver().set_enable_load_reporting(),
12415 TestType().set_enable_load_reporting()),
12416 &TestTypeName);
12417
12418 // Load reporting tests are not run with load reporting disabled.
12419 INSTANTIATE_TEST_SUITE_P(
12420 XdsTest, ClientLoadReportingTest,
12421 ::testing::Values(
12422 TestType().set_enable_load_reporting(),
12423 TestType().set_enable_load_reporting().set_use_fake_resolver()),
12424 &TestTypeName);
12425
12426 // Load reporting tests are not run with load reporting disabled.
12427 INSTANTIATE_TEST_SUITE_P(
12428 XdsTest, ClientLoadReportingWithDropTest,
12429 ::testing::Values(
12430 TestType().set_enable_load_reporting(),
12431 TestType().set_enable_load_reporting().set_use_fake_resolver()),
12432 &TestTypeName);
12433
12434 INSTANTIATE_TEST_SUITE_P(
12435 XdsTest, FaultInjectionTest,
12436 ::testing::Values(
12437 TestType(), TestType().set_enable_rds_testing(),
12438 TestType().set_filter_config_setup(
12439 TestType::FilterConfigSetup::kRouteOverride),
12440 TestType().set_enable_rds_testing().set_filter_config_setup(
12441 TestType::FilterConfigSetup::kRouteOverride)),
12442 &TestTypeName);
12443
12444 INSTANTIATE_TEST_SUITE_P(
12445 XdsTest, BootstrapSourceTest,
12446 ::testing::Values(
12447 TestType().set_bootstrap_source(TestType::kBootstrapFromEnvVar),
12448 TestType().set_bootstrap_source(TestType::kBootstrapFromFile)),
12449 &TestTypeName);
12450
12451 #ifndef DISABLED_XDS_PROTO_IN_CC
12452 // Run CSDS tests with RDS enabled and disabled.
12453 // These need to run with the bootstrap from an env var instead of from
12454 // a channel arg, since there needs to be a global XdsClient instance.
12455 INSTANTIATE_TEST_SUITE_P(
12456 XdsTest, ClientStatusDiscoveryServiceTest,
12457 ::testing::Values(
12458 TestType().set_bootstrap_source(TestType::kBootstrapFromEnvVar),
12459 TestType()
12460 .set_bootstrap_source(TestType::kBootstrapFromEnvVar)
12461 .set_enable_rds_testing(),
12462 TestType()
12463 .set_bootstrap_source(TestType::kBootstrapFromEnvVar)
12464 .set_use_csds_streaming(),
12465 TestType()
12466 .set_bootstrap_source(TestType::kBootstrapFromEnvVar)
12467 .set_enable_rds_testing()
12468 .set_use_csds_streaming()),
12469 &TestTypeName);
12470 INSTANTIATE_TEST_SUITE_P(
12471 XdsTest, CsdsShortAdsTimeoutTest,
12472 ::testing::Values(
12473 TestType().set_bootstrap_source(TestType::kBootstrapFromEnvVar),
12474 TestType()
12475 .set_bootstrap_source(TestType::kBootstrapFromEnvVar)
12476 .set_enable_rds_testing(),
12477 TestType()
12478 .set_bootstrap_source(TestType::kBootstrapFromEnvVar)
12479 .set_use_csds_streaming(),
12480 TestType()
12481 .set_bootstrap_source(TestType::kBootstrapFromEnvVar)
12482 .set_enable_rds_testing()
12483 .set_use_csds_streaming()),
12484 &TestTypeName);
12485 #endif // DISABLED_XDS_PROTO_IN_CC
12486
12487 } // namespace
12488 } // namespace testing
12489 } // namespace grpc
12490
main(int argc,char ** argv)12491 int main(int argc, char** argv) {
12492 grpc::testing::TestEnvironment env(argc, argv);
12493 ::testing::InitGoogleTest(&argc, argv);
12494 grpc::testing::WriteBootstrapFiles();
12495 // Make the backup poller poll very frequently in order to pick up
12496 // updates from all the subchannels's FDs.
12497 GPR_GLOBAL_CONFIG_SET(grpc_client_channel_backup_poll_interval_ms, 1);
12498 #if TARGET_OS_IPHONE
12499 // Workaround Apple CFStream bug
12500 gpr_setenv("grpc_cfstream", "0");
12501 #endif
12502 grpc_core::CertificateProviderRegistry::RegisterCertificateProviderFactory(
12503 absl::make_unique<grpc::testing::FakeCertificateProviderFactory>(
12504 "fake1", &grpc::testing::g_fake1_cert_data_map));
12505 grpc_core::CertificateProviderRegistry::RegisterCertificateProviderFactory(
12506 absl::make_unique<grpc::testing::FakeCertificateProviderFactory>(
12507 "fake2", &grpc::testing::g_fake2_cert_data_map));
12508 grpc_init();
12509 grpc_core::XdsHttpFilterRegistry::RegisterFilter(
12510 absl::make_unique<grpc::testing::NoOpHttpFilter>(
12511 "grpc.testing.client_only_http_filter",
12512 /* supported_on_clients = */ true, /* supported_on_servers = */ false,
12513 /* is_terminal_filter */ false),
12514 {"grpc.testing.client_only_http_filter"});
12515 grpc_core::XdsHttpFilterRegistry::RegisterFilter(
12516 absl::make_unique<grpc::testing::NoOpHttpFilter>(
12517 "grpc.testing.server_only_http_filter",
12518 /* supported_on_clients = */ false, /* supported_on_servers = */ true,
12519 /* is_terminal_filter */ false),
12520 {"grpc.testing.server_only_http_filter"});
12521 grpc_core::XdsHttpFilterRegistry::RegisterFilter(
12522 absl::make_unique<grpc::testing::NoOpHttpFilter>(
12523 "grpc.testing.terminal_http_filter",
12524 /* supported_on_clients = */ true, /* supported_on_servers = */ true,
12525 /* is_terminal_filter */ true),
12526 {"grpc.testing.terminal_http_filter"});
12527 const auto result = RUN_ALL_TESTS();
12528 grpc_shutdown();
12529 return result;
12530 }
12531