1 /*
2 Copyright (c) 2018, 2020, Oracle and/or its affiliates.
3
4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the GNU General Public License, version 2.0,
6 as published by the Free Software Foundation.
7
8 This program is also distributed with certain software (including
9 but not limited to OpenSSL) that is licensed under separate terms,
10 as designated in a particular file or component or in included license
11 documentation. The authors of MySQL hereby grant you an additional
12 permission to link the program and your derivative works with the
13 separately licensed software that they have included with MySQL.
14
15 This program is distributed in the hope that it will be useful,
16 but WITHOUT ANY WARRANTY; without even the implied warranty of
17 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 GNU General Public License for more details.
19
20 You should have received a copy of the GNU General Public License
21 along with this program; if not, write to the Free Software
22 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
23 */
24
25 #include <fstream>
26 #include <stdexcept>
27 #include <thread>
28
29 #include <gmock/gmock.h>
30
31 #ifdef RAPIDJSON_NO_SIZETYPEDEFINE
32 // if we build within the server, it will set RAPIDJSON_NO_SIZETYPEDEFINE
33 // globally and require to include my_rapidjson_size_t.h
34 #include "my_rapidjson_size_t.h"
35 #endif
36 #include <rapidjson/document.h>
37
38 #include "dim.h"
39 #include "mock_server_rest_client.h"
40 #include "mock_server_testutils.h"
41 #include "mysql_session.h"
42 #include "mysqlrouter/cluster_metadata.h"
43 #include "mysqlrouter/rest_client.h"
44 #include "random_generator.h"
45 #include "rest_metadata_client.h"
46 #include "router_component_test.h"
47 #include "router_component_testutils.h"
48 #include "tcp_port_pool.h"
49
50 #define ASSERT_NO_ERROR(expr) \
51 ASSERT_THAT(expr, ::testing::Eq(std::error_code{}))
52
53 using mysqlrouter::ClusterType;
54 using mysqlrouter::MySQLSession;
55 using namespace std::chrono_literals;
56 static constexpr const char kMockServerConnectionsUri[] =
57 "/api/v1/mock_server/connections/";
58
59 static const std::string kRestApiUsername("someuser");
60 static const std::string kRestApiPassword("somepass");
61
62 class ConfigGenerator {
63 std::map<std::string, std::string> defaults_;
64 std::string config_dir_;
65
66 std::string metadata_cache_section_;
67 std::string routing_primary_section_;
68 std::string routing_secondary_section_;
69 std::string monitoring_section_;
70
71 std::vector<uint16_t> metadata_server_ports_;
72 uint16_t router_rw_port_;
73 uint16_t router_ro_port_;
74 uint16_t monitoring_port_;
75 std::string disconnect_on_metadata_unavailable_ =
76 "&disconnect_on_metadata_unavailable=no";
77 std::string disconnect_on_promoted_to_primary_ =
78 "&disconnect_on_promoted_to_primary=no";
79
80 std::chrono::milliseconds metadata_refresh_ttl_;
81
82 public:
ConfigGenerator(const std::map<std::string,std::string> & defaults,const std::string & config_dir,const std::vector<uint16_t> metadata_server_ports,uint16_t router_rw_port,uint16_t router_ro_port,uint16_t monitoring_port,std::chrono::milliseconds metadata_refresh_ttl)83 ConfigGenerator(const std::map<std::string, std::string> &defaults,
84 const std::string &config_dir,
85 const std::vector<uint16_t> metadata_server_ports,
86 uint16_t router_rw_port, uint16_t router_ro_port,
87 uint16_t monitoring_port,
88 std::chrono::milliseconds metadata_refresh_ttl)
89 : defaults_(defaults),
90 config_dir_(config_dir),
91 metadata_server_ports_(metadata_server_ports),
92 router_rw_port_(router_rw_port),
93 router_ro_port_(router_ro_port),
94 monitoring_port_{monitoring_port},
95 metadata_refresh_ttl_{metadata_refresh_ttl} {}
96
disconnect_on_metadata_unavailable(const std::string & value)97 void disconnect_on_metadata_unavailable(const std::string &value) {
98 disconnect_on_metadata_unavailable_ = value;
99 }
100
disconnect_on_promoted_to_primary(const std::string & value)101 void disconnect_on_promoted_to_primary(const std::string &value) {
102 disconnect_on_promoted_to_primary_ = value;
103 }
104
105 // void metadata_refresh_ttl(unsigned ttl) { metadata_refresh_ttl_ = ttl; }
106
add_metadata_cache_section(std::chrono::milliseconds ttl,ClusterType cluster_type=ClusterType::GR_V2)107 void add_metadata_cache_section(
108 std::chrono::milliseconds ttl,
109 ClusterType cluster_type = ClusterType::GR_V2) {
110 // NOT: Those tests are using bootstrap_server_addresses in the static
111 // configuration which is now moved to the dynamic state file. This way we
112 // are testing the backward compatibility of the old
113 // bootstrap_server_addresses still working. If this is ever changed to
114 // use
115 // dynamic state file, a new test should be added to test that
116 // bootstrap_server_addresses is still handled properly.
117 const std::string cluster_type_str =
118 (cluster_type == ClusterType::RS_V2) ? "ar" : "gr";
119 metadata_cache_section_ =
120 "[logger]\n"
121 "level = DEBUG\n\n"
122
123 "[metadata_cache:test]\n"
124 "cluster_type=" +
125 cluster_type_str +
126 "\n"
127 "router_id=1\n"
128 "bootstrap_server_addresses=";
129 size_t i = 0;
130 for (uint16_t port : metadata_server_ports_) {
131 metadata_cache_section_ += "mysql://127.0.0.1:" + std::to_string(port);
132 if (i < metadata_server_ports_.size() - 1) metadata_cache_section_ += ",";
133 }
134 metadata_cache_section_ +=
135 "\n"
136 "user=mysql_router1_user\n"
137 "metadata_cluster=test\n"
138 "connect_timeout=1\n"
139 "ttl=" +
140 std::to_string(ttl.count() / 1000.0) + "\n\n";
141 }
142
get_metadata_cache_routing_section(uint16_t router_port,const std::string & role,const std::string & strategy,bool is_rw=false)143 std::string get_metadata_cache_routing_section(uint16_t router_port,
144 const std::string &role,
145 const std::string &strategy,
146 bool is_rw = false) {
147 std::string result;
148
149 if (is_rw) {
150 result =
151 "[routing:test_default_rw]\n"
152 "bind_port=" +
153 std::to_string(router_port) + "\n" +
154 "destinations=metadata-cache://test/default?role=" + role +
155 disconnect_on_metadata_unavailable_ + "\n" + "protocol=classic\n";
156 } else {
157 result =
158 "[routing:test_default_ro]\n"
159 "bind_port=" +
160 std::to_string(router_port) + "\n" +
161 "destinations=metadata-cache://test/default?role=" + role +
162 disconnect_on_metadata_unavailable_ +
163 disconnect_on_promoted_to_primary_ + "\n" + "protocol=classic\n";
164 }
165
166 if (!strategy.empty())
167 result += std::string("routing_strategy=" + strategy + "\n");
168
169 return result;
170 }
171
add_routing_primary_section()172 void add_routing_primary_section() {
173 routing_primary_section_ = get_metadata_cache_routing_section(
174 router_rw_port_, "PRIMARY", "round-robin", true);
175 }
176
add_routing_secondary_section()177 void add_routing_secondary_section() {
178 routing_secondary_section_ = get_metadata_cache_routing_section(
179 router_ro_port_, "SECONDARY", "round-robin", false);
180 }
181
add_routing_primary_and_secondary_section()182 void add_routing_primary_and_secondary_section() {
183 routing_secondary_section_ = get_metadata_cache_routing_section(
184 router_ro_port_, "PRIMARY_AND_SECONDARY", "round-robin", false);
185 }
186
add_monitoring_section(const std::string & config_dir)187 void add_monitoring_section(const std::string &config_dir) {
188 std::string passwd_filename =
189 mysql_harness::Path(config_dir).join("users").str();
190
191 monitoring_section_ =
192 "[rest_api]\n"
193 "[rest_metadata_cache]\n"
194 "require_realm=somerealm\n"
195 "[http_auth_realm:somerealm]\n"
196 "backend=somebackend\n"
197 "method=basic\n"
198 "name=somename\n"
199 "[http_auth_backend:somebackend]\n"
200 "backend=file\n"
201 "filename=" +
202 passwd_filename +
203 "\n"
204 "[http_server]\n"
205 "port=" +
206 std::to_string(monitoring_port_) + "\n";
207 }
208
make_DEFAULT_section(const std::map<std::string,std::string> * params)209 std::string make_DEFAULT_section(
210 const std::map<std::string, std::string> *params) {
211 auto l = [params](const char *key) -> std::string {
212 return (params->count(key))
213 ? std::string(key) + " = " + params->at(key) + "\n"
214 : "";
215 };
216
217 return std::string("[DEFAULT]\n") + l("logging_folder") +
218 l("plugin_folder") + l("runtime_folder") + l("config_folder") +
219 l("data_folder") + l("keyring_path") + l("master_key_path") +
220 l("master_key_reader") + l("master_key_writer") + "\n";
221 }
222
create_config_file(const std::map<std::string,std::string> * params,const std::string & directory)223 std::string create_config_file(
224 const std::map<std::string, std::string> *params,
225 const std::string &directory) {
226 Path file_path = Path(directory).join("mysqlrouter.conf");
227 std::ofstream ofs_config(file_path.str());
228
229 if (!ofs_config.good()) {
230 throw(std::runtime_error("Could not create config file " +
231 file_path.str()));
232 }
233
234 ofs_config << make_DEFAULT_section(params);
235 ofs_config << metadata_cache_section_ << routing_primary_section_
236 << routing_secondary_section_ << monitoring_section_
237 << std::endl;
238 ofs_config.close();
239
240 return file_path.str();
241 }
242
build_config_file(const std::string & temp_test_dir,ClusterType cluster_type,bool is_primary_and_secondary=false)243 std::string build_config_file(const std::string &temp_test_dir,
244 ClusterType cluster_type,
245 bool is_primary_and_secondary = false) {
246 add_metadata_cache_section(metadata_refresh_ttl_, cluster_type);
247 add_routing_primary_section();
248 add_monitoring_section(temp_test_dir);
249
250 if (is_primary_and_secondary) {
251 add_routing_primary_and_secondary_section();
252 } else {
253 add_routing_secondary_section();
254 }
255
256 init_keyring(defaults_, temp_test_dir);
257
258 return create_config_file(&defaults_, config_dir_);
259 }
260 };
261
262 class RouterRoutingConnectionCommonTest : public RouterComponentTest {
263 public:
SetUp()264 void SetUp() override {
265 RouterComponentTest::SetUp();
266
267 mysql_harness::DIM &dim = mysql_harness::DIM::instance();
268 // RandomGenerator
269 dim.set_RandomGenerator(
270 []() {
271 static mysql_harness::RandomGenerator rg;
272 return &rg;
273 },
274 [](mysql_harness::RandomGeneratorInterface *) {});
275 #if 1
276 {
277 auto &cmd = launch_command(
278 ProcessManager::get_origin().join("mysqlrouter_passwd").str(),
279 {"set",
280 mysql_harness::Path(temp_test_dir_.name()).join("users").str(),
281 kRestApiUsername},
282 EXIT_SUCCESS, true);
283 cmd.register_response("Please enter password", kRestApiPassword + "\n");
284 EXPECT_EQ(cmd.wait_for_exit(), 0);
285 }
286 #endif
287
288 cluster_nodes_ports_ = {
289 port_pool_.get_next_available(), // first is PRIMARY
290 port_pool_.get_next_available(), port_pool_.get_next_available(),
291 port_pool_.get_next_available(), port_pool_.get_next_available()};
292
293 cluster_nodes_http_ports_ = {
294 port_pool_.get_next_available(), port_pool_.get_next_available(),
295 port_pool_.get_next_available(), port_pool_.get_next_available(),
296 port_pool_.get_next_available()};
297
298 router_rw_port_ = port_pool_.get_next_available();
299 router_ro_port_ = port_pool_.get_next_available();
300 monitoring_port_ = port_pool_.get_next_available();
301
302 config_generator_.reset(new ConfigGenerator(
303 get_DEFAULT_defaults(), temp_conf_dir_.name(),
304 {cluster_nodes_ports_[0]}, router_rw_port_, router_ro_port_,
305 monitoring_port_, metadata_refresh_ttl_));
306
307 mock_http_hostname_ = "127.0.0.1";
308 mock_http_uri_ = kMockServerGlobalsRestUri;
309 }
310
launch_router(uint16_t,const std::string & config_file)311 auto &launch_router(uint16_t /* router_port */,
312 const std::string &config_file) {
313 return ProcessManager::launch_router({"-c", config_file});
314 }
315
launch_server(uint16_t cluster_port,const std::string & json_file,uint16_t http_port,size_t number_of_servers=5)316 auto &launch_server(uint16_t cluster_port, const std::string &json_file,
317 uint16_t http_port, size_t number_of_servers = 5) {
318 auto &cluster_node = ProcessManager::launch_mysql_server_mock(
319 get_data_dir().join(json_file).str(), cluster_port, EXIT_SUCCESS, false,
320 http_port);
321
322 std::vector<uint16_t> nodes_ports;
323 nodes_ports.resize(number_of_servers);
324 std::copy_n(cluster_nodes_ports_.begin(), number_of_servers,
325 nodes_ports.begin());
326
327 EXPECT_TRUE(MockServerRestClient(http_port).wait_for_rest_endpoint_ready());
328 set_mock_metadata(http_port, "", nodes_ports);
329 return cluster_node;
330 }
331
setup_cluster(const std::string & js_for_primary,unsigned number_of_servers,uint16_t=0)332 void setup_cluster(const std::string &js_for_primary,
333 unsigned number_of_servers, uint16_t /*my_port*/ = 0) {
334 // launch cluster nodes
335 for (unsigned port = 0; port < number_of_servers; ++port) {
336 const std::string js_file =
337 port == 0 ? js_for_primary : "rest_server_mock_with_gr.js";
338 cluster_nodes_.push_back(
339 &launch_server(cluster_nodes_ports_[port], js_file,
340 cluster_nodes_http_ports_[port], number_of_servers));
341 ASSERT_NO_FATAL_FAILURE(
342 check_port_ready(*cluster_nodes_[port], cluster_nodes_ports_[port]));
343 }
344 }
345
346 struct server_globals {
347 bool primary_removed{false};
348 bool primary_failover{false};
349 bool secondary_failover{false};
350 bool secondary_removed{false};
351 bool cluster_partition{false};
352 bool MD_failed{false};
353 bool GR_primary_failed{false};
354 bool GR_health_failed{false};
355
set_primary_removedRouterRoutingConnectionCommonTest::server_globals356 server_globals &set_primary_removed() {
357 primary_removed = true;
358 return *this;
359 }
set_primary_failoverRouterRoutingConnectionCommonTest::server_globals360 server_globals &set_primary_failover() {
361 primary_failover = true;
362 return *this;
363 }
set_secondary_failoverRouterRoutingConnectionCommonTest::server_globals364 server_globals &set_secondary_failover() {
365 secondary_failover = true;
366 return *this;
367 }
set_secondary_removedRouterRoutingConnectionCommonTest::server_globals368 server_globals &set_secondary_removed() {
369 secondary_removed = true;
370 return *this;
371 }
set_cluster_partitionRouterRoutingConnectionCommonTest::server_globals372 server_globals &set_cluster_partition() {
373 cluster_partition = true;
374 return *this;
375 }
set_MD_failedRouterRoutingConnectionCommonTest::server_globals376 server_globals &set_MD_failed() {
377 MD_failed = true;
378 return *this;
379 }
set_GR_primary_failedRouterRoutingConnectionCommonTest::server_globals380 server_globals &set_GR_primary_failed() {
381 GR_primary_failed = true;
382 return *this;
383 }
set_GR_health_failedRouterRoutingConnectionCommonTest::server_globals384 server_globals &set_GR_health_failed() {
385 GR_health_failed = true;
386 return *this;
387 }
388 };
389
set_additional_globals(uint16_t http_port,const server_globals & globals)390 void set_additional_globals(uint16_t http_port,
391 const server_globals &globals) {
392 auto json_doc = mock_GR_metadata_as_json("", cluster_nodes_ports_);
393 JsonAllocator allocator;
394 json_doc.AddMember("primary_removed", globals.primary_removed, allocator);
395 json_doc.AddMember("primary_failover", globals.primary_failover, allocator);
396 json_doc.AddMember("secondary_failover", globals.secondary_failover,
397 allocator);
398 json_doc.AddMember("secondary_removed", globals.secondary_removed,
399 allocator);
400 json_doc.AddMember("cluster_partition", globals.cluster_partition,
401 allocator);
402 json_doc.AddMember("MD_failed", globals.MD_failed, allocator);
403 json_doc.AddMember("GR_primary_failed", globals.GR_primary_failed,
404 allocator);
405 json_doc.AddMember("GR_health_failed", globals.GR_health_failed, allocator);
406 const auto json_str = json_to_string(json_doc);
407 EXPECT_NO_THROW(MockServerRestClient(http_port).set_globals(json_str));
408 }
409
410 TcpPortPool port_pool_;
411 std::chrono::milliseconds metadata_refresh_ttl_{100};
412 std::chrono::milliseconds wait_for_cache_ready_timeout{
413 metadata_refresh_ttl_ + std::chrono::milliseconds(5000)};
414 std::chrono::milliseconds wait_for_cache_update_timeout{
415 metadata_refresh_ttl_ * 20};
416 std::unique_ptr<ConfigGenerator> config_generator_;
417 TempDirectory temp_test_dir_;
418 TempDirectory temp_conf_dir_;
419 std::vector<uint16_t> cluster_nodes_ports_;
420 std::vector<uint16_t> cluster_nodes_http_ports_;
421 std::vector<ProcessWrapper *> cluster_nodes_;
422 uint16_t router_rw_port_;
423 uint16_t router_ro_port_;
424 uint16_t monitoring_port_;
425
426 // http properties
427 std::string mock_http_hostname_;
428 std::string mock_http_uri_;
429 };
430
431 class RouterRoutingConnectionTest : public RouterRoutingConnectionCommonTest {};
432
vec_from_lines(const std::string & s)433 static std::vector<std::string> vec_from_lines(const std::string &s) {
434 std::vector<std::string> lines;
435 std::istringstream lines_stream{s};
436
437 for (std::string line; std::getline(lines_stream, line);) {
438 lines.push_back(line);
439 }
440
441 return lines;
442 }
443
444 /**
445 * @test
446 * Verify connections through router fail if metadata's schema-version is
447 * too old.
448 */
TEST_F(RouterRoutingConnectionTest,OldSchemaVersion)449 TEST_F(RouterRoutingConnectionTest, OldSchemaVersion) {
450 // preparation
451 //
452 SCOPED_TRACE("// [prep] creating router config");
453 TempDirectory tmp_dir;
454 config_generator_.reset(new ConfigGenerator(
455 get_DEFAULT_defaults(), tmp_dir.name(), {cluster_nodes_ports_[0]},
456 router_rw_port_, router_ro_port_, monitoring_port_,
457 metadata_refresh_ttl_));
458
459 SCOPED_TRACE("// [prep] launch the primary node on port " +
460 std::to_string(cluster_nodes_ports_.at(0)) +
461 " working also as metadata server");
462
463 cluster_nodes_.push_back(&launch_server(cluster_nodes_ports_.at(0),
464 "metadata_old_schema.js",
465 cluster_nodes_http_ports_[0]));
466
467 SCOPED_TRACE("// [prep] wait until mock-servers are started");
468 ASSERT_NO_FATAL_FAILURE(
469 check_port_ready(*cluster_nodes_[0], cluster_nodes_ports_[0]));
470
471 SCOPED_TRACE("// [prep] launching router");
472 auto &router = launch_router(router_rw_port_,
473 config_generator_->build_config_file(
474 temp_test_dir_.name(), ClusterType::GR_V2));
475 ASSERT_NO_FATAL_FAILURE(check_port_ready(router, router_rw_port_));
476
477 SCOPED_TRACE("// [prep] waiting " +
478 std::to_string(wait_for_cache_ready_timeout.count()) +
479 "ms until metadata is initialized (and failed)");
480 RestMetadataClient::MetadataStatus metadata_status;
481 RestMetadataClient rest_metadata_client(mock_http_hostname_, monitoring_port_,
482 kRestApiUsername, kRestApiPassword);
483
484 ASSERT_NO_ERROR(rest_metadata_client.wait_for_cache_fetched(
485 wait_for_cache_ready_timeout, metadata_status,
486 [](const RestMetadataClient::MetadataStatus &cur) {
487 return cur.refresh_failed > 0;
488 }));
489
490 // testing
491 //
492 SCOPED_TRACE("// [test] expect connecting clients to fail");
493 MySQLSession client;
494
495 ASSERT_THROW(client.connect("127.0.0.1", router_rw_port_, "username",
496 "password", "", ""),
497 MySQLSession::Error);
498
499 SCOPED_TRACE("// [test] expect router log to contain error message");
500
501 // posix RE has [0-9], but no \\d
502 // simple RE has \\d, but no [0-9]
503 constexpr const char log_msg_re[]{
504 #ifdef GTEST_USES_POSIX_RE
505 "Unsupported metadata schema on .*\\. Expected Metadata Schema version "
506 "compatible to [0-9]\\.[0-9]\\.[0-9], [0-9]\\.[0-9]\\.[0-9], got "
507 "0\\.0\\.1"
508 #else
509 "Unsupported metadata schema on .*\\. Expected Metadata Schema version "
510 "compatible to \\d\\.\\d\\.\\d, \\d\\.\\d\\.\\d, got 0\\.0\\.1"
511 #endif
512 };
513
514 ASSERT_THAT(vec_from_lines(router.get_full_logfile()),
515 ::testing::Contains(::testing::ContainsRegex(log_msg_re)));
516 }
517
518 /**
519 * @test
520 * Verify that router doesn't start when
521 * disconnect_on_promoted_to_primary has invalid value.
522 */
TEST_F(RouterRoutingConnectionTest,IsRouterFailToStartWhen_disconnect_on_promoted_to_primary_invalid)523 TEST_F(RouterRoutingConnectionTest,
524 IsRouterFailToStartWhen_disconnect_on_promoted_to_primary_invalid) {
525 config_generator_->disconnect_on_promoted_to_primary(
526 "&disconnect_on_promoted_to_primary=bogus");
527 auto &router = ProcessManager::launch_router(
528 {"-c", config_generator_->build_config_file(temp_test_dir_.name(),
529 ClusterType::GR_V2)},
530 EXIT_FAILURE);
531 check_port_not_ready(router, router_ro_port_);
532 }
533
534 /**
535 * @test
536 * Verify that router doesn't start when
537 * disconnect_on_metadata_unavailable has invalid value.
538 */
TEST_F(RouterRoutingConnectionTest,IsRouterFailToStartWhen_disconnect_on_metadata_unavailable_invalid)539 TEST_F(RouterRoutingConnectionTest,
540 IsRouterFailToStartWhen_disconnect_on_metadata_unavailable_invalid) {
541 config_generator_->disconnect_on_metadata_unavailable(
542 "&disconnect_on_metadata_unavailable=bogus");
543 auto &router = ProcessManager::launch_router(
544 {"-c", config_generator_->build_config_file(temp_test_dir_.name(),
545 ClusterType::GR_V2)},
546 EXIT_FAILURE);
547 check_port_not_ready(router, router_ro_port_);
548 }
549
550 struct TracefileTestParam {
551 std::string tracefile;
552 ClusterType cluster_type;
553 std::string param{};
554
TracefileTestParamTracefileTestParam555 TracefileTestParam(const std::string tracefile_,
556 const ClusterType cluster_type_,
557 const std::string param_ = "")
558 : tracefile(tracefile_), cluster_type(cluster_type_), param(param_) {}
559 };
560
561 class IsConnectionsClosedWhenPrimaryRemovedFromClusterTest
562 : public RouterRoutingConnectionTest,
563 public ::testing::WithParamInterface<TracefileTestParam> {};
564
565 /**
566 * @test
567 * Verify that all connections to Primary are closed when Primary is
568 * removed from Cluster
569 */
TEST_P(IsConnectionsClosedWhenPrimaryRemovedFromClusterTest,IsConnectionsClosedWhenPrimaryRemovedFromCluster)570 TEST_P(IsConnectionsClosedWhenPrimaryRemovedFromClusterTest,
571 IsConnectionsClosedWhenPrimaryRemovedFromCluster) {
572 TempDirectory tmp_dir("conf");
573 const std::string tracefile = GetParam().tracefile;
574
575 config_generator_.reset(new ConfigGenerator(
576 get_DEFAULT_defaults(), tmp_dir.name(),
577 {cluster_nodes_ports_[0], cluster_nodes_ports_[1]}, router_rw_port_,
578 router_ro_port_, monitoring_port_, metadata_refresh_ttl_));
579
580 SCOPED_TRACE("// launch the primary node on port " +
581 std::to_string(cluster_nodes_ports_.at(0)) +
582 " working also as metadata server");
583 cluster_nodes_.push_back(&launch_server(cluster_nodes_ports_[0], tracefile,
584 cluster_nodes_http_ports_[0], 4));
585
586 SCOPED_TRACE("// launch the secondary node on port " +
587 std::to_string(cluster_nodes_ports_.at(1)) +
588 " working also as metadata server");
589 cluster_nodes_.push_back(&launch_server(cluster_nodes_ports_[1], tracefile,
590 cluster_nodes_http_ports_[1], 4));
591
592 SCOPED_TRACE("// launch the rest of secondary cluster nodes");
593 for (unsigned port = 2; port < 4; ++port) {
594 cluster_nodes_.push_back(
595 &launch_server(cluster_nodes_ports_[port], tracefile,
596 cluster_nodes_http_ports_[port], 4));
597 }
598
599 SCOPED_TRACE("// wait until mock-servers are started");
600 for (unsigned ndx = 0; ndx < 4; ndx++) {
601 ASSERT_NO_FATAL_FAILURE(
602 check_port_ready(*cluster_nodes_.at(ndx), cluster_nodes_ports_[ndx]));
603 }
604
605 SCOPED_TRACE("// launching router");
606 auto &router = launch_router(
607 router_rw_port_, config_generator_->build_config_file(
608 temp_test_dir_.name(), GetParam().cluster_type));
609 ASSERT_NO_FATAL_FAILURE(check_port_ready(router, router_rw_port_));
610
611 SCOPED_TRACE("// waiting " +
612 std::to_string(wait_for_cache_ready_timeout.count()) +
613 "ms until metadata is initialized");
614 RestMetadataClient::MetadataStatus metadata_status;
615 RestMetadataClient rest_metadata_client(mock_http_hostname_, monitoring_port_,
616 kRestApiUsername, kRestApiPassword);
617
618 ASSERT_NO_ERROR(rest_metadata_client.wait_for_cache_ready(
619 wait_for_cache_ready_timeout, metadata_status));
620
621 SCOPED_TRACE("// connecting clients");
622 std::vector<std::pair<MySQLSession, uint16_t>> clients(2);
623
624 for (auto &client_and_port : clients) {
625 auto &client = client_and_port.first;
626 ASSERT_NO_FATAL_FAILURE(client.connect("127.0.0.1", router_rw_port_,
627 "username", "password", "", ""));
628 std::unique_ptr<MySQLSession::ResultRow> result{
629 client.query_one("select @@port")};
630 client_and_port.second =
631 static_cast<uint16_t>(std::stoul(std::string((*result)[0])));
632 }
633
634 set_additional_globals(cluster_nodes_http_ports_[0],
635 server_globals().set_primary_removed());
636
637 ASSERT_NO_ERROR(rest_metadata_client.wait_for_cache_updated(
638 wait_for_cache_update_timeout, metadata_status));
639
640 // verify that connections to PRIMARY are broken
641 for (auto &client_and_port : clients) {
642 auto &client = client_and_port.first;
643 EXPECT_TRUE(wait_connection_dropped(client));
644 }
645 }
646
647 INSTANTIATE_TEST_SUITE_P(
648 IsConnectionsClosedWhenPrimaryRemovedFromCluster,
649 IsConnectionsClosedWhenPrimaryRemovedFromClusterTest,
650 ::testing::Values(
651 TracefileTestParam(
652 "metadata_3_secondaries_server_removed_from_cluster_v2_gr.js",
653 ClusterType::GR_V2),
654 TracefileTestParam(
655 "metadata_3_secondaries_server_removed_from_cluster.js",
656 ClusterType::GR_V1)));
657
658 class IsConnectionsClosedWhenSecondaryRemovedFromClusterTest
659 : public RouterRoutingConnectionTest,
660 public ::testing::WithParamInterface<TracefileTestParam> {};
661
662 /**
663 * @test
664 * Verify that all connections to Secondary are closed when Secondary is
665 * removed from Cluster.
666 *
667 */
TEST_P(IsConnectionsClosedWhenSecondaryRemovedFromClusterTest,IsConnectionsClosedWhenSecondaryRemovedFromCluster)668 TEST_P(IsConnectionsClosedWhenSecondaryRemovedFromClusterTest,
669 IsConnectionsClosedWhenSecondaryRemovedFromCluster) {
670 const std::string tracefile = GetParam().tracefile;
671 ASSERT_NO_FATAL_FAILURE(setup_cluster(tracefile, 4));
672
673 auto &router = launch_router(
674 router_rw_port_, config_generator_->build_config_file(
675 temp_test_dir_.name(), GetParam().cluster_type));
676 ASSERT_NO_FATAL_FAILURE(check_port_ready(router, router_rw_port_));
677 ASSERT_NO_FATAL_FAILURE(check_port_ready(router, router_ro_port_));
678
679 /*
680 * wait until metadata is initialized
681 */
682 RestMetadataClient::MetadataStatus metadata_status;
683 RestMetadataClient rest_metadata_client(mock_http_hostname_, monitoring_port_,
684 kRestApiUsername, kRestApiPassword);
685
686 ASSERT_NO_ERROR(rest_metadata_client.wait_for_cache_ready(
687 wait_for_cache_ready_timeout, metadata_status));
688
689 // connect clients
690 std::vector<std::pair<MySQLSession, uint16_t>> clients(6);
691
692 for (auto &client_and_port : clients) {
693 auto &client = client_and_port.first;
694 ASSERT_NO_FATAL_FAILURE(client.connect("127.0.0.1", router_ro_port_,
695 "username", "password", "", ""));
696 std::unique_ptr<MySQLSession::ResultRow> result{
697 client.query_one("select @@port")};
698 client_and_port.second =
699 static_cast<uint16_t>(std::stoul(std::string((*result)[0])));
700 }
701
702 set_additional_globals(cluster_nodes_http_ports_[0],
703 server_globals().set_secondary_removed());
704
705 ASSERT_NO_ERROR(rest_metadata_client.wait_for_cache_updated(
706 wait_for_cache_update_timeout, metadata_status));
707
708 // verify that connections to SECONDARY_1 are broken
709 for (auto &client_and_port : clients) {
710 auto &client = client_and_port.first;
711 uint16_t port = client_and_port.second;
712
713 if (port == cluster_nodes_ports_[1])
714 EXPECT_TRUE(wait_connection_dropped(client));
715 else
716 ASSERT_NO_THROW(std::unique_ptr<MySQLSession::ResultRow> result{
717 client.query_one("select @@port")});
718 }
719 }
720
721 INSTANTIATE_TEST_SUITE_P(
722 IsConnectionsClosedWhenSecondaryRemovedFromCluster,
723 IsConnectionsClosedWhenSecondaryRemovedFromClusterTest,
724 ::testing::Values(
725 TracefileTestParam(
726 "metadata_3_secondaries_server_removed_from_cluster_v2_gr.js",
727 ClusterType::GR_V2),
728 TracefileTestParam(
729 "metadata_3_secondaries_server_removed_from_cluster.js",
730 ClusterType::GR_V1)));
731
732 class IsRWConnectionsClosedWhenPrimaryFailoverTest
733 : public RouterRoutingConnectionTest,
734 public ::testing::WithParamInterface<TracefileTestParam> {};
735
736 /**
737 * @test
738 * Verify that when Primary is demoted, then all RW connections
739 * to that server are closed.
740 */
TEST_P(IsRWConnectionsClosedWhenPrimaryFailoverTest,IsRWConnectionsClosedWhenPrimaryFailover)741 TEST_P(IsRWConnectionsClosedWhenPrimaryFailoverTest,
742 IsRWConnectionsClosedWhenPrimaryFailover) {
743 const std::string tracefile = GetParam().tracefile;
744
745 ASSERT_NO_FATAL_FAILURE(setup_cluster(tracefile, 4));
746 auto &router = launch_router(
747 router_ro_port_, config_generator_->build_config_file(
748 temp_test_dir_.name(), GetParam().cluster_type));
749 ASSERT_NO_FATAL_FAILURE(check_port_ready(router, router_rw_port_));
750
751 /*
752 * wait until metadata is initialized
753 */
754 RestMetadataClient::MetadataStatus metadata_status;
755 RestMetadataClient rest_metadata_client(mock_http_hostname_, monitoring_port_,
756 kRestApiUsername, kRestApiPassword);
757
758 ASSERT_NO_ERROR(rest_metadata_client.wait_for_cache_ready(
759 wait_for_cache_ready_timeout, metadata_status));
760
761 // connect clients
762 std::vector<std::pair<MySQLSession, uint16_t>> clients(2);
763
764 for (auto &client_and_port : clients) {
765 auto &client = client_and_port.first;
766 ASSERT_NO_THROW(client.connect("127.0.0.1", router_rw_port_, "username",
767 "password", "", ""));
768 std::unique_ptr<MySQLSession::ResultRow> result{
769 client.query_one("select @@port")};
770 client_and_port.second =
771 static_cast<uint16_t>(std::stoul(std::string((*result)[0])));
772 }
773
774 set_additional_globals(cluster_nodes_http_ports_[0],
775 server_globals().set_primary_failover());
776
777 ASSERT_NO_ERROR(rest_metadata_client.wait_for_cache_updated(
778 wait_for_cache_update_timeout, metadata_status));
779
780 // verify if RW connections to PRIMARY are closed
781 for (auto &client_and_port : clients) {
782 auto &client = client_and_port.first;
783 EXPECT_TRUE(wait_connection_dropped(client));
784 }
785 }
786
787 INSTANTIATE_TEST_SUITE_P(
788 IsRWConnectionsClosedWhenPrimaryFailover,
789 IsRWConnectionsClosedWhenPrimaryFailoverTest,
790 ::testing::Values(
791 TracefileTestParam("metadata_3_secondaries_primary_failover_v2_gr.js",
792 ClusterType::GR_V2),
793 TracefileTestParam("metadata_3_secondaries_primary_failover.js",
794 ClusterType::GR_V1)));
795
796 class IsROConnectionsKeptWhenPrimaryFailoverTest
797 : public RouterRoutingConnectionTest,
798 public ::testing::WithParamInterface<TracefileTestParam> {};
799
800 /**
801 * @test
802 * Verify that when Primary is demoted, then RO connections
803 * to that server are kept.
804 */
TEST_P(IsROConnectionsKeptWhenPrimaryFailoverTest,IsROConnectionsKeptWhenPrimaryFailover)805 TEST_P(IsROConnectionsKeptWhenPrimaryFailoverTest,
806 IsROConnectionsKeptWhenPrimaryFailover) {
807 const std::string tracefile = GetParam().tracefile;
808
809 ASSERT_NO_FATAL_FAILURE(setup_cluster(tracefile, 4));
810
811 config_generator_->disconnect_on_promoted_to_primary("");
812
813 auto &router =
814 launch_router(router_ro_port_,
815 config_generator_->build_config_file(
816 temp_test_dir_.name(), GetParam().cluster_type, true));
817 ASSERT_NO_FATAL_FAILURE(check_port_ready(router, router_rw_port_));
818 ASSERT_NO_FATAL_FAILURE(check_port_ready(router, router_ro_port_));
819
820 /*
821 * wait until metadata is initialized
822 */
823 RestMetadataClient::MetadataStatus metadata_status;
824 RestMetadataClient rest_metadata_client(mock_http_hostname_, monitoring_port_,
825 kRestApiUsername, kRestApiPassword);
826
827 ASSERT_NO_ERROR(rest_metadata_client.wait_for_cache_ready(
828 wait_for_cache_ready_timeout, metadata_status));
829
830 // connect clients
831 std::vector<std::pair<MySQLSession, uint16_t>> clients(4);
832
833 for (auto &client_and_port : clients) {
834 auto &client = client_and_port.first;
835 ASSERT_NO_THROW(client.connect("127.0.0.1", router_ro_port_, "username",
836 "password", "", ""));
837 std::unique_ptr<MySQLSession::ResultRow> result{
838 client.query_one("select @@port")};
839 client_and_port.second =
840 static_cast<uint16_t>(std::stoul(std::string((*result)[0])));
841 }
842
843 set_additional_globals(cluster_nodes_http_ports_[0],
844 server_globals().set_primary_failover());
845
846 ASSERT_NO_ERROR(rest_metadata_client.wait_for_cache_updated(
847 wait_for_cache_update_timeout, metadata_status));
848
849 // verify that RO connections to PRIMARY are kept
850 for (auto &client_and_port : clients) {
851 auto &client = client_and_port.first;
852 ASSERT_NO_THROW(std::unique_ptr<MySQLSession::ResultRow> result{
853 client.query_one("select @@port")});
854 }
855 }
856
857 INSTANTIATE_TEST_SUITE_P(
858 IsROConnectionsKeptWhenPrimaryFailover,
859 IsROConnectionsKeptWhenPrimaryFailoverTest,
860 ::testing::Values(
861 TracefileTestParam("metadata_3_secondaries_primary_failover_v2_gr.js",
862 ClusterType::GR_V2),
863 TracefileTestParam("metadata_3_secondaries_primary_failover.js",
864 ClusterType::GR_V1)));
865
866 class RouterRoutingConnectionPromotedTest
867 : public RouterRoutingConnectionCommonTest,
868 public testing::WithParamInterface<TracefileTestParam> {};
869
870 /**
871 * @test
872 * Verify that when server is promoted from Secondary to Primary and
873 * disconnect_on_promoted_to_primary is set to 'no' (default value) then
874 * connections to that server are not closed.
875 */
TEST_P(RouterRoutingConnectionPromotedTest,IsConnectionsToSecondaryKeptWhenPromotedToPrimary)876 TEST_P(RouterRoutingConnectionPromotedTest,
877 IsConnectionsToSecondaryKeptWhenPromotedToPrimary) {
878 const std::string tracefile = GetParam().tracefile;
879 const std::string param = GetParam().param;
880
881 ASSERT_NO_FATAL_FAILURE(setup_cluster(tracefile, 4));
882
883 config_generator_->disconnect_on_promoted_to_primary(param);
884
885 auto &router = launch_router(
886 router_ro_port_, config_generator_->build_config_file(
887 temp_test_dir_.name(), GetParam().cluster_type));
888 ASSERT_NO_FATAL_FAILURE(check_port_ready(router, router_rw_port_));
889 ASSERT_NO_FATAL_FAILURE(check_port_ready(router, router_ro_port_));
890
891 /*
892 * wait until metadata is initialized
893 */
894 RestMetadataClient::MetadataStatus metadata_status;
895 RestMetadataClient rest_metadata_client(mock_http_hostname_, monitoring_port_,
896 kRestApiUsername, kRestApiPassword);
897
898 ASSERT_NO_ERROR(rest_metadata_client.wait_for_cache_ready(
899 wait_for_cache_ready_timeout, metadata_status));
900
901 // connect clients
902 std::vector<std::pair<MySQLSession, uint16_t>> clients(6);
903
904 for (auto &client_and_port : clients) {
905 auto &client = client_and_port.first;
906 ASSERT_NO_THROW(client.connect("127.0.0.1", router_ro_port_, "username",
907 "password", "", ""));
908 std::unique_ptr<MySQLSession::ResultRow> result{
909 client.query_one("select @@port")};
910 client_and_port.second =
911 static_cast<uint16_t>(std::stoul(std::string((*result)[0])));
912 }
913
914 server_globals globals;
915 globals.primary_failover = true;
916 set_additional_globals(cluster_nodes_http_ports_[0], globals);
917 ASSERT_NO_ERROR(rest_metadata_client.wait_for_cache_updated(
918 wait_for_cache_update_timeout, metadata_status));
919
920 // verify that connections to SECONDARY_1 are kept (all RO connections
921 // should NOT be closed)
922 for (auto &client_and_port : clients) {
923 auto &client = client_and_port.first;
924 ASSERT_NO_THROW(std::unique_ptr<MySQLSession::ResultRow> result{
925 client.query_one("select @@port")});
926 }
927 }
928
929 INSTANTIATE_TEST_SUITE_P(
930 RouterRoutingIsConnectionNotClosedWhenPromoted,
931 RouterRoutingConnectionPromotedTest,
932 ::testing::Values(
933 TracefileTestParam("metadata_3_secondaries_primary_failover_v2_gr.js",
934 ClusterType::GR_V2,
935 "&disconnect_on_promoted_to_primary=no"),
936 TracefileTestParam("metadata_3_secondaries_primary_failover.js",
937 ClusterType::GR_V1,
938 "&disconnect_on_promoted_to_primary=no"),
939 TracefileTestParam("metadata_3_secondaries_primary_failover_v2_gr.js",
940 ClusterType::GR_V2, ""),
941 TracefileTestParam("metadata_3_secondaries_primary_failover.js",
942 ClusterType::GR_V1, "")));
943
944 class IsConnectionToSecondaryClosedWhenPromotedToPrimaryTest
945 : public RouterRoutingConnectionTest,
946 public ::testing::WithParamInterface<TracefileTestParam> {};
947
948 /**
949 * @test
950 * Verify that when server is promoted from Secondary to Primary and
951 * disconnect_on_promoted_to_primary is set to 'yes' then connections
952 * to that server are closed.
953 */
TEST_P(IsConnectionToSecondaryClosedWhenPromotedToPrimaryTest,IsConnectionToSecondaryClosedWhenPromotedToPrimary)954 TEST_P(IsConnectionToSecondaryClosedWhenPromotedToPrimaryTest,
955 IsConnectionToSecondaryClosedWhenPromotedToPrimary) {
956 const std::string tracefile = GetParam().tracefile;
957
958 ASSERT_NO_FATAL_FAILURE(setup_cluster(tracefile, 4));
959
960 config_generator_->disconnect_on_promoted_to_primary(
961 "&disconnect_on_promoted_to_primary=yes");
962
963 auto &router = launch_router(
964 router_ro_port_, config_generator_->build_config_file(
965 temp_test_dir_.name(), GetParam().cluster_type));
966 ASSERT_NO_FATAL_FAILURE(check_port_ready(router, router_ro_port_));
967
968 /*
969 * wait until metadata is initialized
970 */
971 RestMetadataClient::MetadataStatus metadata_status;
972 RestMetadataClient rest_metadata_client(mock_http_hostname_, monitoring_port_,
973 kRestApiUsername, kRestApiPassword);
974
975 ASSERT_NO_ERROR(rest_metadata_client.wait_for_cache_ready(
976 wait_for_cache_ready_timeout, metadata_status));
977
978 // connect clients
979 std::vector<std::pair<MySQLSession, uint16_t>> clients(6);
980
981 for (auto &client_and_port : clients) {
982 auto &client = client_and_port.first;
983 ASSERT_NO_THROW(client.connect("127.0.0.1", router_ro_port_, "username",
984 "password", "", ""));
985 std::unique_ptr<MySQLSession::ResultRow> result{
986 client.query_one("select @@port")};
987 client_and_port.second =
988 static_cast<uint16_t>(std::stoul(std::string((*result)[0])));
989 }
990
991 set_additional_globals(cluster_nodes_http_ports_[0],
992 server_globals().set_primary_failover());
993 ASSERT_NO_ERROR(rest_metadata_client.wait_for_cache_updated(
994 wait_for_cache_update_timeout, metadata_status));
995
996 // verify that connections to SECONDARY_1 are closed
997 for (auto &client_and_port : clients) {
998 auto &client = client_and_port.first;
999 uint16_t port = client_and_port.second;
1000
1001 if (port == cluster_nodes_ports_[1])
1002 EXPECT_TRUE(wait_connection_dropped(client));
1003 else
1004 ASSERT_NO_THROW(std::unique_ptr<MySQLSession::ResultRow> result{
1005 client.query_one("select @@port")});
1006 }
1007 }
1008
1009 INSTANTIATE_TEST_SUITE_P(
1010 IsConnectionToSecondaryClosedWhenPromotedToPrimary,
1011 IsConnectionToSecondaryClosedWhenPromotedToPrimaryTest,
1012 ::testing::Values(
1013 TracefileTestParam("metadata_3_secondaries_primary_failover_v2_gr.js",
1014 ClusterType::GR_V2),
1015 TracefileTestParam("metadata_3_secondaries_primary_failover.js",
1016 ClusterType::GR_V1)));
1017
1018 class IsConnectionToMinorityClosedWhenClusterPartitionTest
1019 : public RouterRoutingConnectionTest,
1020 public ::testing::WithParamInterface<TracefileTestParam> {};
1021
1022 /**
1023 * @test
1024 * Verify that when GR is partitioned, then connections to servers that
1025 * are not in majority are closed.
1026 */
TEST_P(IsConnectionToMinorityClosedWhenClusterPartitionTest,IsConnectionToMinorityClosedWhenClusterPartition)1027 TEST_P(IsConnectionToMinorityClosedWhenClusterPartitionTest,
1028 IsConnectionToMinorityClosedWhenClusterPartition) {
1029 const std::string tracefile = GetParam().tracefile;
1030
1031 /*
1032 * create cluster with 5 servers
1033 */
1034 ASSERT_NO_FATAL_FAILURE(setup_cluster(tracefile, 5));
1035
1036 auto &router = launch_router(
1037 router_ro_port_, config_generator_->build_config_file(
1038 temp_test_dir_.name(), GetParam().cluster_type));
1039 ASSERT_NO_FATAL_FAILURE(check_port_ready(router, router_ro_port_));
1040
1041 /*
1042 * wait until metadata is initialized
1043 */
1044 RestMetadataClient::MetadataStatus metadata_status;
1045 RestMetadataClient rest_metadata_client(mock_http_hostname_, monitoring_port_,
1046 kRestApiUsername, kRestApiPassword);
1047
1048 ASSERT_NO_ERROR(rest_metadata_client.wait_for_cache_ready(
1049 wait_for_cache_ready_timeout, metadata_status));
1050
1051 // connect clients
1052 std::vector<std::pair<MySQLSession, uint16_t>> clients(10);
1053
1054 // connect clients to Primary
1055 for (size_t i = 0; i < 2; ++i) {
1056 auto &client = clients[i].first;
1057 ASSERT_NO_FATAL_FAILURE(client.connect("127.0.0.1", router_rw_port_,
1058 "username", "password", "", ""));
1059 std::unique_ptr<MySQLSession::ResultRow> result{
1060 client.query_one("select @@port")};
1061 clients[i].second =
1062 static_cast<uint16_t>(std::stoul(std::string((*result)[0])));
1063 }
1064
1065 // connect clients to Secondaries
1066 for (size_t i = 2; i < 10; ++i) {
1067 auto &client = clients[i].first;
1068 ASSERT_NO_THROW(client.connect("127.0.0.1", router_ro_port_, "username",
1069 "password", "", ""));
1070 std::unique_ptr<MySQLSession::ResultRow> result{
1071 client.query_one("select @@port")};
1072 clients[i].second =
1073 static_cast<uint16_t>(std::stoul(std::string((*result)[0])));
1074 }
1075
1076 /*
1077 * Partition the cluster:
1078 * - 2 servers are ONLINE: Primary and Secondary_1
1079 * - 3 servers are OFFLINE: Secondary_2, Secondary_3, Secondary_4
1080 *
1081 * Connetions to OFFLINE servers should be closed.
1082 * Since only 2 servers are ONLINE (minority) connections to them should be
1083 * closed as well.
1084 */
1085 set_additional_globals(cluster_nodes_http_ports_[0],
1086 server_globals().set_cluster_partition());
1087
1088 ASSERT_NO_ERROR(rest_metadata_client.wait_for_cache_changed(
1089 wait_for_cache_update_timeout, metadata_status));
1090
1091 /*
1092 * Connections to servers that are offline should be closed
1093 * Connections to servers in minority should be closed as well
1094 */
1095 for (auto &client_and_port : clients) {
1096 auto &client = client_and_port.first;
1097 EXPECT_TRUE(wait_connection_dropped(client));
1098 }
1099 }
1100
1101 INSTANTIATE_TEST_SUITE_P(
1102 IsConnectionToMinorityClosedWhenClusterPartition,
1103 IsConnectionToMinorityClosedWhenClusterPartitionTest,
1104 ::testing::Values(
1105 TracefileTestParam("metadata_4_secondaries_partitioning_v2_gr.js",
1106 ClusterType::GR_V2),
1107 TracefileTestParam("metadata_4_secondaries_partitioning.js",
1108 ClusterType::GR_V1)));
1109
1110 class IsConnectionClosedWhenClusterOverloadedTest
1111 : public RouterRoutingConnectionCommonTest,
1112 public testing::WithParamInterface<TracefileTestParam> {};
1113
1114 /**
1115 * @test
1116 * Verity that when GR is overloaded and
1117 * disconnect_on_metadata_unavailable is set to 'yes' then all connection to
1118 * GR are closed
1119 */
TEST_P(IsConnectionClosedWhenClusterOverloadedTest,IsConnectionClosedWhenClusterOverloaded)1120 TEST_P(IsConnectionClosedWhenClusterOverloadedTest,
1121 IsConnectionClosedWhenClusterOverloaded) {
1122 const std::string tracefile = GetParam().tracefile;
1123
1124 ASSERT_NO_FATAL_FAILURE(setup_cluster(tracefile, 4));
1125
1126 config_generator_->disconnect_on_metadata_unavailable(
1127 "&disconnect_on_metadata_unavailable=yes");
1128
1129 auto &router = launch_router(
1130 router_ro_port_, config_generator_->build_config_file(
1131 temp_test_dir_.name(), GetParam().cluster_type));
1132 ASSERT_NO_FATAL_FAILURE(check_port_ready(router, router_ro_port_));
1133
1134 /*
1135 * wait until metadata is initialized
1136 */
1137 RestMetadataClient::MetadataStatus metadata_status;
1138 RestMetadataClient rest_metadata_client(mock_http_hostname_, monitoring_port_,
1139 kRestApiUsername, kRestApiPassword);
1140
1141 ASSERT_NO_ERROR(rest_metadata_client.wait_for_cache_ready(
1142 wait_for_cache_ready_timeout, metadata_status));
1143
1144 // connect clients
1145 std::vector<std::pair<MySQLSession, uint16_t>> clients(6);
1146
1147 for (auto &client_and_port : clients) {
1148 auto &client = client_and_port.first;
1149 ASSERT_NO_FATAL_FAILURE(client.connect("127.0.0.1", router_ro_port_,
1150 "username", "password", "", ""));
1151
1152 std::unique_ptr<MySQLSession::ResultRow> result{
1153 client.query_one("select @@port")};
1154 client_and_port.second =
1155 static_cast<uint16_t>(std::stoul(std::string((*result)[0])));
1156 }
1157
1158 /*
1159 * There is only 1 metadata server, so then primary
1160 * goes away, metadata is unavailable.
1161 */
1162 MockServerRestClient(cluster_nodes_http_ports_[0])
1163 .send_delete(kMockServerConnectionsUri);
1164 cluster_nodes_[0]->kill();
1165 ASSERT_NO_ERROR(rest_metadata_client.wait_for_cache_changed(
1166 wait_for_cache_update_timeout, metadata_status));
1167
1168 // verify that all connections are closed
1169 for (auto &client_and_port : clients) {
1170 auto &client = client_and_port.first;
1171 EXPECT_TRUE(wait_connection_dropped(client));
1172 }
1173 }
1174
1175 INSTANTIATE_TEST_SUITE_P(
1176 IsConnectionClosedWhenClusterOverloaded,
1177 IsConnectionClosedWhenClusterOverloadedTest,
1178 ::testing::Values(TracefileTestParam("metadata_3_secondaries_pass_v2_gr.js",
1179 ClusterType::GR_V2),
1180 TracefileTestParam("metadata_3_secondaries_pass.js",
1181 ClusterType::GR_V1)));
1182
1183 class RouterRoutingConnectionMDUnavailableTest
1184 : public RouterRoutingConnectionCommonTest,
1185 public testing::WithParamInterface<TracefileTestParam> {};
1186
1187 /**
1188 * @test
1189 * Verify that when GR is overloaded and
1190 * disconnect_on_metadata_unavailable is set to 'no' (default value) then
1191 * connections to GR are NOT closed.
1192 */
TEST_P(RouterRoutingConnectionMDUnavailableTest,IsConnectionKeptWhenClusterOverloaded)1193 TEST_P(RouterRoutingConnectionMDUnavailableTest,
1194 IsConnectionKeptWhenClusterOverloaded) {
1195 const std::string tracefile = GetParam().tracefile;
1196 const std::string param = GetParam().param;
1197
1198 ASSERT_NO_FATAL_FAILURE(setup_cluster(tracefile, 4));
1199
1200 config_generator_->disconnect_on_promoted_to_primary(param);
1201 auto &router = launch_router(
1202 router_ro_port_, config_generator_->build_config_file(
1203 temp_test_dir_.name(), GetParam().cluster_type));
1204 ASSERT_NO_FATAL_FAILURE(check_port_ready(router, router_rw_port_));
1205 ASSERT_NO_FATAL_FAILURE(check_port_ready(router, router_ro_port_));
1206
1207 /*
1208 * wait until metadata is initialized
1209 */
1210 RestMetadataClient::MetadataStatus metadata_status;
1211 RestMetadataClient rest_metadata_client(mock_http_hostname_, monitoring_port_,
1212 kRestApiUsername, kRestApiPassword);
1213
1214 ASSERT_NO_ERROR(rest_metadata_client.wait_for_cache_ready(
1215 wait_for_cache_ready_timeout, metadata_status));
1216
1217 // connect clients
1218 std::vector<std::pair<MySQLSession, uint16_t>> clients(6);
1219
1220 for (auto &client_and_port : clients) {
1221 auto &client = client_and_port.first;
1222 ASSERT_NO_THROW(client.connect("127.0.0.1", router_ro_port_, "username",
1223 "password", "", ""));
1224 std::unique_ptr<MySQLSession::ResultRow> result{
1225 client.query_one("select @@port")};
1226 client_and_port.second =
1227 static_cast<uint16_t>(std::stoul(std::string((*result)[0])));
1228 }
1229
1230 /*
1231 * There is only 1 metadata server, so then primary
1232 * goes away, metadata is unavailable.
1233 */
1234 MockServerRestClient(cluster_nodes_http_ports_[0])
1235 .send_delete(kMockServerConnectionsUri);
1236 cluster_nodes_[0]->kill();
1237 ASSERT_NO_ERROR(rest_metadata_client.wait_for_cache_changed(
1238 wait_for_cache_update_timeout, metadata_status));
1239
1240 // verify if all connections are NOT closed
1241 for (auto &client_and_port : clients) {
1242 auto &client = client_and_port.first;
1243 ASSERT_NO_THROW(std::unique_ptr<MySQLSession::ResultRow> result{
1244 client.query_one("select @@port")});
1245 }
1246 }
1247
1248 INSTANTIATE_TEST_SUITE_P(
1249 RouterRoutingIsConnectionNotClosedWhenMDUnavailable,
1250 RouterRoutingConnectionMDUnavailableTest,
1251 ::testing::Values(
1252 TracefileTestParam("metadata_3_secondaries_pass_v2_gr.js",
1253 ClusterType::GR_V2,
1254 "&disconnect_on_metadata_unavailable=no"),
1255 TracefileTestParam("metadata_3_secondaries_pass.js", ClusterType::GR_V1,
1256 "&disconnect_on_metadata_unavailable=no"),
1257 TracefileTestParam("metadata_3_secondaries_pass_v2_gr.js",
1258 ClusterType::GR_V2, ""),
1259 TracefileTestParam("metadata_3_secondaries_pass.js", ClusterType::GR_V1,
1260 "")));
1261
1262 using server_globals = RouterRoutingConnectionCommonTest::server_globals;
1263
1264 struct MDRefreshTestParam {
1265 ClusterType cluster_type;
1266 std::string tracefile1;
1267 std::string tracefile2;
1268 server_globals globals;
1269
MDRefreshTestParamMDRefreshTestParam1270 MDRefreshTestParam(ClusterType cluster_type_, std::string tracefile1_,
1271 std::string tracefile2_, server_globals globals_)
1272 : cluster_type(cluster_type_),
1273 tracefile1(tracefile1_),
1274 tracefile2(tracefile2_),
1275 globals(globals_) {}
1276 };
1277
1278 class RouterRoutingConnectionMDRefreshTest
1279 : public RouterRoutingConnectionCommonTest,
1280 public testing::WithParamInterface<MDRefreshTestParam> {};
1281
1282 /**
1283 * @test
1284 * Verify if connections are not closed when fetching metadata from
1285 * current metadata server fails, but fetching from subsequent metadata server
1286 * passes.
1287 *
1288 * 1. Start cluster with 1 Primary and 4 Secondary
1289 * 2. Establish 2 RW connections and 8 RO connections
1290 * 3. Fetching MD from Primary fails
1291 * 4. Fetching MD from Secondary passes
1292 * 5. Check if connections are still open.
1293 */
TEST_P(RouterRoutingConnectionMDRefreshTest,IsConnectionNotClosedWhenRrefreshFailedForParticularMDServer)1294 TEST_P(RouterRoutingConnectionMDRefreshTest,
1295 IsConnectionNotClosedWhenRrefreshFailedForParticularMDServer) {
1296 /*
1297 * Primary and first secondary are metadata-cache servers
1298 */
1299 TempDirectory temp_dir("conf");
1300 config_generator_.reset(new ConfigGenerator(
1301 get_DEFAULT_defaults(), temp_dir.name(),
1302 {cluster_nodes_ports_[0], cluster_nodes_ports_[1]}, router_rw_port_,
1303 router_ro_port_, monitoring_port_, metadata_refresh_ttl_));
1304
1305 // launch the primary node working also as metadata server
1306 cluster_nodes_.push_back(&launch_server(cluster_nodes_ports_[0],
1307 GetParam().tracefile1,
1308 cluster_nodes_http_ports_[0], 4));
1309
1310 // launch the secondary node working also as metadata server
1311 cluster_nodes_.push_back(&launch_server(cluster_nodes_ports_[1],
1312 GetParam().tracefile2,
1313 cluster_nodes_http_ports_[1], 4));
1314
1315 // launch the rest of secondary cluster nodes
1316 for (unsigned port = 2; port < 4; ++port) {
1317 cluster_nodes_.push_back(&launch_server(
1318 cluster_nodes_ports_[port], "rest_server_mock_with_gr.js",
1319 cluster_nodes_http_ports_[port], 4));
1320 }
1321
1322 config_generator_->disconnect_on_metadata_unavailable(
1323 "&disconnect_on_metadata_unavailable=yes");
1324 auto &router = launch_router(
1325 router_ro_port_, config_generator_->build_config_file(
1326 temp_test_dir_.name(), GetParam().cluster_type));
1327 ASSERT_NO_FATAL_FAILURE(check_port_ready(router, router_rw_port_));
1328 ASSERT_NO_FATAL_FAILURE(check_port_ready(router, router_ro_port_));
1329
1330 /*
1331 * wait until metadata is initialized
1332 */
1333 RestMetadataClient::MetadataStatus metadata_status;
1334 RestMetadataClient rest_metadata_client(mock_http_hostname_, monitoring_port_,
1335 kRestApiUsername, kRestApiPassword);
1336
1337 ASSERT_NO_ERROR(rest_metadata_client.wait_for_cache_ready(
1338 wait_for_cache_ready_timeout, metadata_status));
1339
1340 // connect clients
1341 std::vector<std::pair<MySQLSession, uint16_t>> clients(10);
1342
1343 for (size_t i = 0; i < 2; ++i) {
1344 auto &client = clients[i].first;
1345 ASSERT_NO_THROW(client.connect("127.0.0.1", router_rw_port_, "username",
1346 "password", "", ""));
1347 std::unique_ptr<MySQLSession::ResultRow> result{
1348 client.query_one("select @@port")};
1349 clients[i].second =
1350 static_cast<uint16_t>(std::stoul(std::string((*result)[0])));
1351 }
1352
1353 for (size_t i = 2; i < 10; ++i) {
1354 auto &client = clients[i].first;
1355 ASSERT_NO_THROW(client.connect("127.0.0.1", router_ro_port_, "username",
1356 "password", "", ""));
1357 std::unique_ptr<MySQLSession::ResultRow> result{
1358 client.query_one("select @@port")};
1359 clients[i].second =
1360 static_cast<uint16_t>(std::stoul(std::string((*result)[0])));
1361 }
1362
1363 ASSERT_TRUE(MockServerRestClient(cluster_nodes_http_ports_[0])
1364 .wait_for_rest_endpoint_ready());
1365 set_additional_globals(cluster_nodes_http_ports_[0], GetParam().globals);
1366 ASSERT_NO_ERROR(rest_metadata_client.wait_for_cache_updated(
1367 wait_for_cache_update_timeout, metadata_status));
1368
1369 // verify if all connections are NOT closed
1370 for (auto &client_and_port : clients) {
1371 auto &client = client_and_port.first;
1372 ASSERT_NO_THROW(std::unique_ptr<MySQLSession::ResultRow> result{
1373 client.query_one("select @@port")});
1374 }
1375 }
1376
1377 MDRefreshTestParam steps[] = {
1378 MDRefreshTestParam(ClusterType::GR_V2,
1379 "metadata_3_secondaries_failed_to_update_v2_gr.js",
1380 "metadata_3_secondaries_pass_v2_gr.js",
1381 server_globals().set_MD_failed()),
1382
1383 MDRefreshTestParam(
1384 ClusterType::GR_V1, "metadata_3_secondaries_failed_to_update.js",
1385 "metadata_3_secondaries_pass.js", server_globals().set_MD_failed()),
1386
1387 MDRefreshTestParam(ClusterType::GR_V2,
1388 "metadata_3_secondaries_failed_to_update_v2_gr.js",
1389 "metadata_3_secondaries_pass_v2_gr.js",
1390 server_globals().set_GR_primary_failed()),
1391
1392 MDRefreshTestParam(ClusterType::GR_V1,
1393 "metadata_3_secondaries_failed_to_update.js",
1394 "metadata_3_secondaries_pass.js",
1395 server_globals().set_GR_primary_failed()),
1396
1397 MDRefreshTestParam(ClusterType::GR_V2,
1398 "metadata_3_secondaries_failed_to_update_v2_gr.js",
1399 "metadata_3_secondaries_pass_v2_gr.js",
1400 server_globals().set_GR_health_failed()),
1401
1402 MDRefreshTestParam(ClusterType::GR_V1,
1403 "metadata_3_secondaries_failed_to_update.js",
1404 "metadata_3_secondaries_pass.js",
1405 server_globals().set_GR_health_failed())};
1406
1407 INSTANTIATE_TEST_SUITE_P(RouterRoutingIsConnectionNotDisabledWhenMDRefresh,
1408 RouterRoutingConnectionMDRefreshTest,
1409 testing::ValuesIn(steps));
1410
main(int argc,char * argv[])1411 int main(int argc, char *argv[]) {
1412 init_windows_sockets();
1413 ProcessManager::set_origin(Path(argv[0]).dirname());
1414 ::testing::InitGoogleTest(&argc, argv);
1415 return RUN_ALL_TESTS();
1416 }
1417