1
2 /**
3 * Copyright (C) 2018-present MongoDB, Inc.
4 *
5 * This program is free software: you can redistribute it and/or modify
6 * it under the terms of the Server Side Public License, version 1,
7 * as published by MongoDB, Inc.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * Server Side Public License for more details.
13 *
14 * You should have received a copy of the Server Side Public License
15 * along with this program. If not, see
16 * <http://www.mongodb.com/licensing/server-side-public-license>.
17 *
18 * As a special exception, the copyright holders give permission to link the
19 * code of portions of this program with the OpenSSL library under certain
20 * conditions as described in each individual source file and distribute
21 * linked combinations including the program with the OpenSSL library. You
22 * must comply with the Server Side Public License in all respects for
23 * all of the code used other than as permitted herein. If you modify file(s)
24 * with this exception, you may extend this exception to your version of the
25 * file(s), but you are not obligated to do so. If you do not wish to do so,
26 * delete this exception statement from your version. If you delete this
27 * exception statement from all source files in the program, then also delete
28 * it in the license file.
29 */
30
31 #define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kSharding
32
33 #include "mongo/platform/basic.h"
34
35 #include <vector>
36
37 #include "mongo/client/connection_string.h"
38 #include "mongo/client/remote_command_targeter_factory_mock.h"
39 #include "mongo/client/remote_command_targeter_mock.h"
40 #include "mongo/db/commands.h"
41 #include "mongo/db/ops/write_ops.h"
42 #include "mongo/db/repl/replication_coordinator_mock.h"
43 #include "mongo/db/s/type_shard_identity.h"
44 #include "mongo/s/catalog/config_server_version.h"
45 #include "mongo/s/catalog/sharding_catalog_manager.h"
46 #include "mongo/s/catalog/type_changelog.h"
47 #include "mongo/s/catalog/type_config_version.h"
48 #include "mongo/s/catalog/type_database.h"
49 #include "mongo/s/catalog/type_shard.h"
50 #include "mongo/s/client/shard_registry.h"
51 #include "mongo/s/cluster_identity_loader.h"
52 #include "mongo/s/config_server_test_fixture.h"
53 #include "mongo/s/write_ops/batched_command_response.h"
54 #include "mongo/util/fail_point_service.h"
55 #include "mongo/util/log.h"
56 #include "mongo/util/scopeguard.h"
57
58 namespace mongo {
59 namespace {
60
61 using executor::RemoteCommandRequest;
62 using executor::RemoteCommandResponse;
63 using std::vector;
64 using unittest::assertGet;
65
66 // TODO (SERVER-27029): This value was chosen to be greater than the time it takes for the hang
67 // analyzer to kick in. Remove once the cause for the test failure has been figured out.
68 const Hours kLongFutureTimeout(8);
69
70 class AddShardTest : public ConfigServerTestFixture {
71 protected:
72 /**
73 * Performs the test setup steps from the parent class and then configures the config shard and
74 * the client name.
75 */
setUp()76 void setUp() override {
77 ConfigServerTestFixture::setUp();
78
79 // Make sure clusterID is written to the config.version collection.
80 ASSERT_OK(ShardingCatalogManager::get(operationContext())
81 ->initializeConfigDatabaseIfNeeded(operationContext()));
82
83 auto clusterIdLoader = ClusterIdentityLoader::get(operationContext());
84 ASSERT_OK(clusterIdLoader->loadClusterId(operationContext(),
85 repl::ReadConcernLevel::kLocalReadConcern));
86 _clusterId = clusterIdLoader->getClusterId();
87 }
88
89 /**
90 * addShard validates the host as a shard. It calls "isMaster" on the host to determine what
91 * kind of host it is -- mongos, regular mongod, config mongod -- and whether the replica set
92 * details are correct. "isMasterResponse" defines the response of the "isMaster" request and
93 * should be a command response BSONObj, or a failed Status.
94 *
95 * ShardingTestFixture::expectGetShards() should be called before this function, otherwise
96 * addShard will never reach the isMaster command -- a find query is called first.
97 */
expectIsMaster(const HostAndPort & target,StatusWith<BSONObj> isMasterResponse)98 void expectIsMaster(const HostAndPort& target, StatusWith<BSONObj> isMasterResponse) {
99 onCommandForAddShard([&, target, isMasterResponse](const RemoteCommandRequest& request) {
100 ASSERT_EQ(request.target, target);
101 ASSERT_EQ(request.dbname, "admin");
102 ASSERT_BSONOBJ_EQ(request.cmdObj, BSON("isMaster" << 1));
103 ASSERT_BSONOBJ_EQ(rpc::makeEmptyMetadata(), request.metadata);
104
105 return isMasterResponse;
106 });
107 }
108
expectListDatabases(const HostAndPort & target,const std::vector<BSONObj> & dbs)109 void expectListDatabases(const HostAndPort& target, const std::vector<BSONObj>& dbs) {
110 onCommandForAddShard([&](const RemoteCommandRequest& request) {
111 ASSERT_EQ(request.target, target);
112 ASSERT_EQ(request.dbname, "admin");
113 ASSERT_BSONOBJ_EQ(request.cmdObj, BSON("listDatabases" << 1 << "nameOnly" << true));
114 ASSERT_BSONOBJ_EQ(rpc::makeEmptyMetadata(), request.metadata);
115
116 BSONArrayBuilder arr;
117 for (const auto& db : dbs) {
118 arr.append(db);
119 }
120
121 return BSON("ok" << 1 << "databases" << arr.obj());
122 });
123 }
124
expectCollectionDrop(const HostAndPort & target,const NamespaceString & nss)125 void expectCollectionDrop(const HostAndPort& target, const NamespaceString& nss) {
126 onCommandForAddShard([&](const RemoteCommandRequest& request) {
127 ASSERT_EQ(request.target, target);
128 ASSERT_EQ(request.dbname, nss.db());
129 ASSERT_BSONOBJ_EQ(request.cmdObj,
130 BSON("drop" << nss.coll() << "writeConcern" << BSON("w"
131 << "majority")));
132 ASSERT_BSONOBJ_EQ(rpc::makeEmptyMetadata(), request.metadata);
133
134 return BSON("ok" << 1);
135 });
136 }
137
expectSetFeatureCompatibilityVersion(const HostAndPort & target,StatusWith<BSONObj> response)138 void expectSetFeatureCompatibilityVersion(const HostAndPort& target,
139 StatusWith<BSONObj> response) {
140 onCommandForAddShard([&, target, response](const RemoteCommandRequest& request) {
141 ASSERT_EQ(request.target, target);
142 ASSERT_EQ(request.dbname, "admin");
143 ASSERT_BSONOBJ_EQ(request.cmdObj,
144 BSON("setFeatureCompatibilityVersion"
145 << "3.4"));
146
147 return response;
148 });
149 }
150
151 /**
152 * Waits for a request for the shardIdentity document to be upserted into a shard from the
153 * config server on addShard.
154 */
expectShardIdentityUpsertReturnSuccess(const HostAndPort & expectedHost,const std::string & expectedShardName)155 void expectShardIdentityUpsertReturnSuccess(const HostAndPort& expectedHost,
156 const std::string& expectedShardName) {
157 // Create the expected upsert shardIdentity command for this shardType.
158 auto upsertCmdObj =
159 ShardingCatalogManager::get(operationContext())
160 ->createShardIdentityUpsertForAddShard(operationContext(), expectedShardName);
161
162 const auto opMsgRequest =
163 OpMsgRequest::fromDBAndBody(NamespaceString::kAdminDb, upsertCmdObj);
164 expectUpdatesReturnSuccess(expectedHost,
165 NamespaceString(NamespaceString::kServerConfigurationNamespace),
166 UpdateOp::parse(opMsgRequest));
167 }
168
expectShardIdentityUpsertReturnFailure(const HostAndPort & expectedHost,const std::string & expectedShardName,const Status & statusToReturn)169 void expectShardIdentityUpsertReturnFailure(const HostAndPort& expectedHost,
170 const std::string& expectedShardName,
171 const Status& statusToReturn) {
172 // Create the expected upsert shardIdentity command for this shardType.
173 auto upsertCmdObj =
174 ShardingCatalogManager::get(operationContext())
175 ->createShardIdentityUpsertForAddShard(operationContext(), expectedShardName);
176
177 const auto opMsgRequest =
178 OpMsgRequest::fromDBAndBody(NamespaceString::kAdminDb, upsertCmdObj);
179 expectUpdatesReturnFailure(expectedHost,
180 NamespaceString(NamespaceString::kServerConfigurationNamespace),
181 UpdateOp::parse(opMsgRequest),
182 statusToReturn);
183 }
184
185 /**
186 * Waits for a set of batched updates and ensures that the host, namespace, and updates exactly
187 * match what's expected. Responds with a success status.
188 */
expectUpdatesReturnSuccess(const HostAndPort & expectedHost,const NamespaceString & expectedNss,const write_ops::Update & expectedUpdateOp)189 void expectUpdatesReturnSuccess(const HostAndPort& expectedHost,
190 const NamespaceString& expectedNss,
191 const write_ops::Update& expectedUpdateOp) {
192 onCommandForAddShard([&](const RemoteCommandRequest& request) {
193 ASSERT_EQUALS(expectedHost, request.target);
194
195 // Check that the db name in the request matches the expected db name.
196 ASSERT_EQUALS(expectedNss.db(), request.dbname);
197
198 const auto opMsgRequest = OpMsgRequest::fromDBAndBody(request.dbname, request.cmdObj);
199 const auto updateOp = UpdateOp::parse(opMsgRequest);
200 ASSERT_EQUALS(expectedNss, expectedUpdateOp.getNamespace());
201
202 const auto& expectedUpdates = expectedUpdateOp.getUpdates();
203 const auto& actualUpdates = updateOp.getUpdates();
204
205 ASSERT_EQUALS(expectedUpdates.size(), actualUpdates.size());
206
207 auto itExpected = expectedUpdates.begin();
208 auto itActual = actualUpdates.begin();
209
210 for (; itActual != actualUpdates.end(); itActual++, itExpected++) {
211 ASSERT_EQ(itExpected->getUpsert(), itActual->getUpsert());
212 ASSERT_EQ(itExpected->getMulti(), itActual->getMulti());
213 ASSERT_BSONOBJ_EQ(itExpected->getQ(), itActual->getQ());
214 ASSERT_BSONOBJ_EQ(itExpected->getU(), itActual->getU());
215 }
216
217 BatchedCommandResponse response;
218 response.setOk(true);
219 response.setNModified(1);
220
221 return response.toBSON();
222 });
223 }
224
225 /**
226 * Waits for a set of batched updates and ensures that the host, namespace, and updates exactly
227 * match what's expected. Responds with a failure status.
228 */
expectUpdatesReturnFailure(const HostAndPort & expectedHost,const NamespaceString & expectedNss,const write_ops::Update & expectedUpdateOp,const Status & statusToReturn)229 void expectUpdatesReturnFailure(const HostAndPort& expectedHost,
230 const NamespaceString& expectedNss,
231 const write_ops::Update& expectedUpdateOp,
232 const Status& statusToReturn) {
233 onCommandForAddShard([&](const RemoteCommandRequest& request) {
234 ASSERT_EQUALS(expectedHost, request.target);
235
236 // Check that the db name in the request matches the expected db name.
237 ASSERT_EQUALS(expectedNss.db(), request.dbname);
238
239 const auto opMsgRequest = OpMsgRequest::fromDBAndBody(request.dbname, request.cmdObj);
240 const auto updateOp = UpdateOp::parse(opMsgRequest);
241 ASSERT_EQUALS(expectedNss, expectedUpdateOp.getNamespace());
242
243 const auto& expectedUpdates = expectedUpdateOp.getUpdates();
244 const auto& actualUpdates = updateOp.getUpdates();
245
246 ASSERT_EQUALS(expectedUpdates.size(), actualUpdates.size());
247
248 auto itExpected = expectedUpdates.begin();
249 auto itActual = actualUpdates.begin();
250
251 for (; itActual != actualUpdates.end(); itActual++, itExpected++) {
252 ASSERT_EQ(itExpected->getUpsert(), itActual->getUpsert());
253 ASSERT_EQ(itExpected->getMulti(), itActual->getMulti());
254 ASSERT_BSONOBJ_EQ(itExpected->getQ(), itActual->getQ());
255 ASSERT_BSONOBJ_EQ(itExpected->getU(), itActual->getU());
256 }
257
258 return statusToReturn;
259 });
260 }
261
262
263 /**
264 * Asserts that a document exists in the config server's config.shards collection corresponding
265 * to 'expectedShard'.
266 */
assertShardExists(const ShardType & expectedShard)267 void assertShardExists(const ShardType& expectedShard) {
268 auto foundShard = assertGet(getShardDoc(operationContext(), expectedShard.getName()));
269
270 ASSERT_EQUALS(expectedShard.getName(), foundShard.getName());
271 ASSERT_EQUALS(expectedShard.getHost(), foundShard.getHost());
272 ASSERT_EQUALS(expectedShard.getMaxSizeMB(), foundShard.getMaxSizeMB());
273 ASSERT_EQUALS(expectedShard.getDraining(), foundShard.getDraining());
274 ASSERT_EQUALS((int)expectedShard.getState(), (int)foundShard.getState());
275 ASSERT_TRUE(foundShard.getTags().empty());
276 }
277
278 /**
279 * Asserts that a document exists in the config server's config.databases collection
280 * corresponding to 'expectedDB'.
281 */
assertDatabaseExists(const DatabaseType & expectedDB)282 void assertDatabaseExists(const DatabaseType& expectedDB) {
283 auto foundDB =
284 assertGet(catalogClient()->getDatabase(operationContext(),
285 expectedDB.getName(),
286 repl::ReadConcernLevel::kMajorityReadConcern))
287 .value;
288
289 ASSERT_EQUALS(expectedDB.getName(), foundDB.getName());
290 ASSERT_EQUALS(expectedDB.getPrimary(), foundDB.getPrimary());
291 ASSERT_EQUALS(expectedDB.getSharded(), foundDB.getSharded());
292 }
293
294 /**
295 * Asserts that a document exists in the config server's config.changelog collection
296 * describing the addShard request for 'addedShard'.
297 */
assertChangeWasLogged(const ShardType & addedShard)298 void assertChangeWasLogged(const ShardType& addedShard) {
299 auto response = assertGet(
300 getConfigShard()->exhaustiveFindOnConfig(operationContext(),
301 ReadPreferenceSetting{
302 ReadPreference::PrimaryOnly},
303 repl::ReadConcernLevel::kLocalReadConcern,
304 NamespaceString("config.changelog"),
305 BSON("what"
306 << "addShard"
307 << "details.name"
308 << addedShard.getName()),
309 BSONObj(),
310 1));
311 ASSERT_EQ(1U, response.docs.size());
312 auto logEntryBSON = response.docs.front();
313 auto logEntry = assertGet(ChangeLogType::fromBSON(logEntryBSON));
314
315 ASSERT_EQUALS(addedShard.getName(), logEntry.getDetails()["name"].String());
316 ASSERT_EQUALS(addedShard.getHost(), logEntry.getDetails()["host"].String());
317 }
318
forwardAddShardNetwork(Date_t when)319 void forwardAddShardNetwork(Date_t when) {
320 networkForAddShard()->enterNetwork();
321 networkForAddShard()->runUntil(when);
322 networkForAddShard()->exitNetwork();
323 }
324
325 OID _clusterId;
326 };
327
TEST_F(AddShardTest,CreateShardIdentityUpsertForAddShard)328 TEST_F(AddShardTest, CreateShardIdentityUpsertForAddShard) {
329 std::string shardName = "shardName";
330
331 BSONObj expectedBSON = BSON("update"
332 << "system.version"
333 << "bypassDocumentValidation"
334 << false
335 << "ordered"
336 << true
337 << "updates"
338 << BSON_ARRAY(
339 BSON("q" << BSON("_id"
340 << "shardIdentity"
341 << "shardName"
342 << shardName
343 << "clusterId"
344 << _clusterId)
345 << "u"
346 << BSON("$set" << BSON("configsvrConnectionString"
347 << replicationCoordinator()
348 ->getConfig()
349 .getConnectionString()
350 .toString()))
351 << "multi"
352 << false
353 << "upsert"
354 << true))
355 << "writeConcern"
356 << BSON("w"
357 << "majority"
358 << "wtimeout"
359 << 60000));
360 ASSERT_BSONOBJ_EQ(expectedBSON,
361 ShardingCatalogManager::get(operationContext())
362 ->createShardIdentityUpsertForAddShard(operationContext(), shardName));
363 }
364
TEST_F(AddShardTest,StandaloneBasicSuccess)365 TEST_F(AddShardTest, StandaloneBasicSuccess) {
366 std::unique_ptr<RemoteCommandTargeterMock> targeter(
367 stdx::make_unique<RemoteCommandTargeterMock>());
368 HostAndPort shardTarget("StandaloneHost:12345");
369 targeter->setConnectionStringReturnValue(ConnectionString(shardTarget));
370 targeter->setFindHostReturnValue(shardTarget);
371
372 targeterFactory()->addTargeterToReturn(ConnectionString(shardTarget), std::move(targeter));
373
374
375 std::string expectedShardName = "StandaloneShard";
376
377 // The shard doc inserted into the config.shards collection on the config server.
378 ShardType expectedShard;
379 expectedShard.setName(expectedShardName);
380 expectedShard.setHost("StandaloneHost:12345");
381 expectedShard.setMaxSizeMB(100);
382 expectedShard.setState(ShardType::ShardState::kShardAware);
383
384 DatabaseType discoveredDB1;
385 discoveredDB1.setName("TestDB1");
386 discoveredDB1.setPrimary(ShardId("StandaloneShard"));
387 discoveredDB1.setSharded(false);
388
389 DatabaseType discoveredDB2;
390 discoveredDB2.setName("TestDB2");
391 discoveredDB2.setPrimary(ShardId("StandaloneShard"));
392 discoveredDB2.setSharded(false);
393
394 auto future = launchAsync([this, expectedShardName] {
395 Client::initThreadIfNotAlready();
396 auto shardName =
397 assertGet(ShardingCatalogManager::get(operationContext())
398 ->addShard(operationContext(),
399 &expectedShardName,
400 assertGet(ConnectionString::parse("StandaloneHost:12345")),
401 100));
402 ASSERT_EQUALS(expectedShardName, shardName);
403 });
404
405 BSONObj commandResponse = BSON("ok" << 1 << "ismaster" << true << "maxWireVersion"
406 << WireVersion::COMMANDS_ACCEPT_WRITE_CONCERN);
407 expectIsMaster(shardTarget, commandResponse);
408
409 // Get databases list from new shard
410 expectListDatabases(
411 shardTarget,
412 std::vector<BSONObj>{BSON("name"
413 << "local"
414 << "sizeOnDisk"
415 << 1000),
416 BSON("name" << discoveredDB1.getName() << "sizeOnDisk" << 2000),
417 BSON("name" << discoveredDB2.getName() << "sizeOnDisk" << 5000)});
418
419 expectCollectionDrop(shardTarget, NamespaceString("config", "system.sessions"));
420
421 // The shardIdentity doc inserted into the admin.system.version collection on the shard.
422 expectShardIdentityUpsertReturnSuccess(shardTarget, expectedShardName);
423
424 // The shard receives the setFeatureCompatibilityVersion command.
425 expectSetFeatureCompatibilityVersion(shardTarget, BSON("ok" << 1));
426
427 // Wait for the addShard to complete before checking the config database
428 future.timed_get(kLongFutureTimeout);
429
430 // Ensure that the shard document was properly added to config.shards.
431 assertShardExists(expectedShard);
432
433 // Ensure that the databases detected from the shard were properly added to config.database.
434 assertDatabaseExists(discoveredDB1);
435 assertDatabaseExists(discoveredDB2);
436
437 assertChangeWasLogged(expectedShard);
438 }
439
TEST_F(AddShardTest,StandaloneGenerateName)440 TEST_F(AddShardTest, StandaloneGenerateName) {
441 std::unique_ptr<RemoteCommandTargeterMock> targeter(
442 stdx::make_unique<RemoteCommandTargeterMock>());
443 HostAndPort shardTarget("StandaloneHost:12345");
444 targeter->setConnectionStringReturnValue(ConnectionString(shardTarget));
445 targeter->setFindHostReturnValue(shardTarget);
446
447 targeterFactory()->addTargeterToReturn(ConnectionString(shardTarget), std::move(targeter));
448
449 ShardType existingShard;
450 existingShard.setName("shard0005");
451 existingShard.setHost("existingHost:12345");
452 existingShard.setMaxSizeMB(100);
453 existingShard.setState(ShardType::ShardState::kShardAware);
454
455 // Add a pre-existing shard so when generating a name for the new shard it will have to go
456 // higher than the existing one.
457 ASSERT_OK(catalogClient()->insertConfigDocument(operationContext(),
458 ShardType::ConfigNS,
459 existingShard.toBSON(),
460 ShardingCatalogClient::kMajorityWriteConcern));
461 assertShardExists(existingShard);
462
463 std::string expectedShardName = "shard0006";
464
465 // The shard doc inserted into the config.shards collection on the config server.
466 ShardType expectedShard;
467 expectedShard.setName(expectedShardName);
468 expectedShard.setHost(shardTarget.toString());
469 expectedShard.setMaxSizeMB(100);
470 expectedShard.setState(ShardType::ShardState::kShardAware);
471
472 DatabaseType discoveredDB1;
473 discoveredDB1.setName("TestDB1");
474 discoveredDB1.setPrimary(ShardId(expectedShardName));
475 discoveredDB1.setSharded(false);
476
477 DatabaseType discoveredDB2;
478 discoveredDB2.setName("TestDB2");
479 discoveredDB2.setPrimary(ShardId(expectedShardName));
480 discoveredDB2.setSharded(false);
481
482 auto future = launchAsync([this, &expectedShardName, &shardTarget] {
483 Client::initThreadIfNotAlready();
484 auto shardName = assertGet(
485 ShardingCatalogManager::get(operationContext())
486 ->addShard(operationContext(), nullptr, ConnectionString(shardTarget), 100));
487 ASSERT_EQUALS(expectedShardName, shardName);
488 });
489
490 BSONObj commandResponse = BSON("ok" << 1 << "ismaster" << true << "maxWireVersion"
491 << WireVersion::COMMANDS_ACCEPT_WRITE_CONCERN);
492 expectIsMaster(shardTarget, commandResponse);
493
494 // Get databases list from new shard
495 expectListDatabases(
496 shardTarget,
497 std::vector<BSONObj>{BSON("name"
498 << "local"
499 << "sizeOnDisk"
500 << 1000),
501 BSON("name" << discoveredDB1.getName() << "sizeOnDisk" << 2000),
502 BSON("name" << discoveredDB2.getName() << "sizeOnDisk" << 5000)});
503
504 expectCollectionDrop(shardTarget, NamespaceString("config", "system.sessions"));
505
506 // The shardIdentity doc inserted into the admin.system.version collection on the shard.
507 expectShardIdentityUpsertReturnSuccess(shardTarget, expectedShardName);
508
509 // The shard receives the setFeatureCompatibilityVersion command.
510 expectSetFeatureCompatibilityVersion(shardTarget, BSON("ok" << 1));
511
512 // Wait for the addShard to complete before checking the config database
513 future.timed_get(kLongFutureTimeout);
514
515 // Ensure that the shard document was properly added to config.shards.
516 assertShardExists(expectedShard);
517
518 // Ensure that the databases detected from the shard were properly added to config.database.
519 assertDatabaseExists(discoveredDB1);
520 assertDatabaseExists(discoveredDB2);
521
522 assertChangeWasLogged(expectedShard);
523 }
524
TEST_F(AddShardTest,AddSCCCConnectionStringAsShard)525 TEST_F(AddShardTest, AddSCCCConnectionStringAsShard) {
526 std::unique_ptr<RemoteCommandTargeterMock> targeter(
527 stdx::make_unique<RemoteCommandTargeterMock>());
528 auto invalidConn =
529 ConnectionString("host1:12345,host2:12345,host3:12345", ConnectionString::INVALID);
530 targeter->setConnectionStringReturnValue(invalidConn);
531
532 auto future = launchAsync([this, invalidConn] {
533 const std::string shardName("StandaloneShard");
534 auto status = ShardingCatalogManager::get(operationContext())
535 ->addShard(operationContext(), &shardName, invalidConn, 100);
536 ASSERT_EQUALS(ErrorCodes::BadValue, status);
537 ASSERT_STRING_CONTAINS(status.getStatus().reason(), "Invalid connection string");
538 });
539
540 future.timed_get(kLongFutureTimeout);
541 }
542
TEST_F(AddShardTest,EmptyShardName)543 TEST_F(AddShardTest, EmptyShardName) {
544 std::unique_ptr<RemoteCommandTargeterMock> targeter(
545 stdx::make_unique<RemoteCommandTargeterMock>());
546 std::string expectedShardName = "";
547
548 auto future = launchAsync([this, expectedShardName] {
549 auto status = ShardingCatalogManager::get(operationContext())
550 ->addShard(operationContext(),
551 &expectedShardName,
552 assertGet(ConnectionString::parse("StandaloneHost:12345")),
553 100);
554 ASSERT_EQUALS(ErrorCodes::BadValue, status);
555 ASSERT_EQUALS("shard name cannot be empty", status.getStatus().reason());
556 });
557
558 future.timed_get(kLongFutureTimeout);
559 }
560
561 // Host is unreachable, cannot verify host.
TEST_F(AddShardTest,UnreachableHost)562 TEST_F(AddShardTest, UnreachableHost) {
563 std::unique_ptr<RemoteCommandTargeterMock> targeter(
564 stdx::make_unique<RemoteCommandTargeterMock>());
565 HostAndPort shardTarget("StandaloneHost:12345");
566 targeter->setConnectionStringReturnValue(ConnectionString(shardTarget));
567 targeter->setFindHostReturnValue(shardTarget);
568
569 targeterFactory()->addTargeterToReturn(ConnectionString(shardTarget), std::move(targeter));
570 std::string expectedShardName = "StandaloneShard";
571
572 auto future = launchAsync([this, &expectedShardName, &shardTarget] {
573 Client::initThreadIfNotAlready();
574 auto status =
575 ShardingCatalogManager::get(operationContext())
576 ->addShard(
577 operationContext(), &expectedShardName, ConnectionString(shardTarget), 100);
578 ASSERT_EQUALS(ErrorCodes::OperationFailed, status);
579 ASSERT_STRING_CONTAINS(status.getStatus().reason(), "host unreachable");
580 });
581
582 Status hostUnreachableStatus = Status(ErrorCodes::HostUnreachable, "host unreachable");
583 expectIsMaster(shardTarget, hostUnreachableStatus);
584
585 future.timed_get(kLongFutureTimeout);
586 }
587
588 // Cannot add mongos as a shard.
TEST_F(AddShardTest,AddMongosAsShard)589 TEST_F(AddShardTest, AddMongosAsShard) {
590 std::unique_ptr<RemoteCommandTargeterMock> targeter(
591 stdx::make_unique<RemoteCommandTargeterMock>());
592 HostAndPort shardTarget("StandaloneHost:12345");
593 targeter->setConnectionStringReturnValue(ConnectionString(shardTarget));
594 targeter->setFindHostReturnValue(shardTarget);
595
596 targeterFactory()->addTargeterToReturn(ConnectionString(shardTarget), std::move(targeter));
597 std::string expectedShardName = "StandaloneShard";
598
599 auto future = launchAsync([this, &expectedShardName, &shardTarget] {
600 Client::initThreadIfNotAlready();
601 auto status =
602 ShardingCatalogManager::get(operationContext())
603 ->addShard(
604 operationContext(), &expectedShardName, ConnectionString(shardTarget), 100);
605 ASSERT_EQUALS(ErrorCodes::IllegalOperation, status);
606 });
607
608 expectIsMaster(shardTarget,
609 BSON("msg"
610 << "isdbgrid"));
611
612 future.timed_get(kLongFutureTimeout);
613 }
614
615 // Attempt to add a pre-v3.4 mongod.
TEST_F(AddShardTest,AddVersion32Shard)616 TEST_F(AddShardTest, AddVersion32Shard) {
617 std::unique_ptr<RemoteCommandTargeterMock> targeter(
618 stdx::make_unique<RemoteCommandTargeterMock>());
619 HostAndPort shardTarget("StandaloneHost:12345");
620 targeter->setConnectionStringReturnValue(ConnectionString(shardTarget));
621 targeter->setFindHostReturnValue(shardTarget);
622
623 targeterFactory()->addTargeterToReturn(ConnectionString(shardTarget), std::move(targeter));
624 std::string expectedShardName = "StandaloneShard";
625
626 auto future = launchAsync([this, &expectedShardName, &shardTarget] {
627 Client::initThreadIfNotAlready();
628 auto status =
629 ShardingCatalogManager::get(operationContext())
630 ->addShard(
631 operationContext(), &expectedShardName, ConnectionString(shardTarget), 100);
632 ASSERT_EQUALS(ErrorCodes::IncompatibleServerVersion, status);
633 });
634
635 // The maxWireVersion indicates that this is a v3.2 shard.
636 BSONObj commandResponse =
637 BSON("ok" << 1 << "ismaster" << true << "maxWireVersion" << WireVersion::FIND_COMMAND);
638 expectIsMaster(shardTarget, commandResponse);
639
640 future.timed_get(kLongFutureTimeout);
641 }
642
643 // A replica set name was found for the host but no name was provided with the host.
TEST_F(AddShardTest,AddReplicaSetShardAsStandalone)644 TEST_F(AddShardTest, AddReplicaSetShardAsStandalone) {
645 std::unique_ptr<RemoteCommandTargeterMock> targeter(
646 stdx::make_unique<RemoteCommandTargeterMock>());
647 HostAndPort shardTarget = HostAndPort("host1:12345");
648 targeter->setConnectionStringReturnValue(ConnectionString(shardTarget));
649 targeter->setFindHostReturnValue(shardTarget);
650
651 targeterFactory()->addTargeterToReturn(ConnectionString(shardTarget), std::move(targeter));
652 std::string expectedShardName = "Standalone";
653
654 auto future = launchAsync([this, expectedShardName, shardTarget] {
655 Client::initThreadIfNotAlready();
656 auto status =
657 ShardingCatalogManager::get(operationContext())
658 ->addShard(
659 operationContext(), &expectedShardName, ConnectionString(shardTarget), 100);
660 ASSERT_EQUALS(ErrorCodes::OperationFailed, status);
661 ASSERT_STRING_CONTAINS(status.getStatus().reason(), "use replica set url format");
662 });
663
664 BSONObj commandResponse = BSON("ok" << 1 << "ismaster" << true << "setName"
665 << "myOtherSet"
666 << "maxWireVersion"
667 << WireVersion::COMMANDS_ACCEPT_WRITE_CONCERN);
668 expectIsMaster(shardTarget, commandResponse);
669
670 future.timed_get(kLongFutureTimeout);
671 }
672
673 // A replica set name was provided with the host but no name was found for the host.
TEST_F(AddShardTest,AddStandaloneHostShardAsReplicaSet)674 TEST_F(AddShardTest, AddStandaloneHostShardAsReplicaSet) {
675 std::unique_ptr<RemoteCommandTargeterMock> targeter(
676 stdx::make_unique<RemoteCommandTargeterMock>());
677 ConnectionString connString =
678 assertGet(ConnectionString::parse("mySet/host1:12345,host2:12345"));
679 HostAndPort shardTarget = connString.getServers().front();
680 targeter->setConnectionStringReturnValue(connString);
681 targeter->setFindHostReturnValue(shardTarget);
682
683 targeterFactory()->addTargeterToReturn(connString, std::move(targeter));
684 std::string expectedShardName = "StandaloneShard";
685
686 auto future = launchAsync([this, expectedShardName, connString] {
687 Client::initThreadIfNotAlready();
688 auto status = ShardingCatalogManager::get(operationContext())
689 ->addShard(operationContext(), &expectedShardName, connString, 100);
690 ASSERT_EQUALS(ErrorCodes::OperationFailed, status);
691 ASSERT_STRING_CONTAINS(status.getStatus().reason(), "host did not return a set name");
692 });
693
694 BSONObj commandResponse = BSON("ok" << 1 << "ismaster" << true << "maxWireVersion"
695 << WireVersion::COMMANDS_ACCEPT_WRITE_CONCERN);
696 expectIsMaster(shardTarget, commandResponse);
697
698 future.timed_get(kLongFutureTimeout);
699 }
700
701 // Provided replica set name does not match found replica set name.
TEST_F(AddShardTest,ReplicaSetMistmatchedReplicaSetName)702 TEST_F(AddShardTest, ReplicaSetMistmatchedReplicaSetName) {
703 std::unique_ptr<RemoteCommandTargeterMock> targeter(
704 stdx::make_unique<RemoteCommandTargeterMock>());
705 ConnectionString connString =
706 assertGet(ConnectionString::parse("mySet/host1:12345,host2:12345"));
707 targeter->setConnectionStringReturnValue(connString);
708 HostAndPort shardTarget = connString.getServers().front();
709 targeter->setFindHostReturnValue(shardTarget);
710
711 targeterFactory()->addTargeterToReturn(connString, std::move(targeter));
712 std::string expectedShardName = "StandaloneShard";
713
714 auto future = launchAsync([this, expectedShardName, connString] {
715 Client::initThreadIfNotAlready();
716 auto status = ShardingCatalogManager::get(operationContext())
717 ->addShard(operationContext(), &expectedShardName, connString, 100);
718 ASSERT_EQUALS(ErrorCodes::OperationFailed, status);
719 ASSERT_STRING_CONTAINS(status.getStatus().reason(), "does not match the actual set name");
720 });
721
722 BSONObj commandResponse = BSON("ok" << 1 << "ismaster" << true << "setName"
723 << "myOtherSet"
724 << "maxWireVersion"
725 << WireVersion::COMMANDS_ACCEPT_WRITE_CONCERN);
726 expectIsMaster(shardTarget, commandResponse);
727
728 future.timed_get(kLongFutureTimeout);
729 }
730
731 // Cannot add config server as a shard.
TEST_F(AddShardTest,ShardIsCSRSConfigServer)732 TEST_F(AddShardTest, ShardIsCSRSConfigServer) {
733 std::unique_ptr<RemoteCommandTargeterMock> targeter(
734 stdx::make_unique<RemoteCommandTargeterMock>());
735 ConnectionString connString =
736 assertGet(ConnectionString::parse("config/host1:12345,host2:12345"));
737 targeter->setConnectionStringReturnValue(connString);
738 HostAndPort shardTarget = connString.getServers().front();
739 targeter->setFindHostReturnValue(shardTarget);
740
741 targeterFactory()->addTargeterToReturn(connString, std::move(targeter));
742 std::string expectedShardName = "StandaloneShard";
743
744 auto future = launchAsync([this, expectedShardName, connString] {
745 Client::initThreadIfNotAlready();
746 auto status = ShardingCatalogManager::get(operationContext())
747 ->addShard(operationContext(), &expectedShardName, connString, 100);
748 ASSERT_EQUALS(ErrorCodes::OperationFailed, status);
749 ASSERT_STRING_CONTAINS(status.getStatus().reason(),
750 "as a shard since it is a config server");
751 });
752
753 BSONObj commandResponse = BSON("ok" << 1 << "ismaster" << true << "setName"
754 << "config"
755 << "configsvr"
756 << true
757 << "maxWireVersion"
758 << WireVersion::COMMANDS_ACCEPT_WRITE_CONCERN);
759 expectIsMaster(shardTarget, commandResponse);
760
761 future.timed_get(kLongFutureTimeout);
762 }
763
764 // One of the hosts is not part of the found replica set.
TEST_F(AddShardTest,ReplicaSetMissingHostsProvidedInSeedList)765 TEST_F(AddShardTest, ReplicaSetMissingHostsProvidedInSeedList) {
766 std::unique_ptr<RemoteCommandTargeterMock> targeter(
767 stdx::make_unique<RemoteCommandTargeterMock>());
768 ConnectionString connString =
769 assertGet(ConnectionString::parse("mySet/host1:12345,host2:12345"));
770 targeter->setConnectionStringReturnValue(connString);
771 HostAndPort shardTarget = connString.getServers().front();
772 targeter->setFindHostReturnValue(shardTarget);
773
774 targeterFactory()->addTargeterToReturn(connString, std::move(targeter));
775 std::string expectedShardName = "StandaloneShard";
776
777 auto future = launchAsync([this, expectedShardName, connString] {
778 Client::initThreadIfNotAlready();
779 auto status = ShardingCatalogManager::get(operationContext())
780 ->addShard(operationContext(), &expectedShardName, connString, 100);
781 ASSERT_EQUALS(ErrorCodes::OperationFailed, status);
782 ASSERT_STRING_CONTAINS(status.getStatus().reason(),
783 "host2:12345 does not belong to replica set");
784 });
785
786 BSONArrayBuilder hosts;
787 hosts.append("host1:12345");
788 BSONObj commandResponse = BSON("ok" << 1 << "ismaster" << true << "setName"
789 << "mySet"
790 << "hosts"
791 << hosts.arr()
792 << "maxWireVersion"
793 << WireVersion::COMMANDS_ACCEPT_WRITE_CONCERN);
794 expectIsMaster(shardTarget, commandResponse);
795
796 future.timed_get(kLongFutureTimeout);
797 }
798
799 // Cannot add a shard with the shard name "config".
TEST_F(AddShardTest,AddShardWithNameConfigFails)800 TEST_F(AddShardTest, AddShardWithNameConfigFails) {
801 std::unique_ptr<RemoteCommandTargeterMock> targeter(
802 stdx::make_unique<RemoteCommandTargeterMock>());
803 ConnectionString connString =
804 assertGet(ConnectionString::parse("mySet/host1:12345,host2:12345"));
805 targeter->setConnectionStringReturnValue(connString);
806 HostAndPort shardTarget = connString.getServers().front();
807 targeter->setFindHostReturnValue(shardTarget);
808
809 targeterFactory()->addTargeterToReturn(connString, std::move(targeter));
810 std::string expectedShardName = "config";
811
812 auto future = launchAsync([this, expectedShardName, connString] {
813 Client::initThreadIfNotAlready();
814 auto status = ShardingCatalogManager::get(operationContext())
815 ->addShard(operationContext(), &expectedShardName, connString, 100);
816 ASSERT_EQUALS(ErrorCodes::BadValue, status);
817 ASSERT_EQUALS(status.getStatus().reason(),
818 "use of shard replica set with name 'config' is not allowed");
819 });
820
821 BSONArrayBuilder hosts;
822 hosts.append("host1:12345");
823 hosts.append("host2:12345");
824 BSONObj commandResponse = BSON("ok" << 1 << "ismaster" << true << "setName"
825 << "mySet"
826 << "hosts"
827 << hosts.arr()
828 << "maxWireVersion"
829 << WireVersion::COMMANDS_ACCEPT_WRITE_CONCERN);
830 expectIsMaster(shardTarget, commandResponse);
831
832 future.timed_get(kLongFutureTimeout);
833 }
834
TEST_F(AddShardTest,ShardContainsExistingDatabase)835 TEST_F(AddShardTest, ShardContainsExistingDatabase) {
836 std::unique_ptr<RemoteCommandTargeterMock> targeter(
837 stdx::make_unique<RemoteCommandTargeterMock>());
838 ConnectionString connString =
839 assertGet(ConnectionString::parse("mySet/host1:12345,host2:12345"));
840 targeter->setConnectionStringReturnValue(connString);
841 HostAndPort shardTarget = connString.getServers().front();
842 targeter->setFindHostReturnValue(shardTarget);
843
844 targeterFactory()->addTargeterToReturn(connString, std::move(targeter));
845 std::string expectedShardName = "mySet";
846
847 DatabaseType existingDB;
848 existingDB.setName("existing");
849 existingDB.setPrimary(ShardId("existingShard"));
850 existingDB.setSharded(false);
851
852 // Add a pre-existing database.
853 ASSERT_OK(catalogClient()->insertConfigDocument(operationContext(),
854 DatabaseType::ConfigNS,
855 existingDB.toBSON(),
856 ShardingCatalogClient::kMajorityWriteConcern));
857 assertDatabaseExists(existingDB);
858
859
860 auto future = launchAsync([this, expectedShardName, connString] {
861 Client::initThreadIfNotAlready();
862 auto status = ShardingCatalogManager::get(operationContext())
863 ->addShard(operationContext(), &expectedShardName, connString, 100);
864 ASSERT_EQUALS(ErrorCodes::OperationFailed, status);
865 ASSERT_STRING_CONTAINS(
866 status.getStatus().reason(),
867 "because a local database 'existing' exists in another existingShard");
868 });
869
870 BSONArrayBuilder hosts;
871 hosts.append("host1:12345");
872 hosts.append("host2:12345");
873 BSONObj commandResponse = BSON("ok" << 1 << "ismaster" << true << "setName"
874 << "mySet"
875 << "hosts"
876 << hosts.arr()
877 << "maxWireVersion"
878 << WireVersion::COMMANDS_ACCEPT_WRITE_CONCERN);
879 expectIsMaster(shardTarget, commandResponse);
880
881 expectListDatabases(shardTarget, {BSON("name" << existingDB.getName())});
882
883 future.timed_get(kLongFutureTimeout);
884 }
885
TEST_F(AddShardTest,SuccessfullyAddReplicaSet)886 TEST_F(AddShardTest, SuccessfullyAddReplicaSet) {
887 std::unique_ptr<RemoteCommandTargeterMock> targeter(
888 stdx::make_unique<RemoteCommandTargeterMock>());
889 ConnectionString connString =
890 assertGet(ConnectionString::parse("mySet/host1:12345,host2:12345"));
891 targeter->setConnectionStringReturnValue(connString);
892 HostAndPort shardTarget = connString.getServers().front();
893 targeter->setFindHostReturnValue(shardTarget);
894 targeterFactory()->addTargeterToReturn(connString, std::move(targeter));
895
896 std::string expectedShardName = "mySet";
897
898 // The shard doc inserted into the config.shards collection on the config server.
899 ShardType expectedShard;
900 expectedShard.setName(expectedShardName);
901 expectedShard.setHost(connString.toString());
902 expectedShard.setMaxSizeMB(100);
903 expectedShard.setState(ShardType::ShardState::kShardAware);
904
905 DatabaseType discoveredDB;
906 discoveredDB.setName("shardDB");
907 discoveredDB.setPrimary(ShardId(expectedShardName));
908 discoveredDB.setSharded(false);
909
910 auto future = launchAsync([this, &expectedShardName, &connString] {
911 Client::initThreadIfNotAlready();
912 auto shardName = assertGet(ShardingCatalogManager::get(operationContext())
913 ->addShard(operationContext(), nullptr, connString, 100));
914 ASSERT_EQUALS(expectedShardName, shardName);
915 });
916
917 BSONArrayBuilder hosts;
918 hosts.append("host1:12345");
919 hosts.append("host2:12345");
920 BSONObj commandResponse = BSON("ok" << 1 << "ismaster" << true << "setName"
921 << "mySet"
922 << "hosts"
923 << hosts.arr()
924 << "maxWireVersion"
925 << WireVersion::COMMANDS_ACCEPT_WRITE_CONCERN);
926 expectIsMaster(shardTarget, commandResponse);
927
928 // Get databases list from new shard
929 expectListDatabases(shardTarget, std::vector<BSONObj>{BSON("name" << discoveredDB.getName())});
930
931 expectCollectionDrop(shardTarget, NamespaceString("config", "system.sessions"));
932
933 // The shardIdentity doc inserted into the admin.system.version collection on the shard.
934 expectShardIdentityUpsertReturnSuccess(shardTarget, expectedShardName);
935
936 // The shard receives the setFeatureCompatibilityVersion command.
937 expectSetFeatureCompatibilityVersion(shardTarget, BSON("ok" << 1));
938
939 // Wait for the addShard to complete before checking the config database
940 future.timed_get(kLongFutureTimeout);
941
942 // Ensure that the shard document was properly added to config.shards.
943 assertShardExists(expectedShard);
944
945 // Ensure that the databases detected from the shard were properly added to config.database.
946 assertDatabaseExists(discoveredDB);
947
948 assertChangeWasLogged(expectedShard);
949 }
950
TEST_F(AddShardTest,ReplicaSetExtraHostsDiscovered)951 TEST_F(AddShardTest, ReplicaSetExtraHostsDiscovered) {
952 std::unique_ptr<RemoteCommandTargeterMock> targeter(
953 stdx::make_unique<RemoteCommandTargeterMock>());
954 ConnectionString seedString =
955 assertGet(ConnectionString::parse("mySet/host1:12345,host2:12345"));
956 ConnectionString fullConnString =
957 assertGet(ConnectionString::parse("mySet/host1:12345,host2:12345,host3:12345"));
958 targeter->setConnectionStringReturnValue(fullConnString);
959 HostAndPort shardTarget = seedString.getServers().front();
960 targeter->setFindHostReturnValue(shardTarget);
961 targeterFactory()->addTargeterToReturn(seedString, std::move(targeter));
962
963 std::string expectedShardName = "mySet";
964
965 // The shard doc inserted into the config.shards collection on the config server.
966 ShardType expectedShard;
967 expectedShard.setName(expectedShardName);
968 expectedShard.setHost(fullConnString.toString());
969 expectedShard.setMaxSizeMB(100);
970 expectedShard.setState(ShardType::ShardState::kShardAware);
971
972 DatabaseType discoveredDB;
973 discoveredDB.setName("shardDB");
974 discoveredDB.setPrimary(ShardId(expectedShardName));
975 discoveredDB.setSharded(false);
976
977 auto future = launchAsync([this, &expectedShardName, &seedString] {
978 Client::initThreadIfNotAlready();
979 auto shardName = assertGet(ShardingCatalogManager::get(operationContext())
980 ->addShard(operationContext(), nullptr, seedString, 100));
981 ASSERT_EQUALS(expectedShardName, shardName);
982 });
983
984 BSONArrayBuilder hosts;
985 hosts.append("host1:12345");
986 hosts.append("host2:12345");
987 BSONObj commandResponse = BSON("ok" << 1 << "ismaster" << true << "setName"
988 << "mySet"
989 << "hosts"
990 << hosts.arr()
991 << "maxWireVersion"
992 << WireVersion::COMMANDS_ACCEPT_WRITE_CONCERN);
993 expectIsMaster(shardTarget, commandResponse);
994
995 // Get databases list from new shard
996 expectListDatabases(shardTarget, std::vector<BSONObj>{BSON("name" << discoveredDB.getName())});
997
998 expectCollectionDrop(shardTarget, NamespaceString("config", "system.sessions"));
999
1000 // The shardIdentity doc inserted into the admin.system.version collection on the shard.
1001 expectShardIdentityUpsertReturnSuccess(shardTarget, expectedShardName);
1002
1003 // The shard receives the setFeatureCompatibilityVersion command.
1004 expectSetFeatureCompatibilityVersion(shardTarget, BSON("ok" << 1));
1005
1006 // Wait for the addShard to complete before checking the config database
1007 future.timed_get(kLongFutureTimeout);
1008
1009 // Ensure that the shard document was properly added to config.shards.
1010 assertShardExists(expectedShard);
1011
1012 // Ensure that the databases detected from the shard were properly added to config.database.
1013 assertDatabaseExists(discoveredDB);
1014
1015 // The changelog entry uses whatever connection string is passed to addShard, even if addShard
1016 // discovered additional hosts.
1017 expectedShard.setHost(seedString.toString());
1018 assertChangeWasLogged(expectedShard);
1019 }
1020
TEST_F(AddShardTest,AddShardSucceedsEvenIfAddingDBsFromNewShardFails)1021 TEST_F(AddShardTest, AddShardSucceedsEvenIfAddingDBsFromNewShardFails) {
1022 std::unique_ptr<RemoteCommandTargeterMock> targeter(
1023 stdx::make_unique<RemoteCommandTargeterMock>());
1024 HostAndPort shardTarget("StandaloneHost:12345");
1025 targeter->setConnectionStringReturnValue(ConnectionString(shardTarget));
1026 targeter->setFindHostReturnValue(shardTarget);
1027
1028 targeterFactory()->addTargeterToReturn(ConnectionString(shardTarget), std::move(targeter));
1029
1030
1031 std::string expectedShardName = "StandaloneShard";
1032
1033 // The shard doc inserted into the config.shards collection on the config server.
1034 ShardType expectedShard;
1035 expectedShard.setName(expectedShardName);
1036 expectedShard.setHost("StandaloneHost:12345");
1037 expectedShard.setMaxSizeMB(100);
1038 expectedShard.setState(ShardType::ShardState::kShardAware);
1039
1040 DatabaseType discoveredDB1;
1041 discoveredDB1.setName("TestDB1");
1042 discoveredDB1.setPrimary(ShardId("StandaloneShard"));
1043 discoveredDB1.setSharded(false);
1044
1045 DatabaseType discoveredDB2;
1046 discoveredDB2.setName("TestDB2");
1047 discoveredDB2.setPrimary(ShardId("StandaloneShard"));
1048 discoveredDB2.setSharded(false);
1049
1050 // Enable fail point to cause all updates to fail. Since we add the databases detected from
1051 // the shard being added with upserts, but we add the shard document itself via insert, this
1052 // will allow the shard to be added but prevent the databases from brought into the cluster.
1053 auto failPoint = getGlobalFailPointRegistry()->getFailPoint("failAllUpdates");
1054 ASSERT(failPoint);
1055 failPoint->setMode(FailPoint::alwaysOn);
1056 ON_BLOCK_EXIT([&] { failPoint->setMode(FailPoint::off); });
1057
1058 auto future = launchAsync([this, &expectedShardName, &shardTarget] {
1059 Client::initThreadIfNotAlready();
1060 auto shardName = assertGet(
1061 ShardingCatalogManager::get(operationContext())
1062 ->addShard(
1063 operationContext(), &expectedShardName, ConnectionString(shardTarget), 100));
1064 ASSERT_EQUALS(expectedShardName, shardName);
1065 });
1066
1067 BSONObj commandResponse = BSON("ok" << 1 << "ismaster" << true << "maxWireVersion"
1068 << WireVersion::COMMANDS_ACCEPT_WRITE_CONCERN);
1069 expectIsMaster(shardTarget, commandResponse);
1070
1071 // Get databases list from new shard
1072 expectListDatabases(
1073 shardTarget,
1074 std::vector<BSONObj>{BSON("name"
1075 << "local"
1076 << "sizeOnDisk"
1077 << 1000),
1078 BSON("name" << discoveredDB1.getName() << "sizeOnDisk" << 2000),
1079 BSON("name" << discoveredDB2.getName() << "sizeOnDisk" << 5000)});
1080
1081 expectCollectionDrop(shardTarget, NamespaceString("config", "system.sessions"));
1082
1083 // The shardIdentity doc inserted into the admin.system.version collection on the shard.
1084 expectShardIdentityUpsertReturnSuccess(shardTarget, expectedShardName);
1085
1086 // The shard receives the setFeatureCompatibilityVersion command.
1087 expectSetFeatureCompatibilityVersion(shardTarget, BSON("ok" << 1));
1088
1089 // Wait for the addShard to complete before checking the config database
1090 future.timed_get(kLongFutureTimeout);
1091
1092 // Ensure that the shard document was properly added to config.shards.
1093 assertShardExists(expectedShard);
1094
1095 // Ensure that the databases detected from the shard were *not* added.
1096 ASSERT_EQUALS(ErrorCodes::NamespaceNotFound,
1097 catalogClient()
1098 ->getDatabase(operationContext(),
1099 discoveredDB1.getName(),
1100 repl::ReadConcernLevel::kMajorityReadConcern)
1101 .getStatus());
1102 ASSERT_EQUALS(ErrorCodes::NamespaceNotFound,
1103 catalogClient()
1104 ->getDatabase(operationContext(),
1105 discoveredDB2.getName(),
1106 repl::ReadConcernLevel::kMajorityReadConcern)
1107 .getStatus());
1108
1109 assertChangeWasLogged(expectedShard);
1110 }
1111
1112 // Tests both that trying to add a shard with the same host as an existing shard but with different
1113 // options fails, and that adding a shard with the same host as an existing shard with the *same*
1114 // options succeeds.
TEST_F(AddShardTest,AddExistingShardStandalone)1115 TEST_F(AddShardTest, AddExistingShardStandalone) {
1116 HostAndPort shardTarget("StandaloneHost:12345");
1117 std::unique_ptr<RemoteCommandTargeterMock> standaloneTargeter(
1118 stdx::make_unique<RemoteCommandTargeterMock>());
1119 standaloneTargeter->setConnectionStringReturnValue(ConnectionString(shardTarget));
1120 standaloneTargeter->setFindHostReturnValue(shardTarget);
1121 targeterFactory()->addTargeterToReturn(ConnectionString(shardTarget),
1122 std::move(standaloneTargeter));
1123
1124 std::unique_ptr<RemoteCommandTargeterMock> replsetTargeter(
1125 stdx::make_unique<RemoteCommandTargeterMock>());
1126 replsetTargeter->setConnectionStringReturnValue(
1127 ConnectionString::forReplicaSet("mySet", {shardTarget}));
1128 replsetTargeter->setFindHostReturnValue(shardTarget);
1129 targeterFactory()->addTargeterToReturn(ConnectionString::forReplicaSet("mySet", {shardTarget}),
1130 std::move(replsetTargeter));
1131
1132 std::string existingShardName = "myShard";
1133 ShardType existingShard;
1134 existingShard.setName(existingShardName);
1135 existingShard.setHost(shardTarget.toString());
1136 existingShard.setMaxSizeMB(100);
1137 existingShard.setState(ShardType::ShardState::kShardAware);
1138
1139 // Make sure the shard already exists.
1140 ASSERT_OK(catalogClient()->insertConfigDocument(operationContext(),
1141 ShardType::ConfigNS,
1142 existingShard.toBSON(),
1143 ShardingCatalogClient::kMajorityWriteConcern));
1144 assertShardExists(existingShard);
1145
1146 // Adding the same standalone host with a different shard name should fail.
1147 std::string differentName = "anotherShardName";
1148 auto future1 = launchAsync([&] {
1149 Client::initThreadIfNotAlready();
1150 ASSERT_EQUALS(ErrorCodes::IllegalOperation,
1151 ShardingCatalogManager::get(operationContext())
1152 ->addShard(operationContext(),
1153 &differentName,
1154 ConnectionString(shardTarget),
1155 existingShard.getMaxSizeMB()));
1156 });
1157 future1.timed_get(kLongFutureTimeout);
1158
1159 // Ensure that the shard document was unchanged.
1160 assertShardExists(existingShard);
1161
1162 // Adding the same standalone host with a different maxSize should fail.
1163 auto future2 = launchAsync([&] {
1164 Client::initThreadIfNotAlready();
1165 ASSERT_EQUALS(ErrorCodes::IllegalOperation,
1166 ShardingCatalogManager::get(operationContext())
1167 ->addShard(operationContext(),
1168 nullptr,
1169 ConnectionString(shardTarget),
1170 existingShard.getMaxSizeMB() + 100));
1171 });
1172 future2.timed_get(kLongFutureTimeout);
1173
1174 // Adding the same standalone host but as part of a replica set should fail.
1175 // Ensures that even if the user changed the standalone shard to a single-node replica set, you
1176 // can't change the sharded cluster's notion of the shard from standalone to replica set just
1177 // by calling addShard.
1178 auto future3 = launchAsync([&] {
1179 Client::initThreadIfNotAlready();
1180 ASSERT_EQUALS(ErrorCodes::IllegalOperation,
1181 ShardingCatalogManager::get(operationContext())
1182 ->addShard(operationContext(),
1183 nullptr,
1184 ConnectionString::forReplicaSet("mySet", {shardTarget}),
1185 existingShard.getMaxSizeMB()));
1186 });
1187 future3.timed_get(kLongFutureTimeout);
1188
1189 // Ensure that the shard document was unchanged.
1190 assertShardExists(existingShard);
1191
1192 // Adding the same standalone host with the same options should succeed.
1193 auto future4 = launchAsync([&] {
1194 Client::initThreadIfNotAlready();
1195 auto shardName = assertGet(ShardingCatalogManager::get(operationContext())
1196 ->addShard(operationContext(),
1197 &existingShardName,
1198 ConnectionString(shardTarget),
1199 existingShard.getMaxSizeMB()));
1200 ASSERT_EQUALS(existingShardName, shardName);
1201 });
1202 future4.timed_get(kLongFutureTimeout);
1203
1204 // Ensure that the shard document was unchanged.
1205 assertShardExists(existingShard);
1206
1207 // Adding the same standalone host with the same options (without explicitly specifying the
1208 // shard name) should succeed.
1209 auto future5 = launchAsync([&] {
1210 Client::initThreadIfNotAlready();
1211 auto shardName = assertGet(ShardingCatalogManager::get(operationContext())
1212 ->addShard(operationContext(),
1213 nullptr,
1214 ConnectionString(shardTarget),
1215 existingShard.getMaxSizeMB()));
1216 ASSERT_EQUALS(existingShardName, shardName);
1217 });
1218 future5.timed_get(kLongFutureTimeout);
1219
1220 // Ensure that the shard document was unchanged.
1221 assertShardExists(existingShard);
1222 }
1223
1224 // Tests both that trying to add a shard with the same replica set as an existing shard but with
1225 // different options fails, and that adding a shard with the same replica set as an existing shard
1226 // with the *same* options succeeds.
TEST_F(AddShardTest,AddExistingShardReplicaSet)1227 TEST_F(AddShardTest, AddExistingShardReplicaSet) {
1228 std::unique_ptr<RemoteCommandTargeterMock> replsetTargeter(
1229 stdx::make_unique<RemoteCommandTargeterMock>());
1230 ConnectionString connString = assertGet(ConnectionString::parse("mySet/host1:12345"));
1231 replsetTargeter->setConnectionStringReturnValue(connString);
1232 HostAndPort shardTarget = connString.getServers().front();
1233 replsetTargeter->setFindHostReturnValue(shardTarget);
1234 targeterFactory()->addTargeterToReturn(connString, std::move(replsetTargeter));
1235
1236 std::string existingShardName = "myShard";
1237 ShardType existingShard;
1238 existingShard.setName(existingShardName);
1239 existingShard.setHost(connString.toString());
1240 existingShard.setMaxSizeMB(100);
1241 existingShard.setState(ShardType::ShardState::kShardAware);
1242
1243 // Make sure the shard already exists.
1244 ASSERT_OK(catalogClient()->insertConfigDocument(operationContext(),
1245 ShardType::ConfigNS,
1246 existingShard.toBSON(),
1247 ShardingCatalogClient::kMajorityWriteConcern));
1248 assertShardExists(existingShard);
1249
1250 // Adding the same connection string with a different shard name should fail.
1251 std::string differentName = "anotherShardName";
1252 auto future1 = launchAsync([&] {
1253 Client::initThreadIfNotAlready();
1254 ASSERT_EQUALS(
1255 ErrorCodes::IllegalOperation,
1256 ShardingCatalogManager::get(operationContext())
1257 ->addShard(
1258 operationContext(), &differentName, connString, existingShard.getMaxSizeMB()));
1259 });
1260 future1.timed_get(kLongFutureTimeout);
1261
1262 // Ensure that the shard document was unchanged.
1263 assertShardExists(existingShard);
1264
1265 // Adding the same connection string with a different maxSize should fail.
1266 auto future2 = launchAsync([&] {
1267 Client::initThreadIfNotAlready();
1268 ASSERT_EQUALS(
1269 ErrorCodes::IllegalOperation,
1270 ShardingCatalogManager::get(operationContext())
1271 ->addShard(
1272 operationContext(), nullptr, connString, existingShard.getMaxSizeMB() + 100));
1273 });
1274 future2.timed_get(kLongFutureTimeout);
1275
1276 // Ensure that the shard document was unchanged.
1277 assertShardExists(existingShard);
1278
1279 // Adding a connecting string with a host of an existing shard but using a different connection
1280 // string type should fail.
1281 // Ensures that even if the user changed the replica set shard to a standalone, you can't change
1282 // the sharded cluster's notion of the shard from replica set to standalone just by calling
1283 // addShard.
1284 auto future3 = launchAsync([&] {
1285 Client::initThreadIfNotAlready();
1286 ASSERT_EQUALS(ErrorCodes::IllegalOperation,
1287 ShardingCatalogManager::get(operationContext())
1288 ->addShard(operationContext(),
1289 nullptr,
1290 ConnectionString(shardTarget),
1291 existingShard.getMaxSizeMB()));
1292 });
1293 future3.timed_get(kLongFutureTimeout);
1294
1295 // Ensure that the shard document was unchanged.
1296 assertShardExists(existingShard);
1297
1298 // Adding a connecting string with the same hosts but a different replica set name should fail.
1299 // Ensures that even if you manually change the shard's replica set name somehow, you can't
1300 // change the replica set name the sharded cluster knows for it just by calling addShard again.
1301 std::string differentSetName = "differentSet";
1302 auto future4 = launchAsync([&] {
1303 Client::initThreadIfNotAlready();
1304 ASSERT_EQUALS(ErrorCodes::IllegalOperation,
1305 ShardingCatalogManager::get(operationContext())
1306 ->addShard(operationContext(),
1307 nullptr,
1308 ConnectionString::forReplicaSet(differentSetName,
1309 connString.getServers()),
1310 existingShard.getMaxSizeMB()));
1311 });
1312 future4.timed_get(kLongFutureTimeout);
1313
1314 // Ensure that the shard document was unchanged.
1315 assertShardExists(existingShard);
1316
1317 // Adding the same host with the same options should succeed.
1318 auto future5 = launchAsync([&] {
1319 Client::initThreadIfNotAlready();
1320 auto shardName = assertGet(ShardingCatalogManager::get(operationContext())
1321 ->addShard(operationContext(),
1322 &existingShardName,
1323 connString,
1324 existingShard.getMaxSizeMB()));
1325 ASSERT_EQUALS(existingShardName, shardName);
1326 });
1327 future5.timed_get(kLongFutureTimeout);
1328
1329 // Adding the same host with the same options (without explicitly specifying the shard name)
1330 // should succeed.
1331 auto future6 = launchAsync([&] {
1332 Client::initThreadIfNotAlready();
1333 auto shardName = assertGet(
1334 ShardingCatalogManager::get(operationContext())
1335 ->addShard(operationContext(), nullptr, connString, existingShard.getMaxSizeMB()));
1336 ASSERT_EQUALS(existingShardName, shardName);
1337 });
1338 future6.timed_get(kLongFutureTimeout);
1339
1340 // Ensure that the shard document was unchanged.
1341 assertShardExists(existingShard);
1342
1343 // Adding the same replica set but different host membership (but otherwise the same options)
1344 // should succeed
1345 auto otherHost = connString.getServers().back();
1346 ConnectionString otherHostConnString = assertGet(ConnectionString::parse("mySet/host2:12345"));
1347 {
1348 // Add a targeter for the different seed string this addShard request will use.
1349 std::unique_ptr<RemoteCommandTargeterMock> otherHostTargeter(
1350 stdx::make_unique<RemoteCommandTargeterMock>());
1351 otherHostTargeter->setConnectionStringReturnValue(otherHostConnString);
1352 otherHostTargeter->setFindHostReturnValue(otherHost);
1353 targeterFactory()->addTargeterToReturn(otherHostConnString, std::move(otherHostTargeter));
1354 }
1355 auto future7 = launchAsync([&] {
1356 Client::initThreadIfNotAlready();
1357 auto shardName = assertGet(ShardingCatalogManager::get(operationContext())
1358 ->addShard(operationContext(),
1359 nullptr,
1360 otherHostConnString,
1361 existingShard.getMaxSizeMB()));
1362 ASSERT_EQUALS(existingShardName, shardName);
1363 });
1364 future7.timed_get(kLongFutureTimeout);
1365
1366 // Ensure that the shard document was unchanged.
1367 assertShardExists(existingShard);
1368 }
1369
1370 } // namespace
1371 } // namespace mongo
1372