1 2 /** 3 * Copyright (C) 2018-present MongoDB, Inc. 4 * 5 * This program is free software: you can redistribute it and/or modify 6 * it under the terms of the Server Side Public License, version 1, 7 * as published by MongoDB, Inc. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * Server Side Public License for more details. 13 * 14 * You should have received a copy of the Server Side Public License 15 * along with this program. If not, see 16 * <http://www.mongodb.com/licensing/server-side-public-license>. 17 * 18 * As a special exception, the copyright holders give permission to link the 19 * code of portions of this program with the OpenSSL library under certain 20 * conditions as described in each individual source file and distribute 21 * linked combinations including the program with the OpenSSL library. You 22 * must comply with the Server Side Public License in all respects for 23 * all of the code used other than as permitted herein. If you modify file(s) 24 * with this exception, you may extend this exception to your version of the 25 * file(s), but you are not obligated to do so. If you do not wish to do so, 26 * delete this exception statement from your version. If you delete this 27 * exception statement from all source files in the program, then also delete 28 * it in the license file. 29 */ 30 31 32 #pragma once 33 34 #include <cstdint> 35 #include <iosfwd> 36 #include <memory> 37 38 #include "mongo/base/status.h" 39 #include "mongo/bson/bsonobj.h" 40 #include "mongo/bson/bsonobjbuilder.h" 41 #include "mongo/bson/timestamp.h" 42 #include "mongo/db/namespace_string.h" 43 #include "mongo/db/repl/callback_completion_guard.h" 44 #include "mongo/db/repl/collection_cloner.h" 45 #include "mongo/db/repl/data_replicator_external_state.h" 46 #include "mongo/db/repl/multiapplier.h" 47 #include "mongo/db/repl/oplog_buffer.h" 48 #include "mongo/db/repl/oplog_fetcher.h" 49 #include "mongo/db/repl/optime.h" 50 #include "mongo/db/repl/rollback_checker.h" 51 #include "mongo/db/repl/sync_source_selector.h" 52 #include "mongo/db/repl/sync_tail.h" 53 #include "mongo/stdx/condition_variable.h" 54 #include "mongo/stdx/functional.h" 55 #include "mongo/stdx/mutex.h" 56 #include "mongo/util/fail_point_service.h" 57 #include "mongo/util/net/hostandport.h" 58 59 namespace mongo { 60 namespace repl { 61 62 // TODO: Remove forward declares once we remove rs_initialsync.cpp and other dependents. 63 // Failpoint which fails initial sync and leaves an oplog entry in the buffer. 64 MONGO_FP_FORWARD_DECLARE(failInitSyncWithBufferedEntriesLeft); 65 66 // Failpoint which causes the initial sync function to hang before copying databases. 67 MONGO_FP_FORWARD_DECLARE(initialSyncHangBeforeCopyingDatabases); 68 69 // Failpoint which causes the initial sync function to hang before calling shouldRetry on a failed 70 // operation. 71 MONGO_FP_FORWARD_DECLARE(initialSyncHangBeforeGettingMissingDocument); 72 73 // Failpoint which stops the applier. 74 MONGO_FP_FORWARD_DECLARE(rsSyncApplyStop); 75 76 struct InitialSyncState; 77 struct MemberState; 78 class ReplicationProcess; 79 class StorageInterface; 80 81 struct InitialSyncerOptions { 82 /** Function to return optime of last operation applied on this node */ 83 using GetMyLastOptimeFn = stdx::function<OpTime()>; 84 85 /** Function to update optime of last operation applied on this node */ 86 using SetMyLastOptimeFn = 87 stdx::function<void(const OpTime&, ReplicationCoordinator::DataConsistency consistency)>; 88 89 /** Function to reset all optimes on this node (e.g. applied & durable). */ 90 using ResetOptimesFn = stdx::function<void()>; 91 92 /** Function to sets this node into a specific follower mode. */ 93 using SetFollowerModeFn = stdx::function<bool(const MemberState&)>; 94 95 /** Function to get this node's slaveDelay. */ 96 using GetSlaveDelayFn = stdx::function<Seconds()>; 97 98 /** 99 * Struct to hold oplog application batch limits. 100 */ 101 using BatchLimits = SyncTail::BatchLimits; 102 103 // Error and retry values 104 Milliseconds syncSourceRetryWait{1000}; 105 Milliseconds initialSyncRetryWait{1000}; 106 Seconds blacklistSyncSourcePenaltyForNetworkConnectionError{10}; 107 Minutes blacklistSyncSourcePenaltyForOplogStartMissing{10}; 108 109 // InitialSyncer waits this long before retrying getApplierBatchCallback() if there are 110 // currently no operations available to apply or if the 'rsSyncApplyStop' failpoint is active. 111 // This default value is based on the duration in BackgroundSync::waitForMore() and 112 // SyncTail::tryPopAndWaitForMore(). 113 Milliseconds getApplierBatchCallbackRetryWait{1000}; 114 115 // Batching settings. 116 BatchLimits batchLimits; 117 118 // Replication settings 119 NamespaceString localOplogNS = NamespaceString("local.oplog.rs"); 120 NamespaceString remoteOplogNS = NamespaceString("local.oplog.rs"); 121 122 GetMyLastOptimeFn getMyLastOptime; 123 SetMyLastOptimeFn setMyLastOptime; 124 ResetOptimesFn resetOptimes; 125 GetSlaveDelayFn getSlaveDelay; 126 127 SyncSourceSelector* syncSourceSelector = nullptr; 128 129 // The oplog fetcher will restart the oplog tailing query this many times on non-cancellation 130 // failures. 131 std::uint32_t oplogFetcherMaxFetcherRestarts = 0; 132 toStringInitialSyncerOptions133 std::string toString() const { 134 return str::stream() << "InitialSyncerOptions -- " 135 << " localOplogNs: " << localOplogNS.toString() 136 << " remoteOplogNS: " << remoteOplogNS.toString(); 137 } 138 }; 139 140 /** 141 * The initial syncer provides services to keep collection in sync by replicating 142 * changes via an oplog source to the local system storage. 143 * 144 * This class will use existing machinery like the Executor to schedule work and 145 * network tasks, as well as provide serial access and synchronization of state. 146 * 147 * 148 * Entry Points: 149 * -- startup: Start initial sync. 150 */ 151 class InitialSyncer { 152 MONGO_DISALLOW_COPYING(InitialSyncer); 153 154 public: 155 /** 156 * Callback function to report last applied optime (with hash) of initial sync. 157 */ 158 typedef stdx::function<void(const StatusWith<OpTimeWithHash>& lastApplied)> OnCompletionFn; 159 160 /** 161 * Callback completion guard for initial syncer. 162 */ 163 using OnCompletionGuard = CallbackCompletionGuard<StatusWith<OpTimeWithHash>>; 164 165 struct InitialSyncAttemptInfo { 166 int durationMillis; 167 Status status; 168 HostAndPort syncSource; 169 170 std::string toString() const; 171 BSONObj toBSON() const; 172 void append(BSONObjBuilder* builder) const; 173 }; 174 175 struct Stats { 176 std::uint32_t failedInitialSyncAttempts{0}; 177 std::uint32_t maxFailedInitialSyncAttempts{0}; 178 Date_t initialSyncStart; 179 Date_t initialSyncEnd; 180 std::vector<InitialSyncer::InitialSyncAttemptInfo> initialSyncAttemptInfos; 181 182 std::string toString() const; 183 BSONObj toBSON() const; 184 void append(BSONObjBuilder* builder) const; 185 }; 186 187 InitialSyncer(InitialSyncerOptions opts, 188 std::unique_ptr<DataReplicatorExternalState> dataReplicatorExternalState, 189 StorageInterface* storage, 190 ReplicationProcess* replicationProcess, 191 const OnCompletionFn& onCompletion); 192 193 virtual ~InitialSyncer(); 194 195 /** 196 * Returns true if an initial sync is currently running or in the process of shutting down. 197 */ 198 bool isActive() const; 199 200 /** 201 * Starts initial sync process, with the provided number of attempts 202 */ 203 Status startup(OperationContext* opCtx, std::uint32_t maxAttempts) noexcept; 204 205 /** 206 * Shuts down replication if "start" has been called, and blocks until shutdown has completed. 207 */ 208 Status shutdown(); 209 210 /** 211 * Block until inactive. 212 */ 213 void join(); 214 215 /** 216 * Returns internal state in a loggable format. 217 */ 218 std::string getDiagnosticString() const; 219 220 /** 221 * Returns stats about the progress of initial sync. If initial sync is not in progress it 222 * returns summary statistics for what occurred during initial sync. 223 */ 224 BSONObj getInitialSyncProgress() const; 225 226 /** 227 * Overrides how executor schedules database work. 228 * 229 * For testing only. 230 */ 231 void setScheduleDbWorkFn_forTest(const CollectionCloner::ScheduleDbWorkFn& scheduleDbWorkFn); 232 233 // State transitions: 234 // PreStart --> Running --> ShuttingDown --> Complete 235 // It is possible to skip intermediate states. For example, calling shutdown() when the data 236 // replicator has not started will transition from PreStart directly to Complete. 237 enum class State { kPreStart, kRunning, kShuttingDown, kComplete }; 238 239 /** 240 * Returns current initial syncer state. 241 * For testing only. 242 */ 243 State getState_forTest() const; 244 245 private: 246 /** 247 * Returns true if we are still processing initial sync tasks (_state is either Running or 248 * Shutdown). 249 */ 250 bool _isActive_inlock() const; 251 252 /** 253 * Cancels all outstanding work. 254 * Used by shutdown() and CompletionGuard::setResultAndCancelRemainingWork(). 255 */ 256 void _cancelRemainingWork_inlock(); 257 258 /** 259 * Returns true if the initial syncer has received a shutdown request (_state is ShuttingDown). 260 */ 261 bool _isShuttingDown() const; 262 bool _isShuttingDown_inlock() const; 263 264 /** 265 * Initial sync flowchart: 266 * 267 * start() 268 * | 269 * | 270 * V 271 * _setUp_inlock() 272 * | 273 * | 274 * V 275 * _startInitialSyncAttemptCallback() 276 * | 277 * | 278 * |<-------+ 279 * | | 280 * | | (bad sync source) 281 * | | 282 * V | 283 * _chooseSyncSourceCallback() 284 * | 285 * | 286 * | (good sync source found) 287 * | 288 * | 289 * V 290 * _truncateOplogAndDropReplicatedDatabases() 291 * | 292 * | 293 * V 294 * _rollbackCheckerResetCallback() 295 * | 296 * | 297 * V 298 * _lastOplogEntryFetcherCallbackForBeginTimestamp() 299 * | 300 * | 301 * V 302 * _fcvFetcherCallback() 303 * | 304 * | 305 * +------------------------------+ 306 * | | 307 * | | 308 * V V 309 * _oplogFetcherCallback() _databasesClonerCallback 310 * | | 311 * | | 312 * | V 313 * | _lastOplogEntryFetcherCallbackForStopTimestamp() 314 * | | | 315 * | | | 316 * | (no ops to apply) | | (have ops to apply) 317 * | | | 318 * | | V 319 * | | _getNextApplierBatchCallback()<-----+ 320 * | | | ^ | 321 * | | | | | 322 * | | | (no docs fetched | | 323 * | | | and end ts not | | 324 * | | | reached) | | 325 * | | | | | 326 * | | V | | 327 * | | _multiApplierCallback()-----+ | 328 * | | | | | 329 * | | | | | 330 * | | | | (docs fetched) | (end ts not 331 * | | | | | reached) 332 * | | | V | 333 * | | | _lastOplogEntryFetcherCallbackAfter- 334 * | | | FetchingMissingDocuments() 335 * | | | | 336 * | | | | 337 * | (reached end timestamp) 338 * | | | | 339 * | V V V 340 * | _rollbackCheckerCheckForRollbackCallback() 341 * | | 342 * | | 343 * +------------------------------+ 344 * | 345 * | 346 * V 347 * _finishInitialSyncAttempt() 348 * | 349 * | 350 * V 351 * _finishCallback() 352 */ 353 354 /** 355 * Sets up internal state to begin initial sync. 356 */ 357 void _setUp_inlock(OperationContext* opCtx, std::uint32_t initialSyncMaxAttempts); 358 359 /** 360 * Tears down internal state before reporting final status to caller. 361 */ 362 void _tearDown_inlock(OperationContext* opCtx, const StatusWith<OpTimeWithHash>& lastApplied); 363 364 /** 365 * Callback to start a single initial sync attempt. 366 */ 367 void _startInitialSyncAttemptCallback(const executor::TaskExecutor::CallbackArgs& callbackArgs, 368 std::uint32_t initialSyncAttempt, 369 std::uint32_t initialSyncMaxAttempts); 370 371 /** 372 * Callback to obtain sync source from sync source selector. 373 * For every initial sync attempt, we will try up to 'numInitialSyncConnectAttempts' times (at 374 * an interval of '_opts.syncSourceRetryWait' ms) to obtain a valid sync source before giving up 375 * and returning ErrorCodes::InitialSyncOplogSourceMissing. 376 */ 377 void _chooseSyncSourceCallback(const executor::TaskExecutor::CallbackArgs& callbackArgs, 378 std::uint32_t chooseSyncSourceAttempt, 379 std::uint32_t chooseSyncSourceMaxAttempts, 380 std::shared_ptr<OnCompletionGuard> onCompletionGuard); 381 382 /** 383 * This function does the following: 384 * 1.) Truncate oplog. 385 * 2.) Drop user databases (replicated dbs). 386 */ 387 Status _truncateOplogAndDropReplicatedDatabases(); 388 389 /** 390 * Callback for rollback checker's first replSetGetRBID command before starting data cloning. 391 */ 392 void _rollbackCheckerResetCallback(const RollbackChecker::Result& result, 393 std::shared_ptr<OnCompletionGuard> onCompletionGuard); 394 395 /** 396 * Callback for first '_lastOplogEntryFetcher' callback. A successful response lets us 397 * determine the starting point for tailing the oplog using the OplogFetcher as well as 398 * setting a reference point for the state of the sync source's oplog when data cloning 399 * completes. 400 */ 401 void _lastOplogEntryFetcherCallbackForBeginTimestamp( 402 const StatusWith<Fetcher::QueryResponse>& result, 403 std::shared_ptr<OnCompletionGuard> onCompletionGuard); 404 405 406 /** 407 * Callback for the '_fCVFetcher'. A successful response lets us check if the remote node 408 * is in a currently acceptable fCV and if it has a 'targetVersion' set. 409 */ 410 void _fcvFetcherCallback(const StatusWith<Fetcher::QueryResponse>& result, 411 std::shared_ptr<OnCompletionGuard> onCompletionGuard, 412 const OpTimeWithHash& lastOpTimeWithHash); 413 414 /** 415 * Callback for oplog fetcher. 416 */ 417 void _oplogFetcherCallback(const Status& status, 418 std::shared_ptr<OnCompletionGuard> onCompletionGuard); 419 420 /** 421 * Callback for DatabasesCloner. 422 */ 423 void _databasesClonerCallback(const Status& status, 424 std::shared_ptr<OnCompletionGuard> onCompletionGuard); 425 426 /** 427 * Callback for second '_lastOplogEntryFetcher' callback. This is scheduled to obtain the stop 428 * timestamp after DatabasesCloner has completed and enables us to determine if the oplog on 429 * the sync source has advanced since we started cloning the databases. 430 */ 431 void _lastOplogEntryFetcherCallbackForStopTimestamp( 432 const StatusWith<Fetcher::QueryResponse>& result, 433 std::shared_ptr<OnCompletionGuard> onCompletionGuard); 434 435 /** 436 * Callback to obtain next batch of operations to apply. 437 */ 438 void _getNextApplierBatchCallback(const executor::TaskExecutor::CallbackArgs& callbackArgs, 439 std::shared_ptr<OnCompletionGuard> onCompletionGuard); 440 441 /** 442 * Callback for MultiApplier completion. 443 */ 444 void _multiApplierCallback(const Status& status, 445 OpTimeWithHash lastApplied, 446 std::uint32_t numApplied, 447 std::shared_ptr<OnCompletionGuard> onCompletionGuard); 448 449 /** 450 * Callback for third '_lastOplogEntryFetcher' callback. This is scheduled after MultiApplier 451 * completed successfully and missing documents were fetched from the sync source while 452 * DataReplicatorExternalState::_multiInitialSyncApply() was processing operations. 453 * This callback will update InitialSyncState::stopTimestamp on success. 454 */ 455 void _lastOplogEntryFetcherCallbackAfterFetchingMissingDocuments( 456 const StatusWith<Fetcher::QueryResponse>& result, 457 std::shared_ptr<OnCompletionGuard> onCompletionGuard); 458 459 /** 460 * Callback for rollback checker's last replSetGetRBID command after cloning data and applying 461 * operations. 462 */ 463 void _rollbackCheckerCheckForRollbackCallback( 464 const RollbackChecker::Result& result, 465 std::shared_ptr<OnCompletionGuard> onCompletionGuard); 466 467 /** 468 * Reports result of current initial sync attempt. May schedule another initial sync attempt 469 * depending on shutdown state and whether we've exhausted all initial sync retries. 470 */ 471 void _finishInitialSyncAttempt(const StatusWith<OpTimeWithHash>& lastApplied); 472 473 /** 474 * Invokes completion callback and transitions state to State::kComplete. 475 */ 476 void _finishCallback(StatusWith<OpTimeWithHash> lastApplied); 477 478 // Obtains a valid sync source from the sync source selector. 479 // Returns error if a sync source cannot be found. 480 StatusWith<HostAndPort> _chooseSyncSource_inlock(); 481 482 /** 483 * Pushes documents from oplog fetcher to blocking queue for 484 * applier to consume. 485 * 486 * Returns a status even though it always returns OK, to conform the interface OplogFetcher 487 * expects for the EnqueueDocumentsFn. 488 */ 489 Status _enqueueDocuments(Fetcher::Documents::const_iterator begin, 490 Fetcher::Documents::const_iterator end, 491 const OplogFetcher::DocumentsInfo& info); 492 493 void _appendInitialSyncProgressMinimal_inlock(BSONObjBuilder* bob) const; 494 BSONObj _getInitialSyncProgress_inlock() const; 495 496 StatusWith<MultiApplier::Operations> _getNextApplierBatch_inlock(); 497 498 /** 499 * Schedules a fetcher to get the last oplog entry from the sync source. 500 */ 501 Status _scheduleLastOplogEntryFetcher_inlock(Fetcher::CallbackFn callback); 502 503 /** 504 * Checks the current oplog application progress (begin and end timestamps). 505 * If necessary, schedules a _getNextApplierBatchCallback() task. 506 * If the stop and end timestamps are inconsistent or if there is an issue scheduling the task, 507 * we set the error status in 'onCompletionGuard' and shut down the OplogFetcher. 508 * Passes 'lock' through to completion guard. 509 */ 510 void _checkApplierProgressAndScheduleGetNextApplierBatch_inlock( 511 const stdx::lock_guard<stdx::mutex>& lock, 512 std::shared_ptr<OnCompletionGuard> onCompletionGuard); 513 514 /** 515 * Schedules a rollback checker to get the rollback ID after data cloning or applying. This 516 * helps us check if a rollback occurred on the sync source. 517 * If we fail to schedule the rollback checker, we set the error status in 'onCompletionGuard' 518 * and shut down the OplogFetcher. 519 * Passes 'lock' through to completion guard. 520 */ 521 void _scheduleRollbackCheckerCheckForRollback_inlock( 522 const stdx::lock_guard<stdx::mutex>& lock, 523 std::shared_ptr<OnCompletionGuard> onCompletionGuard); 524 525 /** 526 * Checks the given status (or embedded status inside the callback args) and current data 527 * replicator shutdown state. If the given status is not OK or if we are shutting down, returns 528 * a new error status that should be passed to _finishCallback. The reason in the new error 529 * status will include 'message'. 530 * Otherwise, returns Status::OK(). 531 */ 532 Status _checkForShutdownAndConvertStatus_inlock( 533 const executor::TaskExecutor::CallbackArgs& callbackArgs, const std::string& message); 534 Status _checkForShutdownAndConvertStatus_inlock(const Status& status, 535 const std::string& message); 536 537 /** 538 * Schedules work to be run by the task executor. 539 * Saves handle if work was successfully scheduled. 540 * Returns scheduleWork status (without the handle). 541 */ 542 Status _scheduleWorkAndSaveHandle_inlock(const executor::TaskExecutor::CallbackFn& work, 543 executor::TaskExecutor::CallbackHandle* handle, 544 const std::string& name); 545 Status _scheduleWorkAtAndSaveHandle_inlock(Date_t when, 546 const executor::TaskExecutor::CallbackFn& work, 547 executor::TaskExecutor::CallbackHandle* handle, 548 const std::string& name); 549 550 /** 551 * Cancels task executor callback handle if not null. 552 */ 553 void _cancelHandle_inlock(executor::TaskExecutor::CallbackHandle handle); 554 555 /** 556 * Starts up component and checks initial syncer's shutdown state at the same time. 557 * If component's startup() fails, resets 'component' (which is assumed to be a unique_ptr 558 * to the component type). 559 */ 560 template <typename Component> 561 Status _startupComponent_inlock(Component& component); 562 563 /** 564 * Shuts down component if not null. 565 */ 566 template <typename Component> 567 void _shutdownComponent_inlock(Component& component); 568 569 // Counts how many documents have been refetched from the source in the current batch. 570 AtomicUInt32 _fetchCount; 571 572 // 573 // All member variables are labeled with one of the following codes indicating the 574 // synchronization rules for accessing them. 575 // 576 // (R) Read-only in concurrent operation; no synchronization required. 577 // (S) Self-synchronizing; access in any way from any context. 578 // (M) Reads and writes guarded by _mutex 579 // (X) Reads and writes must be performed in a callback in _exec 580 // (MX) Must hold _mutex and be in a callback in _exec to write; must either hold 581 // _mutex or be in a callback in _exec to read. 582 583 mutable stdx::mutex _mutex; // (S) 584 const InitialSyncerOptions _opts; // (R) 585 std::unique_ptr<DataReplicatorExternalState> _dataReplicatorExternalState; // (R) 586 executor::TaskExecutor* _exec; // (R) 587 StorageInterface* _storage; // (R) 588 ReplicationProcess* _replicationProcess; // (S) 589 590 // This is invoked with the final status of the initial sync. If startup() fails, this callback 591 // is never invoked. The caller gets the last applied optime with hash when the initial sync 592 // completes successfully or an error status. 593 // '_onCompletion' is cleared on completion (in _finishCallback()) in order to release any 594 // resources that might be held by the callback function object. 595 OnCompletionFn _onCompletion; // (M) 596 597 // Handle to currently scheduled _startInitialSyncAttemptCallback() task. 598 executor::TaskExecutor::CallbackHandle _startInitialSyncAttemptHandle; // (M) 599 600 // Handle to currently scheduled _chooseSyncSourceCallback() task. 601 executor::TaskExecutor::CallbackHandle _chooseSyncSourceHandle; // (M) 602 603 // RollbackChecker to get rollback ID before and after each initial sync attempt. 604 std::unique_ptr<RollbackChecker> _rollbackChecker; // (M) 605 606 // Handle returned from RollbackChecker::reset(). 607 RollbackChecker::CallbackHandle _getBaseRollbackIdHandle; // (M) 608 609 // Handle returned from RollbackChecker::checkForRollback(). 610 RollbackChecker::CallbackHandle _getLastRollbackIdHandle; // (M) 611 612 // Handle to currently scheduled _getNextApplierBatchCallback() task. 613 executor::TaskExecutor::CallbackHandle _getNextApplierBatchHandle; // (M) 614 615 std::unique_ptr<InitialSyncState> _initialSyncState; // (M) 616 std::unique_ptr<OplogFetcher> _oplogFetcher; // (S) 617 std::unique_ptr<Fetcher> _lastOplogEntryFetcher; // (S) 618 std::unique_ptr<Fetcher> _fCVFetcher; // (S) 619 std::unique_ptr<MultiApplier> _applier; // (M) 620 HostAndPort _syncSource; // (M) 621 OpTimeWithHash _lastFetched; // (MX) 622 OpTimeWithHash _lastApplied; // (MX) 623 std::unique_ptr<OplogBuffer> _oplogBuffer; // (M) 624 625 // Used to signal changes in _state. 626 mutable stdx::condition_variable _stateCondition; 627 628 // Current initial syncer state. See comments for State enum class for details. 629 State _state = State::kPreStart; // (M) 630 631 // Passed to CollectionCloner via DatabasesCloner. 632 CollectionCloner::ScheduleDbWorkFn _scheduleDbWorkFn; // (M) 633 634 // Contains stats on the current initial sync request (includes all attempts). 635 // To access these stats in a user-readable format, use getInitialSyncProgress(). 636 Stats _stats; // (M) 637 }; 638 639 } // namespace repl 640 } // namespace mongo 641