1""" 2Utility functions to create MongoDB processes. 3 4Handles all the nitty-gritty parameter conversion. 5""" 6 7from __future__ import absolute_import 8 9import json 10import os 11import os.path 12import stat 13 14from . import process as _process 15from .. import config 16from .. import utils 17 18# The below parameters define the default 'logComponentVerbosity' object passed to mongod processes 19# started either directly via resmoke or those that will get started by the mongo shell. We allow 20# this default to be different for tests run locally and tests run in Evergreen. This allows us, for 21# example, to keep log verbosity high in Evergreen test runs without polluting the logs for 22# developers running local tests. 23 24# The default verbosity setting for any tests that are not started with an Evergreen task id. This 25# will apply to any tests run locally. 26DEFAULT_MONGOD_LOG_COMPONENT_VERBOSITY = {"replication": {"rollback": 2}} 27 28# The default verbosity setting for any tests running in Evergreen i.e. started with an Evergreen 29# task id. 30DEFAULT_EVERGREEN_MONGOD_LOG_COMPONENT_VERBOSITY = {"replication": {"heartbeats": 2, "rollback": 2}} 31 32 33def default_mongod_log_component_verbosity(): 34 """Return the default 'logComponentVerbosity' value to use for mongod processes.""" 35 if config.EVERGREEN_TASK_ID: 36 return DEFAULT_EVERGREEN_MONGOD_LOG_COMPONENT_VERBOSITY 37 return DEFAULT_MONGOD_LOG_COMPONENT_VERBOSITY 38 39 40def mongod_program(logger, executable=None, process_kwargs=None, **kwargs): 41 """ 42 Returns a Process instance that starts a mongod executable with 43 arguments constructed from 'kwargs'. 44 """ 45 46 executable = utils.default_if_none(executable, config.DEFAULT_MONGOD_EXECUTABLE) 47 args = [executable] 48 49 # Apply the --setParameter command line argument. Command line options to resmoke.py override 50 # the YAML configuration. 51 suite_set_parameters = kwargs.pop("set_parameters", {}) 52 53 if config.MONGOD_SET_PARAMETERS is not None: 54 suite_set_parameters.update(utils.load_yaml(config.MONGOD_SET_PARAMETERS)) 55 56 # Set default log verbosity levels if none were specified. 57 if "logComponentVerbosity" not in suite_set_parameters: 58 suite_set_parameters["logComponentVerbosity"] = default_mongod_log_component_verbosity() 59 60 # orphanCleanupDelaySecs controls an artificial delay before cleaning up an orphaned chunk 61 # that has migrated off of a shard, meant to allow most dependent queries on secondaries to 62 # complete first. It defaults to 900, or 15 minutes, which is prohibitively long for tests. 63 # Setting it in the .yml file overrides this. 64 if "shardsvr" in kwargs and "orphanCleanupDelaySecs" not in suite_set_parameters: 65 suite_set_parameters["orphanCleanupDelaySecs"] = 0 66 67 # The LogicalSessionCache does automatic background refreshes in the server. This is 68 # race-y for tests, since tests trigger their own immediate refreshes instead. Turn off 69 # background refreshing for tests. Set in the .yml file to override this. 70 if "disableLogicalSessionCacheRefresh" not in suite_set_parameters: 71 suite_set_parameters["disableLogicalSessionCacheRefresh"] = True 72 73 # The periodic no-op writer writes an oplog entry of type='n' once every 10 seconds. This has 74 # the potential to mask issues such as SERVER-31609 because it allows the operationTime of 75 # cluster to advance even if the client is blocked for other reasons. We should disable the 76 # periodic no-op writer. Set in the .yml file to override this. 77 if "replSet" in kwargs and "writePeriodicNoops" not in suite_set_parameters: 78 suite_set_parameters["writePeriodicNoops"] = False 79 80 # By default the primary waits up to 10 sec to complete a stepdown and to hand off its duties to 81 # a secondary before shutting down in response to SIGTERM. Make it shut down more abruptly. 82 if "replSet" in kwargs and "waitForStepDownOnNonCommandShutdown" not in suite_set_parameters: 83 suite_set_parameters["waitForStepDownOnNonCommandShutdown"] = False 84 85 _apply_set_parameters(args, suite_set_parameters) 86 87 shortcut_opts = { 88 "enableMajorityReadConcern": config.MAJORITY_READ_CONCERN, 89 "nojournal": config.NO_JOURNAL, 90 "nopreallocj": config.NO_PREALLOC_JOURNAL, 91 "serviceExecutor": config.SERVICE_EXECUTOR, 92 "storageEngine": config.STORAGE_ENGINE, 93 "transportLayer": config.TRANSPORT_LAYER, 94 "wiredTigerCollectionConfigString": config.WT_COLL_CONFIG, 95 "wiredTigerEngineConfigString": config.WT_ENGINE_CONFIG, 96 "wiredTigerIndexConfigString": config.WT_INDEX_CONFIG, 97 } 98 99 if config.STORAGE_ENGINE == "rocksdb": 100 shortcut_opts["rocksdbCacheSizeGB"] = config.STORAGE_ENGINE_CACHE_SIZE 101 elif config.STORAGE_ENGINE == "wiredTiger" or config.STORAGE_ENGINE is None: 102 shortcut_opts["wiredTigerCacheSizeGB"] = config.STORAGE_ENGINE_CACHE_SIZE 103 104 # These options are just flags, so they should not take a value. 105 opts_without_vals = ("nojournal", "nopreallocj") 106 107 # Have the --nojournal command line argument to resmoke.py unset the journal option. 108 if shortcut_opts["nojournal"] and "journal" in kwargs: 109 del kwargs["journal"] 110 111 # Ensure that config servers run with journaling enabled. 112 if "configsvr" in kwargs: 113 shortcut_opts["nojournal"] = False 114 kwargs["journal"] = "" 115 116 # Command line options override the YAML configuration. 117 for opt_name in shortcut_opts: 118 opt_value = shortcut_opts[opt_name] 119 if opt_name in opts_without_vals: 120 # Options that are specified as --flag on the command line are represented by a boolean 121 # value where True indicates that the flag should be included in 'kwargs'. 122 if opt_value: 123 kwargs[opt_name] = "" 124 else: 125 # Options that are specified as --key=value on the command line are represented by a 126 # value where None indicates that the key-value pair shouldn't be included in 'kwargs'. 127 if opt_value is not None: 128 kwargs[opt_name] = opt_value 129 130 # Override the storage engine specified on the command line with "wiredTiger" if running a 131 # config server replica set. 132 if "replSet" in kwargs and "configsvr" in kwargs: 133 kwargs["storageEngine"] = "wiredTiger" 134 135 # Apply the rest of the command line arguments. 136 _apply_kwargs(args, kwargs) 137 138 _set_keyfile_permissions(kwargs) 139 140 process_kwargs = utils.default_if_none(process_kwargs, {}) 141 return _process.Process(logger, args, **process_kwargs) 142 143 144def mongos_program(logger, executable=None, process_kwargs=None, **kwargs): 145 """ 146 Returns a Process instance that starts a mongos executable with 147 arguments constructed from 'kwargs'. 148 """ 149 150 executable = utils.default_if_none(executable, config.DEFAULT_MONGOS_EXECUTABLE) 151 args = [executable] 152 153 # Apply the --setParameter command line argument. Command line options to resmoke.py override 154 # the YAML configuration. 155 suite_set_parameters = kwargs.pop("set_parameters", {}) 156 157 if config.MONGOS_SET_PARAMETERS is not None: 158 suite_set_parameters.update(utils.load_yaml(config.MONGOS_SET_PARAMETERS)) 159 160 _apply_set_parameters(args, suite_set_parameters) 161 162 # Apply the rest of the command line arguments. 163 _apply_kwargs(args, kwargs) 164 165 _set_keyfile_permissions(kwargs) 166 167 process_kwargs = utils.default_if_none(process_kwargs, {}) 168 return _process.Process(logger, args, **process_kwargs) 169 170 171def mongo_shell_program(logger, executable=None, connection_string=None, filename=None, 172 process_kwargs=None, **kwargs): 173 """ 174 Returns a Process instance that starts a mongo shell with the given connection string and 175 arguments constructed from 'kwargs'. 176 """ 177 connection_string = utils.default_if_none(config.SHELL_CONN_STRING, connection_string) 178 179 executable = utils.default_if_none(executable, config.DEFAULT_MONGO_EXECUTABLE) 180 args = [executable] 181 182 eval_sb = [] # String builder. 183 global_vars = kwargs.pop("global_vars", {}).copy() 184 185 shortcut_opts = { 186 "enableMajorityReadConcern": (config.MAJORITY_READ_CONCERN, True), 187 "noJournal": (config.NO_JOURNAL, False), 188 "noJournalPrealloc": (config.NO_PREALLOC_JOURNAL, False), 189 "serviceExecutor": (config.SERVICE_EXECUTOR, ""), 190 "storageEngine": (config.STORAGE_ENGINE, ""), 191 "storageEngineCacheSizeGB": (config.STORAGE_ENGINE_CACHE_SIZE, ""), 192 "testName": (os.path.splitext(os.path.basename(filename))[0], ""), 193 "transportLayer": (config.TRANSPORT_LAYER, ""), 194 "wiredTigerCollectionConfigString": (config.WT_COLL_CONFIG, ""), 195 "wiredTigerEngineConfigString": (config.WT_ENGINE_CONFIG, ""), 196 "wiredTigerIndexConfigString": (config.WT_INDEX_CONFIG, ""), 197 } 198 199 test_data = global_vars.get("TestData", {}).copy() 200 for opt_name in shortcut_opts: 201 (opt_value, opt_default) = shortcut_opts[opt_name] 202 if opt_value is not None: 203 test_data[opt_name] = opt_value 204 elif opt_name not in test_data: 205 # Only use 'opt_default' if the property wasn't set in the YAML configuration. 206 test_data[opt_name] = opt_default 207 208 global_vars["TestData"] = test_data 209 210 # Initialize setParameters for mongod and mongos, to be passed to the shell via TestData. Since 211 # they are dictionaries, they will be converted to JavaScript objects when passed to the shell 212 # by the _format_shell_vars() function. 213 mongod_set_parameters = {} 214 if config.MONGOD_SET_PARAMETERS is not None: 215 if "setParameters" in test_data: 216 raise ValueError("setParameters passed via TestData can only be set from either the" 217 " command line or the suite YAML, not both") 218 mongod_set_parameters = utils.load_yaml(config.MONGOD_SET_PARAMETERS) 219 220 # If the 'logComponentVerbosity' setParameter for mongod was not already specified, we set its 221 # value to a default. 222 mongod_set_parameters.setdefault("logComponentVerbosity", 223 default_mongod_log_component_verbosity()) 224 225 test_data["setParameters"] = mongod_set_parameters 226 227 if config.MONGOS_SET_PARAMETERS is not None: 228 if "setParametersMongos" in test_data: 229 raise ValueError("setParametersMongos passed via TestData can only be set from either" 230 " the command line or the suite YAML, not both") 231 mongos_set_parameters = utils.load_yaml(config.MONGOS_SET_PARAMETERS) 232 test_data["setParametersMongos"] = mongos_set_parameters 233 234 if "eval_prepend" in kwargs: 235 eval_sb.append(str(kwargs.pop("eval_prepend"))) 236 237 # If nodb is specified, pass the connection string through TestData so it can be used inside the 238 # test, then delete it so it isn't given as an argument to the mongo shell. 239 if "nodb" in kwargs and connection_string is not None: 240 test_data["connectionString"] = connection_string 241 connection_string = None 242 243 for var_name in global_vars: 244 _format_shell_vars(eval_sb, var_name, global_vars[var_name]) 245 246 if "eval" in kwargs: 247 eval_sb.append(str(kwargs.pop("eval"))) 248 249 # Load this file to allow a callback to validate collections before shutting down mongod. 250 eval_sb.append("load('jstests/libs/override_methods/validate_collections_on_shutdown.js');") 251 252 # Load a callback to check UUID consistency before shutting down a ShardingTest. 253 eval_sb.append( 254 "load('jstests/libs/override_methods/check_uuids_consistent_across_cluster.js');") 255 256 eval_str = "; ".join(eval_sb) 257 args.append("--eval") 258 args.append(eval_str) 259 260 if config.SHELL_READ_MODE is not None: 261 kwargs["readMode"] = config.SHELL_READ_MODE 262 263 if config.SHELL_WRITE_MODE is not None: 264 kwargs["writeMode"] = config.SHELL_WRITE_MODE 265 266 if connection_string is not None: 267 # The --host and --port options are ignored by the mongo shell when an explicit connection 268 # string is specified. We remove these options to avoid any ambiguity with what server the 269 # logged mongo shell invocation will connect to. 270 if "port" in kwargs: 271 kwargs.pop("port") 272 273 if "host" in kwargs: 274 kwargs.pop("host") 275 276 # Apply the rest of the command line arguments. 277 _apply_kwargs(args, kwargs) 278 279 if connection_string is not None: 280 args.append(connection_string) 281 282 # Have the mongos shell run the specified file. 283 args.append(filename) 284 285 _set_keyfile_permissions(test_data) 286 287 process_kwargs = utils.default_if_none(process_kwargs, {}) 288 return _process.Process(logger, args, **process_kwargs) 289 290 291def _format_shell_vars(sb, path, value): 292 """ 293 Formats 'value' in a way that can be passed to --eval. 294 295 If 'value' is a dictionary, then it is unrolled into the creation of 296 a new JSON object with properties assigned for each key of the 297 dictionary. 298 """ 299 300 # Only need to do special handling for JSON objects. 301 if not isinstance(value, dict): 302 sb.append("%s = %s" % (path, json.dumps(value))) 303 return 304 305 # Avoid including curly braces and colons in output so that the command invocation can be 306 # copied and run through bash. 307 sb.append("%s = new Object()" % (path)) 308 for subkey in value: 309 _format_shell_vars(sb, ".".join((path, subkey)), value[subkey]) 310 311 312def dbtest_program(logger, executable=None, suites=None, process_kwargs=None, **kwargs): 313 """ 314 Returns a Process instance that starts a dbtest executable with 315 arguments constructed from 'kwargs'. 316 """ 317 318 executable = utils.default_if_none(executable, config.DEFAULT_DBTEST_EXECUTABLE) 319 args = [executable] 320 321 if suites is not None: 322 args.extend(suites) 323 324 kwargs["enableMajorityReadConcern"] = config.MAJORITY_READ_CONCERN 325 if config.STORAGE_ENGINE is not None: 326 kwargs["storageEngine"] = config.STORAGE_ENGINE 327 328 return generic_program(logger, args, process_kwargs=process_kwargs, **kwargs) 329 330 331def generic_program(logger, args, process_kwargs=None, **kwargs): 332 """ 333 Returns a Process instance that starts an arbitrary executable with 334 arguments constructed from 'kwargs'. The args parameter is an array 335 of strings containing the command to execute. 336 """ 337 338 if not utils.is_string_list(args): 339 raise ValueError("The args parameter must be a list of command arguments") 340 341 _apply_kwargs(args, kwargs) 342 343 process_kwargs = utils.default_if_none(process_kwargs, {}) 344 return _process.Process(logger, args, **process_kwargs) 345 346 347def _format_test_data_set_parameters(set_parameters): 348 """ 349 Converts key-value pairs from 'set_parameters' into the comma 350 delimited list format expected by the parser in servers.js. 351 352 WARNING: the parsing logic in servers.js is very primitive. 353 Non-scalar options such as logComponentVerbosity will not work 354 correctly. 355 """ 356 params = [] 357 for param_name in set_parameters: 358 param_value = set_parameters[param_name] 359 if isinstance(param_value, bool): 360 # Boolean valued setParameters are specified as lowercase strings. 361 param_value = "true" if param_value else "false" 362 elif isinstance(param_value, dict): 363 raise TypeError("Non-scalar setParameter values are not currently supported.") 364 params.append("%s=%s" % (param_name, param_value)) 365 return ",".join(params) 366 367 368def _apply_set_parameters(args, set_parameter): 369 """ 370 Converts key-value pairs from 'kwargs' into --setParameter key=value 371 arguments to an executable and appends them to 'args'. 372 """ 373 374 for param_name in set_parameter: 375 param_value = set_parameter[param_name] 376 # --setParameter takes boolean values as lowercase strings. 377 if isinstance(param_value, bool): 378 param_value = "true" if param_value else "false" 379 args.append("--setParameter") 380 args.append("%s=%s" % (param_name, param_value)) 381 382 383def _apply_kwargs(args, kwargs): 384 """ 385 Converts key-value pairs from 'kwargs' into --key value arguments 386 to an executable and appends them to 'args'. 387 388 A --flag without a value is represented with the empty string. 389 """ 390 391 for arg_name in kwargs: 392 arg_value = str(kwargs[arg_name]) 393 if arg_value: 394 args.append("--%s=%s" % (arg_name, arg_value)) 395 else: 396 args.append("--%s" % (arg_name)) 397 398 399def _set_keyfile_permissions(opts): 400 """ 401 Change the permissions of keyfiles in 'opts' to 600, i.e. only the 402 user can read and write the file. 403 404 This necessary to avoid having the mongod/mongos fail to start up 405 because "permissions on the keyfiles are too open". 406 407 We can't permanently set the keyfile permissions because git is not 408 aware of them. 409 """ 410 if "keyFile" in opts: 411 os.chmod(opts["keyFile"], stat.S_IRUSR | stat.S_IWUSR) 412 if "encryptionKeyFile" in opts: 413 os.chmod(opts["encryptionKeyFile"], stat.S_IRUSR | stat.S_IWUSR) 414