1"""
2Utility functions to create MongoDB processes.
3
4Handles all the nitty-gritty parameter conversion.
5"""
6
7from __future__ import absolute_import
8
9import json
10import os
11import os.path
12import stat
13
14from . import process as _process
15from .. import config
16from .. import utils
17
18# The below parameters define the default 'logComponentVerbosity' object passed to mongod processes
19# started either directly via resmoke or those that will get started by the mongo shell. We allow
20# this default to be different for tests run locally and tests run in Evergreen. This allows us, for
21# example, to keep log verbosity high in Evergreen test runs without polluting the logs for
22# developers running local tests.
23
24# The default verbosity setting for any tests that are not started with an Evergreen task id. This
25# will apply to any tests run locally.
26DEFAULT_MONGOD_LOG_COMPONENT_VERBOSITY = {"replication": {"rollback": 2}}
27
28# The default verbosity setting for any tests running in Evergreen i.e. started with an Evergreen
29# task id.
30DEFAULT_EVERGREEN_MONGOD_LOG_COMPONENT_VERBOSITY = {"replication": {"heartbeats": 2, "rollback": 2}}
31
32
33def default_mongod_log_component_verbosity():
34    """Return the default 'logComponentVerbosity' value to use for mongod processes."""
35    if config.EVERGREEN_TASK_ID:
36        return DEFAULT_EVERGREEN_MONGOD_LOG_COMPONENT_VERBOSITY
37    return DEFAULT_MONGOD_LOG_COMPONENT_VERBOSITY
38
39
40def mongod_program(logger, executable=None, process_kwargs=None, **kwargs):
41    """
42    Returns a Process instance that starts a mongod executable with
43    arguments constructed from 'kwargs'.
44    """
45
46    executable = utils.default_if_none(executable, config.DEFAULT_MONGOD_EXECUTABLE)
47    args = [executable]
48
49    # Apply the --setParameter command line argument. Command line options to resmoke.py override
50    # the YAML configuration.
51    suite_set_parameters = kwargs.pop("set_parameters", {})
52
53    if config.MONGOD_SET_PARAMETERS is not None:
54        suite_set_parameters.update(utils.load_yaml(config.MONGOD_SET_PARAMETERS))
55
56    # Set default log verbosity levels if none were specified.
57    if "logComponentVerbosity" not in suite_set_parameters:
58        suite_set_parameters["logComponentVerbosity"] = default_mongod_log_component_verbosity()
59
60    # orphanCleanupDelaySecs controls an artificial delay before cleaning up an orphaned chunk
61    # that has migrated off of a shard, meant to allow most dependent queries on secondaries to
62    # complete first. It defaults to 900, or 15 minutes, which is prohibitively long for tests.
63    # Setting it in the .yml file overrides this.
64    if "shardsvr" in kwargs and "orphanCleanupDelaySecs" not in suite_set_parameters:
65        suite_set_parameters["orphanCleanupDelaySecs"] = 0
66
67    # The LogicalSessionCache does automatic background refreshes in the server. This is
68    # race-y for tests, since tests trigger their own immediate refreshes instead. Turn off
69    # background refreshing for tests. Set in the .yml file to override this.
70    if "disableLogicalSessionCacheRefresh" not in suite_set_parameters:
71        suite_set_parameters["disableLogicalSessionCacheRefresh"] = True
72
73    # The periodic no-op writer writes an oplog entry of type='n' once every 10 seconds. This has
74    # the potential to mask issues such as SERVER-31609 because it allows the operationTime of
75    # cluster to advance even if the client is blocked for other reasons. We should disable the
76    # periodic no-op writer. Set in the .yml file to override this.
77    if "replSet" in kwargs and "writePeriodicNoops" not in suite_set_parameters:
78        suite_set_parameters["writePeriodicNoops"] = False
79
80    # By default the primary waits up to 10 sec to complete a stepdown and to hand off its duties to
81    # a secondary before shutting down in response to SIGTERM. Make it shut down more abruptly.
82    if "replSet" in kwargs and "waitForStepDownOnNonCommandShutdown" not in suite_set_parameters:
83        suite_set_parameters["waitForStepDownOnNonCommandShutdown"] = False
84
85    _apply_set_parameters(args, suite_set_parameters)
86
87    shortcut_opts = {
88        "enableMajorityReadConcern": config.MAJORITY_READ_CONCERN,
89        "nojournal": config.NO_JOURNAL,
90        "nopreallocj": config.NO_PREALLOC_JOURNAL,
91        "serviceExecutor": config.SERVICE_EXECUTOR,
92        "storageEngine": config.STORAGE_ENGINE,
93        "transportLayer": config.TRANSPORT_LAYER,
94        "wiredTigerCollectionConfigString": config.WT_COLL_CONFIG,
95        "wiredTigerEngineConfigString": config.WT_ENGINE_CONFIG,
96        "wiredTigerIndexConfigString": config.WT_INDEX_CONFIG,
97    }
98
99    if config.STORAGE_ENGINE == "rocksdb":
100        shortcut_opts["rocksdbCacheSizeGB"] = config.STORAGE_ENGINE_CACHE_SIZE
101    elif config.STORAGE_ENGINE == "wiredTiger" or config.STORAGE_ENGINE is None:
102        shortcut_opts["wiredTigerCacheSizeGB"] = config.STORAGE_ENGINE_CACHE_SIZE
103
104    # These options are just flags, so they should not take a value.
105    opts_without_vals = ("nojournal", "nopreallocj")
106
107    # Have the --nojournal command line argument to resmoke.py unset the journal option.
108    if shortcut_opts["nojournal"] and "journal" in kwargs:
109        del kwargs["journal"]
110
111    # Ensure that config servers run with journaling enabled.
112    if "configsvr" in kwargs:
113        shortcut_opts["nojournal"] = False
114        kwargs["journal"] = ""
115
116    # Command line options override the YAML configuration.
117    for opt_name in shortcut_opts:
118        opt_value = shortcut_opts[opt_name]
119        if opt_name in opts_without_vals:
120            # Options that are specified as --flag on the command line are represented by a boolean
121            # value where True indicates that the flag should be included in 'kwargs'.
122            if opt_value:
123                kwargs[opt_name] = ""
124        else:
125            # Options that are specified as --key=value on the command line are represented by a
126            # value where None indicates that the key-value pair shouldn't be included in 'kwargs'.
127            if opt_value is not None:
128                kwargs[opt_name] = opt_value
129
130    # Override the storage engine specified on the command line with "wiredTiger" if running a
131    # config server replica set.
132    if "replSet" in kwargs and "configsvr" in kwargs:
133        kwargs["storageEngine"] = "wiredTiger"
134
135    # Apply the rest of the command line arguments.
136    _apply_kwargs(args, kwargs)
137
138    _set_keyfile_permissions(kwargs)
139
140    process_kwargs = utils.default_if_none(process_kwargs, {})
141    return _process.Process(logger, args, **process_kwargs)
142
143
144def mongos_program(logger, executable=None, process_kwargs=None, **kwargs):
145    """
146    Returns a Process instance that starts a mongos executable with
147    arguments constructed from 'kwargs'.
148    """
149
150    executable = utils.default_if_none(executable, config.DEFAULT_MONGOS_EXECUTABLE)
151    args = [executable]
152
153    # Apply the --setParameter command line argument. Command line options to resmoke.py override
154    # the YAML configuration.
155    suite_set_parameters = kwargs.pop("set_parameters", {})
156
157    if config.MONGOS_SET_PARAMETERS is not None:
158        suite_set_parameters.update(utils.load_yaml(config.MONGOS_SET_PARAMETERS))
159
160    _apply_set_parameters(args, suite_set_parameters)
161
162    # Apply the rest of the command line arguments.
163    _apply_kwargs(args, kwargs)
164
165    _set_keyfile_permissions(kwargs)
166
167    process_kwargs = utils.default_if_none(process_kwargs, {})
168    return _process.Process(logger, args, **process_kwargs)
169
170
171def mongo_shell_program(logger, executable=None, connection_string=None, filename=None,
172                        process_kwargs=None, **kwargs):
173    """
174    Returns a Process instance that starts a mongo shell with the given connection string and
175    arguments constructed from 'kwargs'.
176    """
177    connection_string = utils.default_if_none(config.SHELL_CONN_STRING, connection_string)
178
179    executable = utils.default_if_none(executable, config.DEFAULT_MONGO_EXECUTABLE)
180    args = [executable]
181
182    eval_sb = []  # String builder.
183    global_vars = kwargs.pop("global_vars", {}).copy()
184
185    shortcut_opts = {
186        "enableMajorityReadConcern": (config.MAJORITY_READ_CONCERN, True),
187        "noJournal": (config.NO_JOURNAL, False),
188        "noJournalPrealloc": (config.NO_PREALLOC_JOURNAL, False),
189        "serviceExecutor": (config.SERVICE_EXECUTOR, ""),
190        "storageEngine": (config.STORAGE_ENGINE, ""),
191        "storageEngineCacheSizeGB": (config.STORAGE_ENGINE_CACHE_SIZE, ""),
192        "testName": (os.path.splitext(os.path.basename(filename))[0], ""),
193        "transportLayer": (config.TRANSPORT_LAYER, ""),
194        "wiredTigerCollectionConfigString": (config.WT_COLL_CONFIG, ""),
195        "wiredTigerEngineConfigString": (config.WT_ENGINE_CONFIG, ""),
196        "wiredTigerIndexConfigString": (config.WT_INDEX_CONFIG, ""),
197    }
198
199    test_data = global_vars.get("TestData", {}).copy()
200    for opt_name in shortcut_opts:
201        (opt_value, opt_default) = shortcut_opts[opt_name]
202        if opt_value is not None:
203            test_data[opt_name] = opt_value
204        elif opt_name not in test_data:
205            # Only use 'opt_default' if the property wasn't set in the YAML configuration.
206            test_data[opt_name] = opt_default
207
208    global_vars["TestData"] = test_data
209
210    # Initialize setParameters for mongod and mongos, to be passed to the shell via TestData. Since
211    # they are dictionaries, they will be converted to JavaScript objects when passed to the shell
212    # by the _format_shell_vars() function.
213    mongod_set_parameters = {}
214    if config.MONGOD_SET_PARAMETERS is not None:
215        if "setParameters" in test_data:
216            raise ValueError("setParameters passed via TestData can only be set from either the"
217                             " command line or the suite YAML, not both")
218        mongod_set_parameters = utils.load_yaml(config.MONGOD_SET_PARAMETERS)
219
220    # If the 'logComponentVerbosity' setParameter for mongod was not already specified, we set its
221    # value to a default.
222    mongod_set_parameters.setdefault("logComponentVerbosity",
223                                     default_mongod_log_component_verbosity())
224
225    test_data["setParameters"] = mongod_set_parameters
226
227    if config.MONGOS_SET_PARAMETERS is not None:
228        if "setParametersMongos" in test_data:
229            raise ValueError("setParametersMongos passed via TestData can only be set from either"
230                             " the command line or the suite YAML, not both")
231        mongos_set_parameters = utils.load_yaml(config.MONGOS_SET_PARAMETERS)
232        test_data["setParametersMongos"] = mongos_set_parameters
233
234    if "eval_prepend" in kwargs:
235        eval_sb.append(str(kwargs.pop("eval_prepend")))
236
237    # If nodb is specified, pass the connection string through TestData so it can be used inside the
238    # test, then delete it so it isn't given as an argument to the mongo shell.
239    if "nodb" in kwargs and connection_string is not None:
240        test_data["connectionString"] = connection_string
241        connection_string = None
242
243    for var_name in global_vars:
244        _format_shell_vars(eval_sb, var_name, global_vars[var_name])
245
246    if "eval" in kwargs:
247        eval_sb.append(str(kwargs.pop("eval")))
248
249    # Load this file to allow a callback to validate collections before shutting down mongod.
250    eval_sb.append("load('jstests/libs/override_methods/validate_collections_on_shutdown.js');")
251
252    # Load a callback to check UUID consistency before shutting down a ShardingTest.
253    eval_sb.append(
254        "load('jstests/libs/override_methods/check_uuids_consistent_across_cluster.js');")
255
256    eval_str = "; ".join(eval_sb)
257    args.append("--eval")
258    args.append(eval_str)
259
260    if config.SHELL_READ_MODE is not None:
261        kwargs["readMode"] = config.SHELL_READ_MODE
262
263    if config.SHELL_WRITE_MODE is not None:
264        kwargs["writeMode"] = config.SHELL_WRITE_MODE
265
266    if connection_string is not None:
267        # The --host and --port options are ignored by the mongo shell when an explicit connection
268        # string is specified. We remove these options to avoid any ambiguity with what server the
269        # logged mongo shell invocation will connect to.
270        if "port" in kwargs:
271            kwargs.pop("port")
272
273        if "host" in kwargs:
274            kwargs.pop("host")
275
276    # Apply the rest of the command line arguments.
277    _apply_kwargs(args, kwargs)
278
279    if connection_string is not None:
280        args.append(connection_string)
281
282    # Have the mongos shell run the specified file.
283    args.append(filename)
284
285    _set_keyfile_permissions(test_data)
286
287    process_kwargs = utils.default_if_none(process_kwargs, {})
288    return _process.Process(logger, args, **process_kwargs)
289
290
291def _format_shell_vars(sb, path, value):
292    """
293    Formats 'value' in a way that can be passed to --eval.
294
295    If 'value' is a dictionary, then it is unrolled into the creation of
296    a new JSON object with properties assigned for each key of the
297    dictionary.
298    """
299
300    # Only need to do special handling for JSON objects.
301    if not isinstance(value, dict):
302        sb.append("%s = %s" % (path, json.dumps(value)))
303        return
304
305    # Avoid including curly braces and colons in output so that the command invocation can be
306    # copied and run through bash.
307    sb.append("%s = new Object()" % (path))
308    for subkey in value:
309        _format_shell_vars(sb, ".".join((path, subkey)), value[subkey])
310
311
312def dbtest_program(logger, executable=None, suites=None, process_kwargs=None, **kwargs):
313    """
314    Returns a Process instance that starts a dbtest executable with
315    arguments constructed from 'kwargs'.
316    """
317
318    executable = utils.default_if_none(executable, config.DEFAULT_DBTEST_EXECUTABLE)
319    args = [executable]
320
321    if suites is not None:
322        args.extend(suites)
323
324    kwargs["enableMajorityReadConcern"] = config.MAJORITY_READ_CONCERN
325    if config.STORAGE_ENGINE is not None:
326        kwargs["storageEngine"] = config.STORAGE_ENGINE
327
328    return generic_program(logger, args, process_kwargs=process_kwargs, **kwargs)
329
330
331def generic_program(logger, args, process_kwargs=None, **kwargs):
332    """
333    Returns a Process instance that starts an arbitrary executable with
334    arguments constructed from 'kwargs'. The args parameter is an array
335    of strings containing the command to execute.
336    """
337
338    if not utils.is_string_list(args):
339        raise ValueError("The args parameter must be a list of command arguments")
340
341    _apply_kwargs(args, kwargs)
342
343    process_kwargs = utils.default_if_none(process_kwargs, {})
344    return _process.Process(logger, args, **process_kwargs)
345
346
347def _format_test_data_set_parameters(set_parameters):
348    """
349    Converts key-value pairs from 'set_parameters' into the comma
350    delimited list format expected by the parser in servers.js.
351
352    WARNING: the parsing logic in servers.js is very primitive.
353    Non-scalar options such as logComponentVerbosity will not work
354    correctly.
355    """
356    params = []
357    for param_name in set_parameters:
358        param_value = set_parameters[param_name]
359        if isinstance(param_value, bool):
360            # Boolean valued setParameters are specified as lowercase strings.
361            param_value = "true" if param_value else "false"
362        elif isinstance(param_value, dict):
363            raise TypeError("Non-scalar setParameter values are not currently supported.")
364        params.append("%s=%s" % (param_name, param_value))
365    return ",".join(params)
366
367
368def _apply_set_parameters(args, set_parameter):
369    """
370    Converts key-value pairs from 'kwargs' into --setParameter key=value
371    arguments to an executable and appends them to 'args'.
372    """
373
374    for param_name in set_parameter:
375        param_value = set_parameter[param_name]
376        # --setParameter takes boolean values as lowercase strings.
377        if isinstance(param_value, bool):
378            param_value = "true" if param_value else "false"
379        args.append("--setParameter")
380        args.append("%s=%s" % (param_name, param_value))
381
382
383def _apply_kwargs(args, kwargs):
384    """
385    Converts key-value pairs from 'kwargs' into --key value arguments
386    to an executable and appends them to 'args'.
387
388    A --flag without a value is represented with the empty string.
389    """
390
391    for arg_name in kwargs:
392        arg_value = str(kwargs[arg_name])
393        if arg_value:
394            args.append("--%s=%s" % (arg_name, arg_value))
395        else:
396            args.append("--%s" % (arg_name))
397
398
399def _set_keyfile_permissions(opts):
400    """
401    Change the permissions of keyfiles in 'opts' to 600, i.e. only the
402    user can read and write the file.
403
404    This necessary to avoid having the mongod/mongos fail to start up
405    because "permissions on the keyfiles are too open".
406
407    We can't permanently set the keyfile permissions because git is not
408    aware of them.
409    """
410    if "keyFile" in opts:
411        os.chmod(opts["keyFile"], stat.S_IRUSR | stat.S_IWUSR)
412    if "encryptionKeyFile" in opts:
413        os.chmod(opts["encryptionKeyFile"], stat.S_IRUSR | stat.S_IWUSR)
414