1""" 2Functions used for CLI argument handling 3""" 4 5import copy 6import fnmatch 7import inspect 8import logging 9import re 10import shlex 11from collections import namedtuple 12 13import salt.utils.data 14import salt.utils.jid 15import salt.utils.versions 16import salt.utils.yaml 17from salt.exceptions import SaltInvocationError 18 19log = logging.getLogger(__name__) 20 21 22KWARG_REGEX = re.compile(r"^([^\d\W][\w.-]*)=(?!=)(.*)$", re.UNICODE) 23 24 25def _getargspec(func): 26 """ 27 Python 3 wrapper for inspect.getargsspec 28 29 inspect.getargsspec is deprecated and will be removed in Python 3.6. 30 """ 31 _ArgSpec = namedtuple("ArgSpec", "args varargs keywords defaults") 32 33 args, varargs, varkw, defaults, kwonlyargs, _, ann = inspect.getfullargspec( 34 func 35 ) # pylint: disable=no-member 36 if kwonlyargs or ann: 37 raise ValueError( 38 "Function has keyword-only arguments or annotations" 39 ", use getfullargspec() API which can support them" 40 ) 41 return _ArgSpec(args, varargs, varkw, defaults) 42 43 44def clean_kwargs(**kwargs): 45 """ 46 Return a dict without any of the __pub* keys (or any other keys starting 47 with a dunder) from the kwargs dict passed into the execution module 48 functions. These keys are useful for tracking what was used to invoke 49 the function call, but they may not be desirable to have if passing the 50 kwargs forward wholesale. 51 52 Usage example: 53 54 .. code-block:: python 55 56 kwargs = __utils__['args.clean_kwargs'](**kwargs) 57 """ 58 ret = {} 59 for key, val in kwargs.items(): 60 if not key.startswith("__"): 61 ret[key] = val 62 return ret 63 64 65def invalid_kwargs(invalid_kwargs, raise_exc=True): 66 """ 67 Raise a SaltInvocationError if invalid_kwargs is non-empty 68 """ 69 if invalid_kwargs: 70 if isinstance(invalid_kwargs, dict): 71 new_invalid = ["{}={}".format(x, y) for x, y in invalid_kwargs.items()] 72 invalid_kwargs = new_invalid 73 msg = "The following keyword arguments are not valid: {}".format( 74 ", ".join(invalid_kwargs) 75 ) 76 if raise_exc: 77 raise SaltInvocationError(msg) 78 else: 79 return msg 80 81 82def condition_input(args, kwargs): 83 """ 84 Return a single arg structure for the publisher to safely use 85 """ 86 ret = [] 87 for arg in args: 88 if isinstance(arg, int) and salt.utils.jid.is_jid(str(arg)): 89 ret.append(str(arg)) 90 else: 91 ret.append(arg) 92 if isinstance(kwargs, dict) and kwargs: 93 kw_ = {"__kwarg__": True} 94 for key, val in kwargs.items(): 95 kw_[key] = val 96 return ret + [kw_] 97 return ret 98 99 100def parse_input(args, condition=True, no_parse=None): 101 """ 102 Parse out the args and kwargs from a list of input values. Optionally, 103 return the args and kwargs without passing them to condition_input(). 104 105 Don't pull args with key=val apart if it has a newline in it. 106 """ 107 if no_parse is None: 108 no_parse = () 109 _args = [] 110 _kwargs = {} 111 for arg in args: 112 if isinstance(arg, str): 113 arg_name, arg_value = parse_kwarg(arg) 114 if arg_name: 115 _kwargs[arg_name] = ( 116 yamlify_arg(arg_value) if arg_name not in no_parse else arg_value 117 ) 118 else: 119 _args.append(yamlify_arg(arg)) 120 elif isinstance(arg, dict): 121 # Yes, we're popping this key off and adding it back if 122 # condition_input is called below, but this is the only way to 123 # gracefully handle both CLI and API input. 124 if arg.pop("__kwarg__", False) is True: 125 _kwargs.update(arg) 126 else: 127 _args.append(arg) 128 else: 129 _args.append(arg) 130 if condition: 131 return condition_input(_args, _kwargs) 132 return _args, _kwargs 133 134 135def parse_kwarg(string_): 136 """ 137 Parses the string and looks for the following kwarg format: 138 139 "{argument name}={argument value}" 140 141 For example: "my_message=Hello world" 142 143 Returns the kwarg name and value, or (None, None) if the regex was not 144 matched. 145 """ 146 try: 147 return KWARG_REGEX.match(string_).groups() 148 except AttributeError: 149 return None, None 150 151 152def yamlify_arg(arg): 153 """ 154 yaml.safe_load the arg 155 """ 156 if not isinstance(arg, str): 157 return arg 158 159 # YAML loads empty (or all whitespace) strings as None: 160 # 161 # >>> import yaml 162 # >>> yaml.load('') is None 163 # True 164 # >>> yaml.load(' ') is None 165 # True 166 # 167 # Similarly, YAML document start/end markers would not load properly if 168 # passed through PyYAML, as loading '---' results in None and '...' raises 169 # an exception. 170 # 171 # Therefore, skip YAML loading for these cases and just return the string 172 # that was passed in. 173 if arg.strip() in ("", "---", "..."): 174 return arg 175 176 elif "_" in arg and all([x in "0123456789_" for x in arg.strip()]): 177 # When the stripped string includes just digits and underscores, the 178 # underscores are ignored and the digits are combined together and 179 # loaded as an int. We don't want that, so return the original value. 180 return arg 181 182 else: 183 if any(np_char in arg for np_char in ("\t", "\r", "\n")): 184 # Don't mess with this CLI arg, since it has one or more 185 # non-printable whitespace char. Since the CSafeLoader will 186 # sanitize these chars rather than raise an exception, just 187 # skip YAML loading of this argument and keep the argument as 188 # passed on the CLI. 189 return arg 190 191 try: 192 # Explicit late import to avoid circular import. DO NOT MOVE THIS. 193 import salt.utils.yaml 194 195 original_arg = arg 196 if "#" in arg: 197 # Only yamlify if it parses into a non-string type, to prevent 198 # loss of content due to # as comment character 199 parsed_arg = salt.utils.yaml.safe_load(arg) 200 if isinstance(parsed_arg, str) or parsed_arg is None: 201 return arg 202 return parsed_arg 203 if arg == "None": 204 arg = None 205 else: 206 arg = salt.utils.yaml.safe_load(arg) 207 208 if isinstance(arg, dict): 209 # dicts must be wrapped in curly braces 210 if isinstance(original_arg, str) and not original_arg.startswith("{"): 211 return original_arg 212 else: 213 return arg 214 215 elif isinstance(arg, list): 216 # lists must be wrapped in brackets 217 if isinstance(original_arg, str) and not original_arg.startswith("["): 218 return original_arg 219 else: 220 return arg 221 222 elif arg is None or isinstance(arg, (list, float, int, str)): 223 # yaml.safe_load will load '|' and '!' as '', don't let it do that. 224 if arg == "" and original_arg in ("|", "!"): 225 return original_arg 226 # yaml.safe_load will treat '#' as a comment, so a value of '#' 227 # will become None. Keep this value from being stomped as well. 228 elif arg is None and original_arg.strip().startswith("#"): 229 return original_arg 230 # Other times, yaml.safe_load will load '!' as None. Prevent that. 231 elif arg is None and original_arg == "!": 232 return original_arg 233 else: 234 return arg 235 else: 236 # we don't support this type 237 return original_arg 238 except Exception: # pylint: disable=broad-except 239 # In case anything goes wrong... 240 return original_arg 241 242 243def get_function_argspec(func, is_class_method=None): 244 """ 245 A small wrapper around getargspec that also supports callable classes and wrapped functions 246 247 If the given function is a wrapper around another function (i.e. has a 248 ``__wrapped__`` attribute), return the functions specification of the underlying 249 function. 250 251 :param is_class_method: Pass True if you are sure that the function being passed 252 is a class method. The reason for this is that on Python 3 253 ``inspect.ismethod`` only returns ``True`` for bound methods, 254 while on Python 2, it returns ``True`` for bound and unbound 255 methods. So, on Python 3, in case of a class method, you'd 256 need the class to which the function belongs to be instantiated 257 and this is not always wanted. 258 """ 259 if not callable(func): 260 raise TypeError("{} is not a callable".format(func)) 261 262 while hasattr(func, "__wrapped__"): 263 func = func.__wrapped__ 264 265 if is_class_method is True: 266 aspec = _getargspec(func) 267 del aspec.args[0] # self 268 elif inspect.isfunction(func): 269 aspec = _getargspec(func) 270 elif inspect.ismethod(func): 271 aspec = _getargspec(func) 272 del aspec.args[0] # self 273 elif isinstance(func, object): 274 aspec = _getargspec(func.__call__) 275 del aspec.args[0] # self 276 else: 277 try: 278 sig = inspect.signature(func) 279 except TypeError: 280 raise TypeError("Cannot inspect argument list for '{}'".format(func)) 281 else: 282 # argspec-related functions are deprecated in Python 3 in favor of 283 # the new inspect.Signature class, and will be removed at some 284 # point in the Python 3 lifecycle. So, build a namedtuple which 285 # looks like the result of a Python 2 argspec. 286 _ArgSpec = namedtuple("ArgSpec", "args varargs keywords defaults") 287 args = [] 288 defaults = [] 289 varargs = keywords = None 290 for param in sig.parameters.values(): 291 if param.kind == param.POSITIONAL_OR_KEYWORD: 292 args.append(param.name) 293 if param.default is not inspect._empty: 294 defaults.append(param.default) 295 elif param.kind == param.VAR_POSITIONAL: 296 varargs = param.name 297 elif param.kind == param.VAR_KEYWORD: 298 keywords = param.name 299 if is_class_method: 300 del args[0] 301 aspec = _ArgSpec(args, varargs, keywords, tuple(defaults) or None) 302 303 return aspec 304 305 306def shlex_split(s, **kwargs): 307 """ 308 Only split if variable is a string 309 """ 310 if isinstance(s, str): 311 # On PY2, shlex.split will fail with unicode types if there are 312 # non-ascii characters in the string. So, we need to make sure we 313 # invoke it with a str type, and then decode the resulting string back 314 # to unicode to return it. 315 return salt.utils.data.decode( 316 shlex.split(salt.utils.stringutils.to_str(s), **kwargs) 317 ) 318 else: 319 return s 320 321 322def arg_lookup(fun, aspec=None): 323 """ 324 Return a dict containing the arguments and default arguments to the 325 function. 326 """ 327 ret = {"kwargs": {}} 328 if aspec is None: 329 aspec = get_function_argspec(fun) 330 if aspec.defaults: 331 ret["kwargs"] = dict(zip(aspec.args[::-1], aspec.defaults[::-1])) 332 ret["args"] = [arg for arg in aspec.args if arg not in ret["kwargs"]] 333 return ret 334 335 336def argspec_report(functions, module=""): 337 """ 338 Pass in a functions dict as it is returned from the loader and return the 339 argspec function signatures 340 """ 341 ret = {} 342 if "*" in module or "." in module: 343 for fun in fnmatch.filter(functions, module): 344 try: 345 aspec = get_function_argspec(functions[fun]) 346 except TypeError: 347 # this happens if not callable 348 continue 349 350 args, varargs, kwargs, defaults = aspec 351 352 ret[fun] = {} 353 ret[fun]["args"] = args if args else None 354 ret[fun]["defaults"] = defaults if defaults else None 355 ret[fun]["varargs"] = True if varargs else None 356 ret[fun]["kwargs"] = True if kwargs else None 357 358 else: 359 # "sys" should just match sys without also matching sysctl 360 module_dot = module + "." 361 362 for fun in functions: 363 if fun.startswith(module_dot): 364 try: 365 aspec = get_function_argspec(functions[fun]) 366 except TypeError: 367 # this happens if not callable 368 continue 369 370 args, varargs, kwargs, defaults = aspec 371 372 ret[fun] = {} 373 ret[fun]["args"] = args if args else None 374 ret[fun]["defaults"] = defaults if defaults else None 375 ret[fun]["varargs"] = True if varargs else None 376 ret[fun]["kwargs"] = True if kwargs else None 377 378 return ret 379 380 381def split_input(val, mapper=None): 382 """ 383 Take an input value and split it into a list, returning the resulting list 384 """ 385 if mapper is None: 386 mapper = lambda x: x 387 if isinstance(val, list): 388 return list(map(mapper, val)) 389 try: 390 return list(map(mapper, [x.strip() for x in val.split(",")])) 391 except AttributeError: 392 return list(map(mapper, [x.strip() for x in str(val).split(",")])) 393 394 395def test_mode(**kwargs): 396 """ 397 Examines the kwargs passed and returns True if any kwarg which matching 398 "Test" in any variation on capitalization (i.e. "TEST", "Test", "TeSt", 399 etc) contains a True value (as determined by salt.utils.data.is_true). 400 """ 401 # Once is_true is moved, remove this import and fix the ref below 402 import salt.utils 403 404 for arg, value in kwargs.items(): 405 try: 406 if arg.lower() == "test" and salt.utils.data.is_true(value): 407 return True 408 except AttributeError: 409 continue 410 return False 411 412 413def format_call( 414 fun, data, initial_ret=None, expected_extra_kws=(), is_class_method=None 415): 416 """ 417 Build the required arguments and keyword arguments required for the passed 418 function. 419 420 :param fun: The function to get the argspec from 421 :param data: A dictionary containing the required data to build the 422 arguments and keyword arguments. 423 :param initial_ret: The initial return data pre-populated as dictionary or 424 None 425 :param expected_extra_kws: Any expected extra keyword argument names which 426 should not trigger a :ref:`SaltInvocationError` 427 :param is_class_method: Pass True if you are sure that the function being passed 428 is a class method. The reason for this is that on Python 3 429 ``inspect.ismethod`` only returns ``True`` for bound methods, 430 while on Python 2, it returns ``True`` for bound and unbound 431 methods. So, on Python 3, in case of a class method, you'd 432 need the class to which the function belongs to be instantiated 433 and this is not always wanted. 434 :returns: A dictionary with the function required arguments and keyword 435 arguments. 436 """ 437 ret = initial_ret is not None and initial_ret or {} 438 439 ret["args"] = [] 440 ret["kwargs"] = {} 441 442 aspec = get_function_argspec(fun, is_class_method=is_class_method) 443 444 arg_data = arg_lookup(fun, aspec) 445 args = arg_data["args"] 446 kwargs = arg_data["kwargs"] 447 448 # Since we WILL be changing the data dictionary, let's change a copy of it 449 data = data.copy() 450 451 missing_args = [] 452 453 for key in kwargs: 454 try: 455 kwargs[key] = data.pop(key) 456 except KeyError: 457 # Let's leave the default value in place 458 pass 459 460 while args: 461 arg = args.pop(0) 462 try: 463 ret["args"].append(data.pop(arg)) 464 except KeyError: 465 missing_args.append(arg) 466 467 if missing_args: 468 used_args_count = len(ret["args"]) + len(args) 469 args_count = used_args_count + len(missing_args) 470 raise SaltInvocationError( 471 "{} takes at least {} argument{} ({} given)".format( 472 fun.__name__, args_count, args_count > 1 and "s" or "", used_args_count 473 ) 474 ) 475 476 ret["kwargs"].update(kwargs) 477 478 if aspec.keywords: 479 # The function accepts **kwargs, any non expected extra keyword 480 # arguments will made available. 481 for key, value in data.items(): 482 if key in expected_extra_kws: 483 continue 484 ret["kwargs"][key] = value 485 486 # No need to check for extra keyword arguments since they are all 487 # **kwargs now. Return 488 return ret 489 490 # Did not return yet? Lets gather any remaining and unexpected keyword 491 # arguments 492 extra = {} 493 for key, value in data.items(): 494 if key in expected_extra_kws: 495 continue 496 extra[key] = copy.deepcopy(value) 497 498 if extra: 499 # Found unexpected keyword arguments, raise an error to the user 500 if len(extra) == 1: 501 msg = "'{0[0]}' is an invalid keyword argument for '{1}'".format( 502 list(extra.keys()), 503 ret.get( 504 # In case this is being called for a state module 505 "full", 506 # Not a state module, build the name 507 "{}.{}".format(fun.__module__, fun.__name__), 508 ), 509 ) 510 else: 511 msg = "{} and '{}' are invalid keyword arguments for '{}'".format( 512 ", ".join(["'{}'".format(e) for e in extra][:-1]), 513 list(extra.keys())[-1], 514 ret.get( 515 # In case this is being called for a state module 516 "full", 517 # Not a state module, build the name 518 "{}.{}".format(fun.__module__, fun.__name__), 519 ), 520 ) 521 522 raise SaltInvocationError(msg) 523 return ret 524 525 526def parse_function(s): 527 """ 528 Parse a python-like function call syntax. 529 530 For example: module.function(arg, arg, kw=arg, kw=arg) 531 532 This function takes care only about the function name and arguments list carying on quoting 533 and bracketing. It doesn't perform identifiers and other syntax validity check. 534 535 Returns a tuple of three values: function name string, arguments list and keyword arguments 536 dictionary. 537 """ 538 sh = shlex.shlex(s, posix=True) 539 sh.escapedquotes = "\"'" 540 word = [] 541 args = [] 542 kwargs = {} 543 brackets = [] 544 key = None 545 token = None 546 for token in sh: 547 if token == "(": 548 break 549 word.append(token) 550 if not word or token != "(": 551 return None, None, None 552 fname = "".join(word) 553 word = [] 554 good = False 555 for token in sh: 556 if token in "[{(": 557 word.append(token) 558 brackets.append(token) 559 elif (token == "," or token == ")") and not brackets: 560 if key: 561 kwargs[key] = "".join(word) 562 elif word: 563 args.append("".join(word)) 564 if token == ")": 565 good = True 566 break 567 key = None 568 word = [] 569 elif token in "]})": 570 if not brackets or token != {"[": "]", "{": "}", "(": ")"}[brackets.pop()]: 571 break 572 word.append(token) 573 elif token == "=" and not brackets: 574 key = "".join(word) 575 word = [] 576 continue 577 else: 578 word.append(token) 579 if good: 580 return fname, args, kwargs 581 else: 582 return None, None, None 583 584 585def prepare_kwargs(all_kwargs, class_init_kwargs): 586 """ 587 Filter out the kwargs used for the init of the class and the kwargs used to 588 invoke the command required. 589 590 all_kwargs 591 All the kwargs the Execution Function has been invoked. 592 593 class_init_kwargs 594 The kwargs of the ``__init__`` of the class. 595 """ 596 fun_kwargs = {} 597 init_kwargs = {} 598 for karg, warg in all_kwargs.items(): 599 if karg not in class_init_kwargs: 600 if warg is not None: 601 fun_kwargs[karg] = warg 602 continue 603 if warg is not None: 604 init_kwargs[karg] = warg 605 return init_kwargs, fun_kwargs 606