1""" 2Return/control aspects of the grains data 3 4Grains set or altered with this module are stored in the 'grains' 5file on the minions. By default, this file is located at: ``/etc/salt/grains`` 6 7.. Note:: 8 9 This does **NOT** override any grains set in the minion config file. 10""" 11 12 13import collections 14import logging 15import math 16import operator 17import os 18import random 19from collections.abc import Mapping 20from functools import reduce # pylint: disable=redefined-builtin 21 22import salt.utils.compat 23import salt.utils.data 24import salt.utils.files 25import salt.utils.json 26import salt.utils.platform 27import salt.utils.yaml 28from salt.defaults import DEFAULT_TARGET_DELIM 29from salt.exceptions import SaltException 30 31__proxyenabled__ = ["*"] 32 33# Seed the grains dict so cython will build 34__grains__ = {} 35 36# Change the default outputter to make it more readable 37__outputter__ = { 38 "items": "nested", 39 "item": "nested", 40 "setval": "nested", 41} 42 43# http://stackoverflow.com/a/12414913/127816 44_infinitedict = lambda: collections.defaultdict(_infinitedict) 45 46_non_existent_key = "NonExistentValueMagicNumberSpK3hnufdHfeBUXCfqVK" 47 48log = logging.getLogger(__name__) 49 50 51def _serial_sanitizer(instr): 52 """Replaces the last 1/4 of a string with X's""" 53 length = len(instr) 54 index = int(math.floor(length * 0.75)) 55 return "{}{}".format(instr[:index], "X" * (length - index)) 56 57 58_FQDN_SANITIZER = lambda x: "MINION.DOMAINNAME" 59_HOSTNAME_SANITIZER = lambda x: "MINION" 60_DOMAINNAME_SANITIZER = lambda x: "DOMAINNAME" 61 62 63# A dictionary of grain -> function mappings for sanitizing grain output. This 64# is used when the 'sanitize' flag is given. 65_SANITIZERS = { 66 "serialnumber": _serial_sanitizer, 67 "domain": _DOMAINNAME_SANITIZER, 68 "fqdn": _FQDN_SANITIZER, 69 "id": _FQDN_SANITIZER, 70 "host": _HOSTNAME_SANITIZER, 71 "localhost": _HOSTNAME_SANITIZER, 72 "nodename": _HOSTNAME_SANITIZER, 73} 74 75 76def get(key, default="", delimiter=DEFAULT_TARGET_DELIM, ordered=True): 77 """ 78 Attempt to retrieve the named value from grains, if the named value is not 79 available return the passed default. The default return is an empty string. 80 81 The value can also represent a value in a nested dict using a ":" delimiter 82 for the dict. This means that if a dict in grains looks like this:: 83 84 {'pkg': {'apache': 'httpd'}} 85 86 To retrieve the value associated with the apache key in the pkg dict this 87 key can be passed:: 88 89 pkg:apache 90 91 92 :param delimiter: 93 Specify an alternate delimiter to use when traversing a nested dict. 94 This is useful for when the desired key contains a colon. See CLI 95 example below for usage. 96 97 .. versionadded:: 2014.7.0 98 99 :param ordered: 100 Outputs an ordered dict if applicable (default: True) 101 102 .. versionadded:: 2016.11.0 103 104 CLI Example: 105 106 .. code-block:: bash 107 108 salt '*' grains.get pkg:apache 109 salt '*' grains.get abc::def|ghi delimiter='|' 110 """ 111 if ordered is True: 112 grains = __grains__ 113 else: 114 grains = salt.utils.json.loads(salt.utils.json.dumps(__grains__)) 115 return salt.utils.data.traverse_dict_and_list(grains, key, default, delimiter) 116 117 118def has_value(key): 119 """ 120 Determine whether a key exists in the grains dictionary. 121 122 Given a grains dictionary that contains the following structure:: 123 124 {'pkg': {'apache': 'httpd'}} 125 126 One would determine if the apache key in the pkg dict exists by:: 127 128 pkg:apache 129 130 CLI Example: 131 132 .. code-block:: bash 133 134 salt '*' grains.has_value pkg:apache 135 """ 136 return ( 137 salt.utils.data.traverse_dict_and_list(__grains__, key, KeyError) 138 is not KeyError 139 ) 140 141 142def items(sanitize=False): 143 """ 144 Return all of the minion's grains 145 146 CLI Example: 147 148 .. code-block:: bash 149 150 salt '*' grains.items 151 152 Sanitized CLI Example: 153 154 .. code-block:: bash 155 156 salt '*' grains.items sanitize=True 157 """ 158 if salt.utils.data.is_true(sanitize): 159 out = dict(__grains__) 160 for key, func in _SANITIZERS.items(): 161 if key in out: 162 out[key] = func(out[key]) 163 return out 164 else: 165 return dict(__grains__) 166 167 168def item(*args, **kwargs): 169 """ 170 Return one or more grains 171 172 CLI Example: 173 174 .. code-block:: bash 175 176 salt '*' grains.item os 177 salt '*' grains.item os osrelease oscodename 178 179 Sanitized CLI Example: 180 181 .. code-block:: bash 182 183 salt '*' grains.item host sanitize=True 184 """ 185 ret = {} 186 default = kwargs.get("default", "") 187 delimiter = kwargs.get("delimiter", DEFAULT_TARGET_DELIM) 188 189 try: 190 for arg in args: 191 ret[arg] = salt.utils.data.traverse_dict_and_list( 192 __grains__, arg, default, delimiter 193 ) 194 except KeyError: 195 pass 196 197 if salt.utils.data.is_true(kwargs.get("sanitize")): 198 for arg, func in _SANITIZERS.items(): 199 if arg in ret: 200 ret[arg] = func(ret[arg]) 201 return ret 202 203 204def setvals(grains, destructive=False, refresh_pillar=True): 205 """ 206 Set new grains values in the grains config file 207 208 destructive 209 If an operation results in a key being removed, delete the key, too. 210 Defaults to False. 211 212 refresh_pillar 213 Whether pillar will be refreshed. 214 Defaults to True. 215 216 CLI Example: 217 218 .. code-block:: bash 219 220 salt '*' grains.setvals "{'key1': 'val1', 'key2': 'val2'}" 221 """ 222 new_grains = grains 223 if not isinstance(new_grains, Mapping): 224 raise SaltException("setvals grains must be a dictionary.") 225 grains = {} 226 if os.path.isfile(__opts__["conf_file"]): 227 if salt.utils.platform.is_proxy(): 228 gfn = os.path.join( 229 os.path.dirname(__opts__["conf_file"]), 230 "proxy.d", 231 __opts__["id"], 232 "grains", 233 ) 234 else: 235 gfn = os.path.join(os.path.dirname(__opts__["conf_file"]), "grains") 236 elif os.path.isdir(__opts__["conf_file"]): 237 if salt.utils.platform.is_proxy(): 238 gfn = os.path.join( 239 __opts__["conf_file"], "proxy.d", __opts__["id"], "grains" 240 ) 241 else: 242 gfn = os.path.join(__opts__["conf_file"], "grains") 243 else: 244 if salt.utils.platform.is_proxy(): 245 gfn = os.path.join( 246 os.path.dirname(__opts__["conf_file"]), 247 "proxy.d", 248 __opts__["id"], 249 "grains", 250 ) 251 else: 252 gfn = os.path.join(os.path.dirname(__opts__["conf_file"]), "grains") 253 254 if os.path.isfile(gfn): 255 with salt.utils.files.fopen(gfn, "rb") as fp_: 256 try: 257 grains = salt.utils.yaml.safe_load(fp_) 258 except salt.utils.yaml.YAMLError as exc: 259 return "Unable to read existing grains file: {}".format(exc) 260 if not isinstance(grains, dict): 261 grains = {} 262 for key, val in new_grains.items(): 263 if val is None and destructive is True: 264 if key in grains: 265 del grains[key] 266 if key in __grains__: 267 del __grains__[key] 268 else: 269 grains[key] = val 270 __grains__[key] = val 271 try: 272 with salt.utils.files.fopen(gfn, "w+", encoding="utf-8") as fp_: 273 salt.utils.yaml.safe_dump(grains, fp_, default_flow_style=False) 274 except OSError: 275 log.error("Unable to write to grains file at %s. Check permissions.", gfn) 276 fn_ = os.path.join(__opts__["cachedir"], "module_refresh") 277 try: 278 with salt.utils.files.flopen(fn_, "w+"): 279 pass 280 except OSError: 281 log.error("Unable to write to cache file %s. Check permissions.", fn_) 282 if not __opts__.get("local", False): 283 # Refresh the grains 284 __salt__["saltutil.refresh_grains"](refresh_pillar=refresh_pillar) 285 # Return the grains we just set to confirm everything was OK 286 return new_grains 287 288 289def setval(key, val, destructive=False, refresh_pillar=True): 290 """ 291 Set a grains value in the grains config file 292 293 key 294 The grain key to be set. 295 296 val 297 The value to set the grain key to. 298 299 destructive 300 If an operation results in a key being removed, delete the key, too. 301 Defaults to False. 302 303 refresh_pillar 304 Whether pillar will be refreshed. 305 Defaults to True. 306 307 CLI Example: 308 309 .. code-block:: bash 310 311 salt '*' grains.setval key val 312 salt '*' grains.setval key "{'sub-key': 'val', 'sub-key2': 'val2'}" 313 """ 314 return setvals({key: val}, destructive, refresh_pillar=refresh_pillar) 315 316 317def append(key, val, convert=False, delimiter=DEFAULT_TARGET_DELIM): 318 """ 319 .. versionadded:: 0.17.0 320 321 Append a value to a list in the grains config file. If the grain doesn't 322 exist, the grain key is added and the value is appended to the new grain 323 as a list item. 324 325 key 326 The grain key to be appended to 327 328 val 329 The value to append to the grain key 330 331 convert 332 If convert is True, convert non-list contents into a list. 333 If convert is False and the grain contains non-list contents, an error 334 is given. Defaults to False. 335 336 delimiter 337 The key can be a nested dict key. Use this parameter to 338 specify the delimiter you use, instead of the default ``:``. 339 You can now append values to a list in nested dictionary grains. If the 340 list doesn't exist at this level, it will be created. 341 342 .. versionadded:: 2014.7.6 343 344 CLI Example: 345 346 .. code-block:: bash 347 348 salt '*' grains.append key val 349 """ 350 grains = get(key, [], delimiter) 351 if convert: 352 if not isinstance(grains, list): 353 grains = [] if grains is None else [grains] 354 if not isinstance(grains, list): 355 return "The key {} is not a valid list".format(key) 356 if val in grains: 357 return "The val {} was already in the list {}".format(val, key) 358 if isinstance(val, list): 359 for item in val: 360 grains.append(item) 361 else: 362 grains.append(val) 363 364 while delimiter in key: 365 key, rest = key.rsplit(delimiter, 1) 366 _grain = get(key, _infinitedict(), delimiter) 367 if isinstance(_grain, dict): 368 _grain.update({rest: grains}) 369 grains = _grain 370 371 return setval(key, grains) 372 373 374def remove(key, val, delimiter=DEFAULT_TARGET_DELIM): 375 """ 376 .. versionadded:: 0.17.0 377 378 Remove a value from a list in the grains config file 379 380 key 381 The grain key to remove. 382 383 val 384 The value to remove. 385 386 delimiter 387 The key can be a nested dict key. Use this parameter to 388 specify the delimiter you use, instead of the default ``:``. 389 You can now append values to a list in nested dictionary grains. If the 390 list doesn't exist at this level, it will be created. 391 392 .. versionadded:: 2015.8.2 393 394 CLI Example: 395 396 .. code-block:: bash 397 398 salt '*' grains.remove key val 399 """ 400 grains = get(key, [], delimiter) 401 if not isinstance(grains, list): 402 return "The key {} is not a valid list".format(key) 403 if val not in grains: 404 return "The val {} was not in the list {}".format(val, key) 405 grains.remove(val) 406 407 while delimiter in key: 408 key, rest = key.rsplit(delimiter, 1) 409 _grain = get(key, None, delimiter) 410 if isinstance(_grain, dict): 411 _grain.update({rest: grains}) 412 grains = _grain 413 414 return setval(key, grains) 415 416 417def delkey(key, force=False): 418 """ 419 .. versionadded:: 2017.7.0 420 421 Remove a grain completely from the grain system, this will remove the 422 grain key and value 423 424 key 425 The grain key from which to delete the value. 426 427 force 428 Force remove the grain even when it is a mapped value. 429 Defaults to False 430 431 CLI Example: 432 433 .. code-block:: bash 434 435 salt '*' grains.delkey key 436 """ 437 return delval(key, destructive=True, force=force) 438 439 440def delval(key, destructive=False, force=False): 441 """ 442 .. versionadded:: 0.17.0 443 444 Delete a grain value from the grains config file. This will just set the 445 grain value to ``None``. To completely remove the grain, run ``grains.delkey`` 446 or pass ``destructive=True`` to ``grains.delval``. 447 448 key 449 The grain key from which to delete the value. 450 451 destructive 452 Delete the key, too. Defaults to False. 453 454 force 455 Force remove the grain even when it is a mapped value. 456 Defaults to False 457 458 CLI Example: 459 460 .. code-block:: bash 461 462 salt '*' grains.delval key 463 """ 464 return set(key, None, destructive=destructive, force=force) 465 466 467def ls(): # pylint: disable=C0103 468 """ 469 Return a list of all available grains 470 471 CLI Example: 472 473 .. code-block:: bash 474 475 salt '*' grains.ls 476 """ 477 return sorted(__grains__) 478 479 480def filter_by(lookup_dict, grain="os_family", merge=None, default="default", base=None): 481 """ 482 .. versionadded:: 0.17.0 483 484 Look up the given grain in a given dictionary for the current OS and return 485 the result 486 487 Although this may occasionally be useful at the CLI, the primary intent of 488 this function is for use in Jinja to make short work of creating lookup 489 tables for OS-specific data. For example: 490 491 .. code-block:: jinja 492 493 {% set apache = salt['grains.filter_by']({ 494 'Debian': {'pkg': 'apache2', 'srv': 'apache2'}, 495 'RedHat': {'pkg': 'httpd', 'srv': 'httpd'}, 496 }, default='Debian') %} 497 498 myapache: 499 pkg.installed: 500 - name: {{ apache.pkg }} 501 service.running: 502 - name: {{ apache.srv }} 503 504 Values in the lookup table may be overridden by values in Pillar. An 505 example Pillar to override values in the example above could be as follows: 506 507 .. code-block:: yaml 508 509 apache: 510 lookup: 511 pkg: apache_13 512 srv: apache 513 514 The call to ``filter_by()`` would be modified as follows to reference those 515 Pillar values: 516 517 .. code-block:: jinja 518 519 {% set apache = salt['grains.filter_by']({ 520 ... 521 }, merge=salt['pillar.get']('apache:lookup')) %} 522 523 524 :param lookup_dict: A dictionary, keyed by a grain, containing a value or 525 values relevant to systems matching that grain. For example, a key 526 could be the grain for an OS and the value could the name of a package 527 on that particular OS. 528 529 .. versionchanged:: 2016.11.0 530 531 The dictionary key could be a globbing pattern. The function will 532 return the corresponding ``lookup_dict`` value where grain value 533 matches the pattern. For example: 534 535 .. code-block:: bash 536 537 # this will render 'got some salt' if Minion ID begins from 'salt' 538 salt '*' grains.filter_by '{salt*: got some salt, default: salt is not here}' id 539 540 :param grain: The name of a grain to match with the current system's 541 grains. For example, the value of the "os_family" grain for the current 542 system could be used to pull values from the ``lookup_dict`` 543 dictionary. 544 545 .. versionchanged:: 2016.11.0 546 547 The grain value could be a list. The function will return the 548 ``lookup_dict`` value for a first found item in the list matching 549 one of the ``lookup_dict`` keys. 550 551 :param merge: A dictionary to merge with the results of the grain selection 552 from ``lookup_dict``. This allows Pillar to override the values in the 553 ``lookup_dict``. This could be useful, for example, to override the 554 values for non-standard package names such as when using a different 555 Python version from the default Python version provided by the OS 556 (e.g., ``python26-mysql`` instead of ``python-mysql``). 557 558 :param default: default lookup_dict's key used if the grain does not exists 559 or if the grain value has no match on lookup_dict. If unspecified 560 the value is "default". 561 562 .. versionadded:: 2014.1.0 563 564 :param base: A lookup_dict key to use for a base dictionary. The 565 grain-selected ``lookup_dict`` is merged over this and then finally 566 the ``merge`` dictionary is merged. This allows common values for 567 each case to be collected in the base and overridden by the grain 568 selection dictionary and the merge dictionary. Default is unset. 569 570 .. versionadded:: 2015.5.0 571 572 CLI Example: 573 574 .. code-block:: bash 575 576 salt '*' grains.filter_by '{Debian: Debheads rule, RedHat: I love my hat}' 577 # this one will render {D: {E: I, G: H}, J: K} 578 salt '*' grains.filter_by '{A: B, C: {D: {E: F, G: H}}}' 'xxx' '{D: {E: I}, J: K}' 'C' 579 # next one renders {A: {B: G}, D: J} 580 salt '*' grains.filter_by '{default: {A: {B: C}, D: E}, F: {A: {B: G}}, H: {D: I}}' 'xxx' '{D: J}' 'F' 'default' 581 # next same as above when default='H' instead of 'F' renders {A: {B: C}, D: J} 582 """ 583 return salt.utils.data.filter_by( 584 lookup_dict=lookup_dict, 585 lookup=grain, 586 traverse=__grains__, 587 merge=merge, 588 default=default, 589 base=base, 590 ) 591 592 593def _dict_from_path(path, val, delimiter=DEFAULT_TARGET_DELIM): 594 """ 595 Given a lookup string in the form of 'foo:bar:baz" return a nested 596 dictionary of the appropriate depth with the final segment as a value. 597 598 >>> _dict_from_path('foo:bar:baz', 'somevalue') 599 {"foo": {"bar": {"baz": "somevalue"}} 600 """ 601 nested_dict = _infinitedict() 602 keys = path.rsplit(delimiter) 603 lastplace = reduce(operator.getitem, keys[:-1], nested_dict) 604 lastplace[keys[-1]] = val 605 606 return nested_dict 607 608 609def get_or_set_hash( 610 name, length=8, chars="abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)" 611): 612 """ 613 Perform a one-time generation of a hash and write it to the local grains. 614 If that grain has already been set return the value instead. 615 616 This is useful for generating passwords or keys that are specific to a 617 single minion that don't need to be stored somewhere centrally. 618 619 State Example: 620 621 .. code-block:: yaml 622 623 some_mysql_user: 624 mysql_user: 625 - present 626 - host: localhost 627 - password: {{ salt['grains.get_or_set_hash']('mysql:some_mysql_user') }} 628 629 CLI Example: 630 631 .. code-block:: bash 632 633 salt '*' grains.get_or_set_hash 'django:SECRET_KEY' 50 634 635 .. warning:: 636 637 This function could return strings which may contain characters which are reserved 638 as directives by the YAML parser, such as strings beginning with ``%``. To avoid 639 issues when using the output of this function in an SLS file containing YAML+Jinja, 640 surround the call with single quotes. 641 """ 642 salt.utils.versions.warn_until( 643 "Phosphorus", 644 "The 'grains.get_or_set_hash' function has been deprecated and it's " 645 "functionality will be completely removed. Reference pillar and SDB " 646 "documentation for secure ways to manage sensitive information. Grains " 647 "are an insecure way to store secrets.", 648 ) 649 ret = get(name, None) 650 651 if ret is None: 652 val = "".join([random.SystemRandom().choice(chars) for _ in range(length)]) 653 654 if DEFAULT_TARGET_DELIM in name: 655 root, rest = name.split(DEFAULT_TARGET_DELIM, 1) 656 curr = get(root, _infinitedict()) 657 val = _dict_from_path(rest, val) 658 curr.update(val) 659 setval(root, curr) 660 else: 661 setval(name, val) 662 663 return get(name) 664 665 666def set(key, val="", force=False, destructive=False, delimiter=DEFAULT_TARGET_DELIM): 667 """ 668 Set a key to an arbitrary value. It is used like setval but works 669 with nested keys. 670 671 This function is conservative. It will only overwrite an entry if 672 its value and the given one are not a list or a dict. The ``force`` 673 parameter is used to allow overwriting in all cases. 674 675 .. versionadded:: 2015.8.0 676 677 :param force: Force writing over existing entry if given or existing 678 values are list or dict. Defaults to False. 679 :param destructive: If an operation results in a key being removed, 680 delete the key, too. Defaults to False. 681 :param delimiter: 682 Specify an alternate delimiter to use when traversing a nested dict, 683 the default being ``:`` 684 685 CLI Example: 686 687 .. code-block:: bash 688 689 salt '*' grains.set 'apps:myApp:port' 2209 690 salt '*' grains.set 'apps:myApp' '{port: 2209}' 691 """ 692 693 ret = {"comment": "", "changes": {}, "result": True} 694 695 # Get val type 696 _new_value_type = "simple" 697 if isinstance(val, dict): 698 _new_value_type = "complex" 699 elif isinstance(val, list): 700 _new_value_type = "complex" 701 702 _non_existent = object() 703 _existing_value = get(key, _non_existent, delimiter) 704 _value = _existing_value 705 706 _existing_value_type = "simple" 707 if _existing_value is _non_existent: 708 _existing_value_type = None 709 elif isinstance(_existing_value, dict): 710 _existing_value_type = "complex" 711 elif isinstance(_existing_value, list): 712 _existing_value_type = "complex" 713 714 if ( 715 _existing_value_type is not None 716 and _existing_value == val 717 and (val is not None or destructive is not True) 718 ): 719 ret["comment"] = "Grain is already set" 720 return ret 721 722 if _existing_value is not None and not force: 723 if _existing_value_type == "complex": 724 ret["comment"] = ( 725 "The key '{}' exists but is a dict or a list. " 726 "Use 'force=True' to overwrite.".format(key) 727 ) 728 ret["result"] = False 729 return ret 730 elif _new_value_type == "complex" and _existing_value_type is not None: 731 ret["comment"] = ( 732 "The key '{}' exists and the given value is a dict or a " 733 "list. Use 'force=True' to overwrite.".format(key) 734 ) 735 ret["result"] = False 736 return ret 737 else: 738 _value = val 739 else: 740 _value = val 741 742 # Process nested grains 743 while delimiter in key: 744 key, rest = key.rsplit(delimiter, 1) 745 _existing_value = get(key, {}, delimiter) 746 if isinstance(_existing_value, dict): 747 if _value is None and destructive: 748 if rest in _existing_value.keys(): 749 _existing_value.pop(rest) 750 else: 751 _existing_value.update({rest: _value}) 752 elif isinstance(_existing_value, list): 753 _list_updated = False 754 for _index, _item in enumerate(_existing_value): 755 if _item == rest: 756 _existing_value[_index] = {rest: _value} 757 _list_updated = True 758 elif isinstance(_item, dict) and rest in _item: 759 _item.update({rest: _value}) 760 _list_updated = True 761 if not _list_updated: 762 _existing_value.append({rest: _value}) 763 elif _existing_value == rest or force: 764 _existing_value = {rest: _value} 765 else: 766 ret["comment"] = ( 767 "The key '{}' value is '{}', which is different from " 768 "the provided key '{}'. Use 'force=True' to overwrite.".format( 769 key, _existing_value, rest 770 ) 771 ) 772 ret["result"] = False 773 return ret 774 _value = _existing_value 775 776 _setval_ret = setval(key, _value, destructive=destructive) 777 if isinstance(_setval_ret, dict): 778 ret["changes"] = _setval_ret 779 else: 780 ret["comment"] = _setval_ret 781 ret["result"] = False 782 return ret 783 784 785def equals(key, value): 786 """ 787 Used to make sure the minion's grain key/value matches. 788 789 Returns ``True`` if matches otherwise ``False``. 790 791 .. versionadded:: 2017.7.0 792 793 CLI Example: 794 795 .. code-block:: bash 796 797 salt '*' grains.equals fqdn <expected_fqdn> 798 salt '*' grains.equals systemd:version 219 799 """ 800 return str(value) == str(get(key)) 801 802 803# Provide a jinja function call compatible get aliased as fetch 804fetch = get 805