1""" 2Functions for manipulating, inspecting, or otherwise working with data types 3and data structures. 4""" 5 6 7import copy 8import datetime 9import fnmatch 10import functools 11import logging 12import re 13from collections.abc import Mapping, MutableMapping, Sequence 14 15import salt.utils.dictupdate 16import salt.utils.stringutils 17import salt.utils.yaml 18from salt.defaults import DEFAULT_TARGET_DELIM 19from salt.exceptions import SaltException 20from salt.utils.decorators.jinja import jinja_filter 21from salt.utils.odict import OrderedDict 22 23try: 24 import jmespath 25except ImportError: 26 jmespath = None 27 28log = logging.getLogger(__name__) 29 30 31class CaseInsensitiveDict(MutableMapping): 32 """ 33 Inspired by requests' case-insensitive dict implementation, but works with 34 non-string keys as well. 35 """ 36 37 def __init__(self, init=None, **kwargs): 38 """ 39 Force internal dict to be ordered to ensure a consistent iteration 40 order, irrespective of case. 41 """ 42 self._data = OrderedDict() 43 self.update(init or {}, **kwargs) 44 45 def __len__(self): 46 return len(self._data) 47 48 def __setitem__(self, key, value): 49 # Store the case-sensitive key so it is available for dict iteration 50 self._data[to_lowercase(key)] = (key, value) 51 52 def __delitem__(self, key): 53 del self._data[to_lowercase(key)] 54 55 def __getitem__(self, key): 56 return self._data[to_lowercase(key)][1] 57 58 def __iter__(self): 59 return (item[0] for item in self._data.values()) 60 61 def __eq__(self, rval): 62 if not isinstance(rval, Mapping): 63 # Comparing to non-mapping type (e.g. int) is always False 64 return False 65 return dict(self.items_lower()) == dict(CaseInsensitiveDict(rval).items_lower()) 66 67 def __repr__(self): 68 return repr(dict(self.items())) 69 70 def items_lower(self): 71 """ 72 Returns a generator iterating over keys and values, with the keys all 73 being lowercase. 74 """ 75 return ((key, val[1]) for key, val in self._data.items()) 76 77 def copy(self): 78 """ 79 Returns a copy of the object 80 """ 81 return CaseInsensitiveDict(self._data.items()) 82 83 84def __change_case(data, attr, preserve_dict_class=False): 85 """ 86 Calls data.attr() if data has an attribute/method called attr. 87 Processes data recursively if data is a Mapping or Sequence. 88 For Mapping, processes both keys and values. 89 """ 90 try: 91 return getattr(data, attr)() 92 except AttributeError: 93 pass 94 95 data_type = data.__class__ 96 97 if isinstance(data, Mapping): 98 return (data_type if preserve_dict_class else dict)( 99 ( 100 __change_case(key, attr, preserve_dict_class), 101 __change_case(val, attr, preserve_dict_class), 102 ) 103 for key, val in data.items() 104 ) 105 if isinstance(data, Sequence): 106 return data_type( 107 __change_case(item, attr, preserve_dict_class) for item in data 108 ) 109 return data 110 111 112def to_lowercase(data, preserve_dict_class=False): 113 """ 114 Recursively changes everything in data to lowercase. 115 """ 116 return __change_case(data, "lower", preserve_dict_class) 117 118 119def to_uppercase(data, preserve_dict_class=False): 120 """ 121 Recursively changes everything in data to uppercase. 122 """ 123 return __change_case(data, "upper", preserve_dict_class) 124 125 126@jinja_filter("compare_dicts") 127def compare_dicts(old=None, new=None): 128 """ 129 Compare before and after results from various salt functions, returning a 130 dict describing the changes that were made. 131 """ 132 ret = {} 133 for key in set(new or {}).union(old or {}): 134 if key not in old: 135 # New key 136 ret[key] = {"old": "", "new": new[key]} 137 elif key not in new: 138 # Key removed 139 ret[key] = {"new": "", "old": old[key]} 140 elif new[key] != old[key]: 141 # Key modified 142 ret[key] = {"old": old[key], "new": new[key]} 143 return ret 144 145 146@jinja_filter("compare_lists") 147def compare_lists(old=None, new=None): 148 """ 149 Compare before and after results from various salt functions, returning a 150 dict describing the changes that were made 151 """ 152 ret = {} 153 for item in new: 154 if item not in old: 155 ret.setdefault("new", []).append(item) 156 for item in old: 157 if item not in new: 158 ret.setdefault("old", []).append(item) 159 return ret 160 161 162def _remove_circular_refs(ob, _seen=None): 163 """ 164 Generic method to remove circular references from objects. 165 This has been taken from author Martijn Pieters 166 https://stackoverflow.com/questions/44777369/ 167 remove-circular-references-in-dicts-lists-tuples/44777477#44777477 168 :param ob: dict, list, typle, set, and frozenset 169 Standard python object 170 :param object _seen: 171 Object that has circular reference 172 :returns: 173 Cleaned Python object 174 :rtype: 175 type(ob) 176 """ 177 if _seen is None: 178 _seen = set() 179 if id(ob) in _seen: 180 # Here we caught a circular reference. 181 # Alert user and cleanup to continue. 182 log.exception( 183 "Caught a circular reference in data structure below." 184 "Cleaning and continuing execution.\n%r\n", 185 ob, 186 ) 187 return None 188 _seen.add(id(ob)) 189 res = ob 190 if isinstance(ob, dict): 191 res = { 192 _remove_circular_refs(k, _seen): _remove_circular_refs(v, _seen) 193 for k, v in ob.items() 194 } 195 elif isinstance(ob, (list, tuple, set, frozenset)): 196 res = type(ob)(_remove_circular_refs(v, _seen) for v in ob) 197 # remove id again; only *nested* references count 198 _seen.remove(id(ob)) 199 return res 200 201 202def decode( 203 data, 204 encoding=None, 205 errors="strict", 206 keep=False, 207 normalize=False, 208 preserve_dict_class=False, 209 preserve_tuples=False, 210 to_str=False, 211): 212 """ 213 Generic function which will decode whichever type is passed, if necessary. 214 Optionally use to_str=True to ensure strings are str types and not unicode 215 on Python 2. 216 217 If `strict` is True, and `keep` is False, and we fail to decode, a 218 UnicodeDecodeError will be raised. Passing `keep` as True allows for the 219 original value to silently be returned in cases where decoding fails. This 220 can be useful for cases where the data passed to this function is likely to 221 contain binary blobs, such as in the case of cp.recv. 222 223 If `normalize` is True, then unicodedata.normalize() will be used to 224 normalize unicode strings down to a single code point per glyph. It is 225 recommended not to normalize unless you know what you're doing. For 226 instance, if `data` contains a dictionary, it is possible that normalizing 227 will lead to data loss because the following two strings will normalize to 228 the same value: 229 230 - u'\\u044f\\u0438\\u0306\\u0446\\u0430.txt' 231 - u'\\u044f\\u0439\\u0446\\u0430.txt' 232 233 One good use case for normalization is in the test suite. For example, on 234 some platforms such as Mac OS, os.listdir() will produce the first of the 235 two strings above, in which "й" is represented as two code points (i.e. one 236 for the base character, and one for the breve mark). Normalizing allows for 237 a more reliable test case. 238 239 """ 240 # Clean data object before decoding to avoid circular references 241 data = _remove_circular_refs(data) 242 243 _decode_func = ( 244 salt.utils.stringutils.to_unicode 245 if not to_str 246 else salt.utils.stringutils.to_str 247 ) 248 if isinstance(data, Mapping): 249 return decode_dict( 250 data, 251 encoding, 252 errors, 253 keep, 254 normalize, 255 preserve_dict_class, 256 preserve_tuples, 257 to_str, 258 ) 259 if isinstance(data, list): 260 return decode_list( 261 data, 262 encoding, 263 errors, 264 keep, 265 normalize, 266 preserve_dict_class, 267 preserve_tuples, 268 to_str, 269 ) 270 if isinstance(data, tuple): 271 return ( 272 decode_tuple( 273 data, encoding, errors, keep, normalize, preserve_dict_class, to_str 274 ) 275 if preserve_tuples 276 else decode_list( 277 data, 278 encoding, 279 errors, 280 keep, 281 normalize, 282 preserve_dict_class, 283 preserve_tuples, 284 to_str, 285 ) 286 ) 287 if isinstance(data, datetime.datetime): 288 return data.isoformat() 289 try: 290 data = _decode_func(data, encoding, errors, normalize) 291 except TypeError: 292 # to_unicode raises a TypeError when input is not a 293 # string/bytestring/bytearray. This is expected and simply means we 294 # are going to leave the value as-is. 295 pass 296 except UnicodeDecodeError: 297 if not keep: 298 raise 299 return data 300 301 302def decode_dict( 303 data, 304 encoding=None, 305 errors="strict", 306 keep=False, 307 normalize=False, 308 preserve_dict_class=False, 309 preserve_tuples=False, 310 to_str=False, 311): 312 """ 313 Decode all string values to Unicode. Optionally use to_str=True to ensure 314 strings are str types and not unicode on Python 2. 315 """ 316 # Clean data object before decoding to avoid circular references 317 data = _remove_circular_refs(data) 318 319 # Make sure we preserve OrderedDicts 320 ret = data.__class__() if preserve_dict_class else {} 321 for key, value in data.items(): 322 if isinstance(key, tuple): 323 key = ( 324 decode_tuple( 325 key, encoding, errors, keep, normalize, preserve_dict_class, to_str 326 ) 327 if preserve_tuples 328 else decode_list( 329 key, 330 encoding, 331 errors, 332 keep, 333 normalize, 334 preserve_dict_class, 335 preserve_tuples, 336 to_str, 337 ) 338 ) 339 else: 340 try: 341 key = decode( 342 key, 343 encoding, 344 errors, 345 keep, 346 normalize, 347 preserve_dict_class, 348 preserve_tuples, 349 to_str, 350 ) 351 352 except TypeError: 353 # to_unicode raises a TypeError when input is not a 354 # string/bytestring/bytearray. This is expected and simply 355 # means we are going to leave the value as-is. 356 pass 357 except UnicodeDecodeError: 358 if not keep: 359 raise 360 361 if isinstance(value, list): 362 value = decode_list( 363 value, 364 encoding, 365 errors, 366 keep, 367 normalize, 368 preserve_dict_class, 369 preserve_tuples, 370 to_str, 371 ) 372 elif isinstance(value, tuple): 373 value = ( 374 decode_tuple( 375 value, 376 encoding, 377 errors, 378 keep, 379 normalize, 380 preserve_dict_class, 381 to_str, 382 ) 383 if preserve_tuples 384 else decode_list( 385 value, 386 encoding, 387 errors, 388 keep, 389 normalize, 390 preserve_dict_class, 391 preserve_tuples, 392 to_str, 393 ) 394 ) 395 elif isinstance(value, Mapping): 396 value = decode_dict( 397 value, 398 encoding, 399 errors, 400 keep, 401 normalize, 402 preserve_dict_class, 403 preserve_tuples, 404 to_str, 405 ) 406 else: 407 try: 408 value = decode( 409 value, 410 encoding, 411 errors, 412 keep, 413 normalize, 414 preserve_dict_class, 415 preserve_tuples, 416 to_str, 417 ) 418 except TypeError as e: 419 # to_unicode raises a TypeError when input is not a 420 # string/bytestring/bytearray. This is expected and simply 421 # means we are going to leave the value as-is. 422 pass 423 except UnicodeDecodeError: 424 if not keep: 425 raise 426 427 ret[key] = value 428 return ret 429 430 431def decode_list( 432 data, 433 encoding=None, 434 errors="strict", 435 keep=False, 436 normalize=False, 437 preserve_dict_class=False, 438 preserve_tuples=False, 439 to_str=False, 440): 441 """ 442 Decode all string values to Unicode. Optionally use to_str=True to ensure 443 strings are str types and not unicode on Python 2. 444 """ 445 # Clean data object before decoding to avoid circular references 446 data = _remove_circular_refs(data) 447 448 ret = [] 449 for item in data: 450 if isinstance(item, list): 451 item = decode_list( 452 item, 453 encoding, 454 errors, 455 keep, 456 normalize, 457 preserve_dict_class, 458 preserve_tuples, 459 to_str, 460 ) 461 elif isinstance(item, tuple): 462 item = ( 463 decode_tuple( 464 item, encoding, errors, keep, normalize, preserve_dict_class, to_str 465 ) 466 if preserve_tuples 467 else decode_list( 468 item, 469 encoding, 470 errors, 471 keep, 472 normalize, 473 preserve_dict_class, 474 preserve_tuples, 475 to_str, 476 ) 477 ) 478 elif isinstance(item, Mapping): 479 item = decode_dict( 480 item, 481 encoding, 482 errors, 483 keep, 484 normalize, 485 preserve_dict_class, 486 preserve_tuples, 487 to_str, 488 ) 489 else: 490 try: 491 item = decode( 492 item, 493 encoding, 494 errors, 495 keep, 496 normalize, 497 preserve_dict_class, 498 preserve_tuples, 499 to_str, 500 ) 501 502 except TypeError: 503 # to_unicode raises a TypeError when input is not a 504 # string/bytestring/bytearray. This is expected and simply 505 # means we are going to leave the value as-is. 506 pass 507 except UnicodeDecodeError: 508 if not keep: 509 raise 510 511 ret.append(item) 512 return ret 513 514 515def decode_tuple( 516 data, 517 encoding=None, 518 errors="strict", 519 keep=False, 520 normalize=False, 521 preserve_dict_class=False, 522 to_str=False, 523): 524 """ 525 Decode all string values to Unicode. Optionally use to_str=True to ensure 526 strings are str types and not unicode on Python 2. 527 """ 528 return tuple( 529 decode_list( 530 data, encoding, errors, keep, normalize, preserve_dict_class, True, to_str 531 ) 532 ) 533 534 535def encode( 536 data, 537 encoding=None, 538 errors="strict", 539 keep=False, 540 preserve_dict_class=False, 541 preserve_tuples=False, 542): 543 """ 544 Generic function which will encode whichever type is passed, if necessary 545 546 If `strict` is True, and `keep` is False, and we fail to encode, a 547 UnicodeEncodeError will be raised. Passing `keep` as True allows for the 548 original value to silently be returned in cases where encoding fails. This 549 can be useful for cases where the data passed to this function is likely to 550 contain binary blobs. 551 552 """ 553 # Clean data object before encoding to avoid circular references 554 data = _remove_circular_refs(data) 555 556 if isinstance(data, Mapping): 557 return encode_dict( 558 data, encoding, errors, keep, preserve_dict_class, preserve_tuples 559 ) 560 if isinstance(data, list): 561 return encode_list( 562 data, encoding, errors, keep, preserve_dict_class, preserve_tuples 563 ) 564 if isinstance(data, tuple): 565 return ( 566 encode_tuple(data, encoding, errors, keep, preserve_dict_class) 567 if preserve_tuples 568 else encode_list( 569 data, encoding, errors, keep, preserve_dict_class, preserve_tuples 570 ) 571 ) 572 try: 573 return salt.utils.stringutils.to_bytes(data, encoding, errors) 574 except TypeError: 575 # to_bytes raises a TypeError when input is not a 576 # string/bytestring/bytearray. This is expected and simply 577 # means we are going to leave the value as-is. 578 pass 579 except UnicodeEncodeError: 580 if not keep: 581 raise 582 return data 583 584 585@jinja_filter("json_decode_dict") # Remove this for Aluminium 586@jinja_filter("json_encode_dict") 587def encode_dict( 588 data, 589 encoding=None, 590 errors="strict", 591 keep=False, 592 preserve_dict_class=False, 593 preserve_tuples=False, 594): 595 """ 596 Encode all string values to bytes 597 """ 598 # Clean data object before encoding to avoid circular references 599 data = _remove_circular_refs(data) 600 ret = data.__class__() if preserve_dict_class else {} 601 for key, value in data.items(): 602 if isinstance(key, tuple): 603 key = ( 604 encode_tuple(key, encoding, errors, keep, preserve_dict_class) 605 if preserve_tuples 606 else encode_list( 607 key, encoding, errors, keep, preserve_dict_class, preserve_tuples 608 ) 609 ) 610 else: 611 try: 612 key = salt.utils.stringutils.to_bytes(key, encoding, errors) 613 except TypeError: 614 # to_bytes raises a TypeError when input is not a 615 # string/bytestring/bytearray. This is expected and simply 616 # means we are going to leave the value as-is. 617 pass 618 except UnicodeEncodeError: 619 if not keep: 620 raise 621 622 if isinstance(value, list): 623 value = encode_list( 624 value, encoding, errors, keep, preserve_dict_class, preserve_tuples 625 ) 626 elif isinstance(value, tuple): 627 value = ( 628 encode_tuple(value, encoding, errors, keep, preserve_dict_class) 629 if preserve_tuples 630 else encode_list( 631 value, encoding, errors, keep, preserve_dict_class, preserve_tuples 632 ) 633 ) 634 elif isinstance(value, Mapping): 635 value = encode_dict( 636 value, encoding, errors, keep, preserve_dict_class, preserve_tuples 637 ) 638 else: 639 try: 640 value = salt.utils.stringutils.to_bytes(value, encoding, errors) 641 except TypeError: 642 # to_bytes raises a TypeError when input is not a 643 # string/bytestring/bytearray. This is expected and simply 644 # means we are going to leave the value as-is. 645 pass 646 except UnicodeEncodeError: 647 if not keep: 648 raise 649 650 ret[key] = value 651 return ret 652 653 654@jinja_filter("json_decode_list") # Remove this for Aluminium 655@jinja_filter("json_encode_list") 656def encode_list( 657 data, 658 encoding=None, 659 errors="strict", 660 keep=False, 661 preserve_dict_class=False, 662 preserve_tuples=False, 663): 664 """ 665 Encode all string values to bytes 666 """ 667 # Clean data object before encoding to avoid circular references 668 data = _remove_circular_refs(data) 669 670 ret = [] 671 for item in data: 672 if isinstance(item, list): 673 item = encode_list( 674 item, encoding, errors, keep, preserve_dict_class, preserve_tuples 675 ) 676 elif isinstance(item, tuple): 677 item = ( 678 encode_tuple(item, encoding, errors, keep, preserve_dict_class) 679 if preserve_tuples 680 else encode_list( 681 item, encoding, errors, keep, preserve_dict_class, preserve_tuples 682 ) 683 ) 684 elif isinstance(item, Mapping): 685 item = encode_dict( 686 item, encoding, errors, keep, preserve_dict_class, preserve_tuples 687 ) 688 else: 689 try: 690 item = salt.utils.stringutils.to_bytes(item, encoding, errors) 691 except TypeError: 692 # to_bytes raises a TypeError when input is not a 693 # string/bytestring/bytearray. This is expected and simply 694 # means we are going to leave the value as-is. 695 pass 696 except UnicodeEncodeError: 697 if not keep: 698 raise 699 700 ret.append(item) 701 return ret 702 703 704def encode_tuple( 705 data, encoding=None, errors="strict", keep=False, preserve_dict_class=False 706): 707 """ 708 Encode all string values to Unicode 709 """ 710 return tuple(encode_list(data, encoding, errors, keep, preserve_dict_class, True)) 711 712 713@jinja_filter("exactly_n_true") 714def exactly_n(iterable, amount=1): 715 """ 716 Tests that exactly N items in an iterable are "truthy" (neither None, 717 False, nor 0). 718 """ 719 i = iter(iterable) 720 return all(any(i) for j in range(amount)) and not any(i) 721 722 723@jinja_filter("exactly_one_true") 724def exactly_one(iterable): 725 """ 726 Check if only one item is not None, False, or 0 in an iterable. 727 """ 728 return exactly_n(iterable) 729 730 731def filter_by(lookup_dict, lookup, traverse, merge=None, default="default", base=None): 732 """ 733 Common code to filter data structures like grains and pillar 734 """ 735 ret = None 736 # Default value would be an empty list if lookup not found 737 val = traverse_dict_and_list(traverse, lookup, []) 738 739 # Iterate over the list of values to match against patterns in the 740 # lookup_dict keys 741 for each in val if isinstance(val, list) else [val]: 742 for key in lookup_dict: 743 test_key = key if isinstance(key, str) else str(key) 744 test_each = each if isinstance(each, str) else str(each) 745 if fnmatch.fnmatchcase(test_each, test_key): 746 ret = lookup_dict[key] 747 break 748 if ret is not None: 749 break 750 751 if ret is None: 752 ret = lookup_dict.get(default, None) 753 754 if base and base in lookup_dict: 755 base_values = lookup_dict[base] 756 if ret is None: 757 ret = base_values 758 759 elif isinstance(base_values, Mapping): 760 if not isinstance(ret, Mapping): 761 raise SaltException( 762 "filter_by default and look-up values must both be dictionaries." 763 ) 764 ret = salt.utils.dictupdate.update(copy.deepcopy(base_values), ret) 765 766 if merge: 767 if not isinstance(merge, Mapping): 768 raise SaltException("filter_by merge argument must be a dictionary.") 769 770 if ret is None: 771 ret = merge 772 else: 773 salt.utils.dictupdate.update(ret, copy.deepcopy(merge)) 774 775 return ret 776 777 778def traverse_dict(data, key, default=None, delimiter=DEFAULT_TARGET_DELIM): 779 """ 780 Traverse a dict using a colon-delimited (or otherwise delimited, using the 781 'delimiter' param) target string. The target 'foo:bar:baz' will return 782 data['foo']['bar']['baz'] if this value exists, and will otherwise return 783 the dict in the default argument. 784 """ 785 ptr = data 786 try: 787 for each in key.split(delimiter): 788 ptr = ptr[each] 789 except (KeyError, IndexError, TypeError): 790 # Encountered a non-indexable value in the middle of traversing 791 return default 792 return ptr 793 794 795@jinja_filter("traverse") 796def traverse_dict_and_list(data, key, default=None, delimiter=DEFAULT_TARGET_DELIM): 797 """ 798 Traverse a dict or list using a colon-delimited (or otherwise delimited, 799 using the 'delimiter' param) target string. The target 'foo:bar:0' will 800 return data['foo']['bar'][0] if this value exists, and will otherwise 801 return the dict in the default argument. 802 Function will automatically determine the target type. 803 The target 'foo:bar:0' will return data['foo']['bar'][0] if data like 804 {'foo':{'bar':['baz']}} , if data like {'foo':{'bar':{'0':'baz'}}} 805 then return data['foo']['bar']['0'] 806 """ 807 ptr = data 808 if isinstance(key, str): 809 key = key.split(delimiter) 810 811 if isinstance(key, int): 812 key = [key] 813 814 for each in key: 815 if isinstance(ptr, list): 816 try: 817 idx = int(each) 818 except ValueError: 819 embed_match = False 820 # Index was not numeric, lets look at any embedded dicts 821 for embedded in (x for x in ptr if isinstance(x, dict)): 822 try: 823 ptr = embedded[each] 824 embed_match = True 825 break 826 except KeyError: 827 pass 828 if not embed_match: 829 # No embedded dicts matched, return the default 830 return default 831 else: 832 embed_match = False 833 # Index was numeric, lets look at any embedded dicts 834 # using the converted version of each. 835 for embedded in (x for x in ptr if isinstance(x, dict)): 836 try: 837 ptr = embedded[idx] 838 embed_match = True 839 break 840 except KeyError: 841 pass 842 if not embed_match: 843 try: 844 ptr = ptr[idx] 845 except IndexError: 846 return default 847 else: 848 try: 849 ptr = ptr[each] 850 except KeyError: 851 # Late import to avoid circular import 852 import salt.utils.args 853 854 # YAML-load the current key (catches integer/float dict keys) 855 try: 856 loaded_key = salt.utils.args.yamlify_arg(each) 857 except Exception: # pylint: disable=broad-except 858 return default 859 if loaded_key == each: 860 # After YAML-loading, the desired key is unchanged. This 861 # means that the KeyError caught above is a legitimate 862 # failure to match the desired key. Therefore, return the 863 # default. 864 return default 865 else: 866 # YAML-loading the key changed its value, so re-check with 867 # the loaded key. This is how we can match a numeric key 868 # with a string-based expression. 869 try: 870 ptr = ptr[loaded_key] 871 except (KeyError, TypeError): 872 return default 873 except TypeError: 874 return default 875 return ptr 876 877 878def subdict_match( 879 data, expr, delimiter=DEFAULT_TARGET_DELIM, regex_match=False, exact_match=False 880): 881 """ 882 Check for a match in a dictionary using a delimiter character to denote 883 levels of subdicts, and also allowing the delimiter character to be 884 matched. Thus, 'foo:bar:baz' will match data['foo'] == 'bar:baz' and 885 data['foo']['bar'] == 'baz'. The latter would take priority over the 886 former, as more deeply-nested matches are tried first. 887 """ 888 889 def _match(target, pattern, regex_match=False, exact_match=False): 890 # XXX: A lot of this logic is here because of supporting PY2 and PY3, 891 # now that we only support PY3 we should probably re-visit what's going 892 # on here. 893 try: 894 target = str(target).lower() 895 except UnicodeDecodeError: 896 target = salt.utils.stringutils.to_unicode(target).lower() 897 try: 898 pattern = str(pattern).lower() 899 except UnicodeDecodeError: 900 pattern = salt.utils.stringutils.to_unicode(pattern).lower() 901 902 if regex_match: 903 try: 904 return re.match(pattern, target) 905 except Exception: # pylint: disable=broad-except 906 log.error("Invalid regex '%s' in match", pattern) 907 return False 908 else: 909 return ( 910 target == pattern if exact_match else fnmatch.fnmatch(target, pattern) 911 ) 912 913 def _dict_match(target, pattern, regex_match=False, exact_match=False): 914 ret = False 915 wildcard = pattern.startswith("*:") 916 if wildcard: 917 pattern = pattern[2:] 918 919 if pattern == "*": 920 # We are just checking that the key exists 921 ret = True 922 if not ret and pattern in target: 923 # We might want to search for a key 924 ret = True 925 if not ret and subdict_match( 926 target, pattern, regex_match=regex_match, exact_match=exact_match 927 ): 928 ret = True 929 if not ret and wildcard: 930 for key in target: 931 if isinstance(target[key], dict): 932 if _dict_match( 933 target[key], 934 pattern, 935 regex_match=regex_match, 936 exact_match=exact_match, 937 ): 938 return True 939 elif isinstance(target[key], list): 940 for item in target[key]: 941 if _match( 942 item, 943 pattern, 944 regex_match=regex_match, 945 exact_match=exact_match, 946 ): 947 return True 948 elif _match( 949 target[key], 950 pattern, 951 regex_match=regex_match, 952 exact_match=exact_match, 953 ): 954 return True 955 return ret 956 957 splits = expr.split(delimiter) 958 num_splits = len(splits) 959 if num_splits == 1: 960 # Delimiter not present, this can't possibly be a match 961 return False 962 963 # If we have 4 splits, then we have three delimiters. Thus, the indexes we 964 # want to use are 3, 2, and 1, in that order. 965 for idx in range(num_splits - 1, 0, -1): 966 key = delimiter.join(splits[:idx]) 967 if key == "*": 968 # We are matching on everything under the top level, so we need to 969 # treat the match as the entire data being passed in 970 matchstr = expr 971 match = data 972 else: 973 matchstr = delimiter.join(splits[idx:]) 974 match = traverse_dict_and_list(data, key, {}, delimiter=delimiter) 975 log.debug( 976 "Attempting to match '%s' in '%s' using delimiter '%s'", 977 matchstr, 978 key, 979 delimiter, 980 ) 981 if match == {}: 982 continue 983 if isinstance(match, dict): 984 if _dict_match( 985 match, matchstr, regex_match=regex_match, exact_match=exact_match 986 ): 987 return True 988 continue 989 if isinstance(match, (list, tuple)): 990 # We are matching a single component to a single list member 991 for member in match: 992 if isinstance(member, dict): 993 if _dict_match( 994 member, 995 matchstr, 996 regex_match=regex_match, 997 exact_match=exact_match, 998 ): 999 return True 1000 if _match( 1001 member, matchstr, regex_match=regex_match, exact_match=exact_match 1002 ): 1003 return True 1004 continue 1005 if _match(match, matchstr, regex_match=regex_match, exact_match=exact_match): 1006 return True 1007 return False 1008 1009 1010@jinja_filter("substring_in_list") 1011def substr_in_list(string_to_search_for, list_to_search): 1012 """ 1013 Return a boolean value that indicates whether or not a given 1014 string is present in any of the strings which comprise a list 1015 """ 1016 return any(string_to_search_for in s for s in list_to_search) 1017 1018 1019def is_dictlist(data): 1020 """ 1021 Returns True if data is a list of one-element dicts (as found in many SLS 1022 schemas), otherwise returns False 1023 """ 1024 if isinstance(data, list): 1025 for element in data: 1026 if isinstance(element, dict): 1027 if len(element) != 1: 1028 return False 1029 else: 1030 return False 1031 return True 1032 return False 1033 1034 1035def repack_dictlist(data, strict=False, recurse=False, key_cb=None, val_cb=None): 1036 """ 1037 Takes a list of one-element dicts (as found in many SLS schemas) and 1038 repacks into a single dictionary. 1039 """ 1040 if isinstance(data, str): 1041 try: 1042 data = salt.utils.yaml.safe_load(data) 1043 except salt.utils.yaml.parser.ParserError as err: 1044 log.error(err) 1045 return {} 1046 1047 if key_cb is None: 1048 key_cb = lambda x: x 1049 if val_cb is None: 1050 val_cb = lambda x, y: y 1051 1052 valid_non_dict = ((str,), (int,), float) 1053 if isinstance(data, list): 1054 for element in data: 1055 if isinstance(element, valid_non_dict): 1056 continue 1057 if isinstance(element, dict): 1058 if len(element) != 1: 1059 log.error( 1060 "Invalid input for repack_dictlist: key/value pairs " 1061 "must contain only one element (data passed: %s).", 1062 element, 1063 ) 1064 return {} 1065 else: 1066 log.error( 1067 "Invalid input for repack_dictlist: element %s is " 1068 "not a string/dict/numeric value", 1069 element, 1070 ) 1071 return {} 1072 else: 1073 log.error( 1074 "Invalid input for repack_dictlist, data passed is not a list (%s)", data 1075 ) 1076 return {} 1077 1078 ret = {} 1079 for element in data: 1080 if isinstance(element, valid_non_dict): 1081 ret[key_cb(element)] = None 1082 else: 1083 key = next(iter(element)) 1084 val = element[key] 1085 if is_dictlist(val): 1086 if recurse: 1087 ret[key_cb(key)] = repack_dictlist(val, recurse=recurse) 1088 elif strict: 1089 log.error( 1090 "Invalid input for repack_dictlist: nested dictlist " 1091 "found, but recurse is set to False" 1092 ) 1093 return {} 1094 else: 1095 ret[key_cb(key)] = val_cb(key, val) 1096 else: 1097 ret[key_cb(key)] = val_cb(key, val) 1098 return ret 1099 1100 1101@jinja_filter("is_list") 1102def is_list(value): 1103 """ 1104 Check if a variable is a list. 1105 """ 1106 return isinstance(value, list) 1107 1108 1109@jinja_filter("is_iter") 1110def is_iter(thing, ignore=(str,)): 1111 """ 1112 Test if an object is iterable, but not a string type. 1113 1114 Test if an object is an iterator or is iterable itself. By default this 1115 does not return True for string objects. 1116 1117 The `ignore` argument defaults to a list of string types that are not 1118 considered iterable. This can be used to also exclude things like 1119 dictionaries or named tuples. 1120 1121 Based on https://bitbucket.org/petershinners/yter 1122 """ 1123 if ignore and isinstance(thing, ignore): 1124 return False 1125 try: 1126 iter(thing) 1127 return True 1128 except TypeError: 1129 return False 1130 1131 1132@jinja_filter("sorted_ignorecase") 1133def sorted_ignorecase(to_sort): 1134 """ 1135 Sort a list of strings ignoring case. 1136 1137 >>> L = ['foo', 'Foo', 'bar', 'Bar'] 1138 >>> sorted(L) 1139 ['Bar', 'Foo', 'bar', 'foo'] 1140 >>> sorted(L, key=lambda x: x.lower()) 1141 ['bar', 'Bar', 'foo', 'Foo'] 1142 >>> 1143 """ 1144 return sorted(to_sort, key=lambda x: x.lower()) 1145 1146 1147def is_true(value=None): 1148 """ 1149 Returns a boolean value representing the "truth" of the value passed. The 1150 rules for what is a "True" value are: 1151 1152 1. Integer/float values greater than 0 1153 2. The string values "True" and "true" 1154 3. Any object for which bool(obj) returns True 1155 """ 1156 # First, try int/float conversion 1157 try: 1158 value = int(value) 1159 except (ValueError, TypeError): 1160 pass 1161 try: 1162 value = float(value) 1163 except (ValueError, TypeError): 1164 pass 1165 1166 # Now check for truthiness 1167 if isinstance(value, ((int,), float)): 1168 return value > 0 1169 if isinstance(value, str): 1170 return str(value).lower() == "true" 1171 return bool(value) 1172 1173 1174@jinja_filter("mysql_to_dict") 1175def mysql_to_dict(data, key): 1176 """ 1177 Convert MySQL-style output to a python dictionary 1178 """ 1179 ret = {} 1180 headers = [""] 1181 for line in data: 1182 if not line: 1183 continue 1184 if line.startswith("+"): 1185 continue 1186 comps = line.split("|") 1187 for idx, comp in enumerate(comps): 1188 comps[idx] = comp.strip() 1189 if len(headers) > 1: 1190 index = len(headers) - 1 1191 row = {} 1192 for field in range(index): 1193 if field < 1: 1194 continue 1195 row[headers[field]] = salt.utils.stringutils.to_num(comps[field]) 1196 ret[row[key]] = row 1197 else: 1198 headers = comps 1199 return ret 1200 1201 1202def simple_types_filter(data): 1203 """ 1204 Convert the data list, dictionary into simple types, i.e., int, float, string, 1205 bool, etc. 1206 """ 1207 if data is None: 1208 return data 1209 1210 simpletypes_keys = ((str,), str, (int,), float, bool) 1211 simpletypes_values = tuple(list(simpletypes_keys) + [list, tuple]) 1212 1213 if isinstance(data, (list, tuple)): 1214 simplearray = [] 1215 for value in data: 1216 if value is not None: 1217 if isinstance(value, (dict, list)): 1218 value = simple_types_filter(value) 1219 elif not isinstance(value, simpletypes_values): 1220 value = repr(value) 1221 simplearray.append(value) 1222 return simplearray 1223 1224 if isinstance(data, dict): 1225 simpledict = {} 1226 for key, value in data.items(): 1227 if key is not None and not isinstance(key, simpletypes_keys): 1228 key = repr(key) 1229 if value is not None and isinstance(value, (dict, list, tuple)): 1230 value = simple_types_filter(value) 1231 elif value is not None and not isinstance(value, simpletypes_values): 1232 value = repr(value) 1233 simpledict[key] = value 1234 return simpledict 1235 1236 return data 1237 1238 1239def stringify(data): 1240 """ 1241 Given an iterable, returns its items as a list, with any non-string items 1242 converted to unicode strings. 1243 """ 1244 ret = [] 1245 for item in data: 1246 if not isinstance(item, str): 1247 item = str(item) 1248 ret.append(item) 1249 return ret 1250 1251 1252@jinja_filter("json_query") 1253def json_query(data, expr): 1254 """ 1255 Query data using JMESPath language (http://jmespath.org). 1256 1257 Requires the https://github.com/jmespath/jmespath.py library. 1258 1259 :param data: A complex data structure to query 1260 :param expr: A JMESPath expression (query) 1261 :returns: The query result 1262 1263 .. code-block:: jinja 1264 1265 {"services": [ 1266 {"name": "http", "host": "1.2.3.4", "port": 80}, 1267 {"name": "smtp", "host": "1.2.3.5", "port": 25}, 1268 {"name": "ssh", "host": "1.2.3.6", "port": 22}, 1269 ]} | json_query("services[].port") }} 1270 1271 will be rendered as: 1272 1273 .. code-block:: text 1274 1275 [80, 25, 22] 1276 """ 1277 if jmespath is None: 1278 err = "json_query requires jmespath module installed" 1279 log.error(err) 1280 raise RuntimeError(err) 1281 return jmespath.search(expr, data) 1282 1283 1284def _is_not_considered_falsey(value, ignore_types=()): 1285 """ 1286 Helper function for filter_falsey to determine if something is not to be 1287 considered falsey. 1288 1289 :param any value: The value to consider 1290 :param list ignore_types: The types to ignore when considering the value. 1291 1292 :return bool 1293 """ 1294 return isinstance(value, bool) or type(value) in ignore_types or value 1295 1296 1297def filter_falsey(data, recurse_depth=None, ignore_types=()): 1298 """ 1299 Helper function to remove items from an iterable with falsey value. 1300 Removes ``None``, ``{}`` and ``[]``, 0, '' (but does not remove ``False``). 1301 Recurses into sub-iterables if ``recurse`` is set to ``True``. 1302 1303 :param dict/list data: Source iterable (dict, OrderedDict, list, set, ...) to process. 1304 :param int recurse_depth: Recurse this many levels into values that are dicts 1305 or lists to also process those. Default: 0 (do not recurse) 1306 :param list ignore_types: Contains types that can be falsey but must not 1307 be filtered. Default: Only booleans are not filtered. 1308 1309 :return type(data) 1310 1311 .. versionadded:: 3000 1312 """ 1313 filter_element = ( 1314 functools.partial( 1315 filter_falsey, recurse_depth=recurse_depth - 1, ignore_types=ignore_types 1316 ) 1317 if recurse_depth 1318 else lambda x: x 1319 ) 1320 1321 if isinstance(data, dict): 1322 processed_elements = [ 1323 (key, filter_element(value)) for key, value in data.items() 1324 ] 1325 return type(data)( 1326 [ 1327 (key, value) 1328 for key, value in processed_elements 1329 if _is_not_considered_falsey(value, ignore_types=ignore_types) 1330 ] 1331 ) 1332 if is_iter(data): 1333 processed_elements = (filter_element(value) for value in data) 1334 return type(data)( 1335 [ 1336 value 1337 for value in processed_elements 1338 if _is_not_considered_falsey(value, ignore_types=ignore_types) 1339 ] 1340 ) 1341 return data 1342 1343 1344def recursive_diff( 1345 old, new, ignore_keys=None, ignore_order=False, ignore_missing_keys=False 1346): 1347 """ 1348 Performs a recursive diff on mappings and/or iterables and returns the result 1349 in a {'old': values, 'new': values}-style. 1350 Compares dicts and sets unordered (obviously), OrderedDicts and Lists ordered 1351 (but only if both ``old`` and ``new`` are of the same type), 1352 all other Mapping types unordered, and all other iterables ordered. 1353 1354 :param mapping/iterable old: Mapping or Iterable to compare from. 1355 :param mapping/iterable new: Mapping or Iterable to compare to. 1356 :param list ignore_keys: List of keys to ignore when comparing Mappings. 1357 :param bool ignore_order: Compare ordered mapping/iterables as if they were unordered. 1358 :param bool ignore_missing_keys: Do not return keys only present in ``old`` 1359 but missing in ``new``. Only works for regular dicts. 1360 1361 :return dict: Returns dict with keys 'old' and 'new' containing the differences. 1362 """ 1363 ignore_keys = ignore_keys or [] 1364 res = {} 1365 ret_old = copy.deepcopy(old) 1366 ret_new = copy.deepcopy(new) 1367 if ( 1368 isinstance(old, OrderedDict) 1369 and isinstance(new, OrderedDict) 1370 and not ignore_order 1371 ): 1372 append_old, append_new = [], [] 1373 if len(old) != len(new): 1374 min_length = min(len(old), len(new)) 1375 # The list coercion is required for Py3 1376 append_old = list(old.keys())[min_length:] 1377 append_new = list(new.keys())[min_length:] 1378 # Compare ordered 1379 for (key_old, key_new) in zip(old, new): 1380 if key_old == key_new: 1381 if key_old in ignore_keys: 1382 del ret_old[key_old] 1383 del ret_new[key_new] 1384 else: 1385 res = recursive_diff( 1386 old[key_old], 1387 new[key_new], 1388 ignore_keys=ignore_keys, 1389 ignore_order=ignore_order, 1390 ignore_missing_keys=ignore_missing_keys, 1391 ) 1392 if not res: # Equal 1393 del ret_old[key_old] 1394 del ret_new[key_new] 1395 else: 1396 ret_old[key_old] = res["old"] 1397 ret_new[key_new] = res["new"] 1398 else: 1399 if key_old in ignore_keys: 1400 del ret_old[key_old] 1401 if key_new in ignore_keys: 1402 del ret_new[key_new] 1403 # If the OrderedDicts were of inequal length, add the remaining key/values. 1404 for item in append_old: 1405 ret_old[item] = old[item] 1406 for item in append_new: 1407 ret_new[item] = new[item] 1408 ret = {"old": ret_old, "new": ret_new} if ret_old or ret_new else {} 1409 elif isinstance(old, Mapping) and isinstance(new, Mapping): 1410 # Compare unordered 1411 for key in set(list(old) + list(new)): 1412 if key in ignore_keys: 1413 ret_old.pop(key, None) 1414 ret_new.pop(key, None) 1415 elif ignore_missing_keys and key in old and key not in new: 1416 del ret_old[key] 1417 elif key in old and key in new: 1418 res = recursive_diff( 1419 old[key], 1420 new[key], 1421 ignore_keys=ignore_keys, 1422 ignore_order=ignore_order, 1423 ignore_missing_keys=ignore_missing_keys, 1424 ) 1425 if not res: # Equal 1426 del ret_old[key] 1427 del ret_new[key] 1428 else: 1429 ret_old[key] = res["old"] 1430 ret_new[key] = res["new"] 1431 ret = {"old": ret_old, "new": ret_new} if ret_old or ret_new else {} 1432 elif isinstance(old, set) and isinstance(new, set): 1433 ret = {"old": old - new, "new": new - old} if old - new or new - old else {} 1434 elif is_iter(old) and is_iter(new): 1435 # Create a list so we can edit on an index-basis. 1436 list_old = list(ret_old) 1437 list_new = list(ret_new) 1438 if ignore_order: 1439 for item_old in old: 1440 for item_new in new: 1441 res = recursive_diff( 1442 item_old, 1443 item_new, 1444 ignore_keys=ignore_keys, 1445 ignore_order=ignore_order, 1446 ignore_missing_keys=ignore_missing_keys, 1447 ) 1448 if not res: 1449 list_old.remove(item_old) 1450 list_new.remove(item_new) 1451 continue 1452 else: 1453 remove_indices = [] 1454 for index, (iter_old, iter_new) in enumerate(zip(old, new)): 1455 res = recursive_diff( 1456 iter_old, 1457 iter_new, 1458 ignore_keys=ignore_keys, 1459 ignore_order=ignore_order, 1460 ignore_missing_keys=ignore_missing_keys, 1461 ) 1462 if not res: # Equal 1463 remove_indices.append(index) 1464 else: 1465 list_old[index] = res["old"] 1466 list_new[index] = res["new"] 1467 for index in reversed(remove_indices): 1468 list_old.pop(index) 1469 list_new.pop(index) 1470 # Instantiate a new whatever-it-was using the list as iterable source. 1471 # This may not be the most optimized in way of speed and memory usage, 1472 # but it will work for all iterable types. 1473 ret = ( 1474 {"old": type(old)(list_old), "new": type(new)(list_new)} 1475 if list_old or list_new 1476 else {} 1477 ) 1478 else: 1479 ret = {} if old == new else {"old": ret_old, "new": ret_new} 1480 return ret 1481 1482 1483def get_value(obj, path, default=None): 1484 """ 1485 Get the values for a given path. 1486 1487 :param path: 1488 keys of the properties in the tree separated by colons. 1489 One segment in the path can be replaced by an id surrounded by curly braces. 1490 This will match all items in a list of dictionary. 1491 1492 :param default: 1493 default value to return when no value is found 1494 1495 :return: 1496 a list of dictionaries, with at least the "value" key providing the actual value. 1497 If a placeholder was used, the placeholder id will be a key providing the replacement for it. 1498 Note that a value that wasn't found in the tree will be an empty list. 1499 This ensures we can make the difference with a None value set by the user. 1500 """ 1501 res = [{"value": obj}] 1502 if path: 1503 key = path[: path.find(":")] if ":" in path else path 1504 next_path = path[path.find(":") + 1 :] if ":" in path else None 1505 1506 if key.startswith("{") and key.endswith("}"): 1507 placeholder_name = key[1:-1] 1508 # There will be multiple values to get here 1509 items = [] 1510 if obj is None: 1511 return res 1512 if isinstance(obj, dict): 1513 items = obj.items() 1514 elif isinstance(obj, list): 1515 items = enumerate(obj) 1516 1517 def _append_placeholder(value_dict, key): 1518 value_dict[placeholder_name] = key 1519 return value_dict 1520 1521 values = [ 1522 [ 1523 _append_placeholder(item, key) 1524 for item in get_value(val, next_path, default) 1525 ] 1526 for key, val in items 1527 ] 1528 1529 # flatten the list 1530 values = [y for x in values for y in x] 1531 return values 1532 elif isinstance(obj, dict): 1533 if key not in obj.keys(): 1534 return [{"value": default}] 1535 1536 value = obj.get(key) 1537 if res is not None: 1538 res = get_value(value, next_path, default) 1539 else: 1540 res = [{"value": value}] 1541 else: 1542 return [{"value": default if obj is not None else obj}] 1543 return res 1544