1""" 2The Salt loader is the core to Salt's plugin system, the loader scans 3directories for python loadable code and organizes the code into the 4plugin interfaces used by Salt. 5""" 6 7import contextlib 8import logging 9import os 10import re 11import time 12import types 13 14import salt.config 15import salt.defaults.events 16import salt.defaults.exitcodes 17import salt.loader.context 18import salt.syspaths 19import salt.utils.args 20import salt.utils.context 21import salt.utils.data 22import salt.utils.dictupdate 23import salt.utils.event 24import salt.utils.files 25import salt.utils.lazy 26import salt.utils.odict 27import salt.utils.platform 28import salt.utils.stringutils 29import salt.utils.versions 30from salt.exceptions import LoaderError 31from salt.template import check_render_pipe_str 32from salt.utils import entrypoints 33 34from .lazy import SALT_BASE_PATH, FilterDictWrapper, LazyLoader 35 36log = logging.getLogger(__name__) 37 38# Because on the cloud drivers we do `from salt.cloud.libcloudfuncs import *` 39# which simplifies code readability, it adds some unsupported functions into 40# the driver's module scope. 41# We list un-supported functions here. These will be removed from the loaded. 42# TODO: remove the need for this cross-module code. Maybe use NotImplemented 43LIBCLOUD_FUNCS_NOT_SUPPORTED = ( 44 "parallels.avail_sizes", 45 "parallels.avail_locations", 46 "proxmox.avail_sizes", 47) 48 49SALT_INTERNAL_LOADERS_PATHS = ( 50 str(SALT_BASE_PATH / "auth"), 51 str(SALT_BASE_PATH / "beacons"), 52 str(SALT_BASE_PATH / "cache"), 53 str(SALT_BASE_PATH / "client" / "ssh" / "wrapper"), 54 str(SALT_BASE_PATH / "cloud" / "clouds"), 55 str(SALT_BASE_PATH / "engines"), 56 str(SALT_BASE_PATH / "executors"), 57 str(SALT_BASE_PATH / "fileserver"), 58 str(SALT_BASE_PATH / "grains"), 59 str(SALT_BASE_PATH / "log" / "handlers"), 60 str(SALT_BASE_PATH / "matchers"), 61 str(SALT_BASE_PATH / "metaproxy"), 62 str(SALT_BASE_PATH / "modules"), 63 str(SALT_BASE_PATH / "netapi"), 64 str(SALT_BASE_PATH / "output"), 65 str(SALT_BASE_PATH / "pillar"), 66 str(SALT_BASE_PATH / "proxy"), 67 str(SALT_BASE_PATH / "queues"), 68 str(SALT_BASE_PATH / "renderers"), 69 str(SALT_BASE_PATH / "returners"), 70 str(SALT_BASE_PATH / "roster"), 71 str(SALT_BASE_PATH / "runners"), 72 str(SALT_BASE_PATH / "sdb"), 73 str(SALT_BASE_PATH / "serializers"), 74 str(SALT_BASE_PATH / "spm" / "pkgdb"), 75 str(SALT_BASE_PATH / "spm" / "pkgfiles"), 76 str(SALT_BASE_PATH / "states"), 77 str(SALT_BASE_PATH / "thorium"), 78 str(SALT_BASE_PATH / "tokens"), 79 str(SALT_BASE_PATH / "tops"), 80 str(SALT_BASE_PATH / "utils"), 81 str(SALT_BASE_PATH / "wheel"), 82) 83 84 85def static_loader( 86 opts, 87 ext_type, 88 tag, 89 pack=None, 90 int_type=None, 91 ext_dirs=True, 92 ext_type_dirs=None, 93 base_path=None, 94 filter_name=None, 95): 96 funcs = LazyLoader( 97 _module_dirs( 98 opts, 99 ext_type, 100 tag, 101 int_type, 102 ext_dirs, 103 ext_type_dirs, 104 base_path, 105 ), 106 opts, 107 tag=tag, 108 pack=pack, 109 ) 110 ret = {} 111 funcs._load_all() 112 if filter_name: 113 funcs = FilterDictWrapper(funcs, filter_name) 114 for key in funcs: 115 ret[key] = funcs[key] 116 return ret 117 118 119def _module_dirs( 120 opts, 121 ext_type, 122 tag=None, 123 int_type=None, 124 ext_dirs=True, 125 ext_type_dirs=None, 126 base_path=None, 127 load_extensions=True, 128): 129 if tag is None: 130 tag = ext_type 131 sys_types = os.path.join(base_path or str(SALT_BASE_PATH), int_type or ext_type) 132 ext_types = os.path.join(opts["extension_modules"], ext_type) 133 134 if not sys_types.startswith(SALT_INTERNAL_LOADERS_PATHS): 135 raise RuntimeError( 136 "{!r} is not considered a salt internal loader path. If this " 137 "is a new loader being added, please also add it to " 138 "{}.SALT_INTERNAL_LOADERS_PATHS.".format(sys_types, __name__) 139 ) 140 141 ext_type_types = [] 142 if ext_dirs: 143 if ext_type_dirs is None: 144 ext_type_dirs = "{}_dirs".format(tag) 145 if ext_type_dirs in opts: 146 ext_type_types.extend(opts[ext_type_dirs]) 147 if ext_type_dirs and load_extensions is True: 148 for entry_point in entrypoints.iter_entry_points("salt.loader"): 149 with catch_entry_points_exception(entry_point) as ctx: 150 loaded_entry_point = entry_point.load() 151 if ctx.exception_caught: 152 continue 153 154 # Old way of defining loader entry points 155 # [options.entry_points] 156 # salt.loader= 157 # runner_dirs = thirpartypackage.loader:func_to_get_list_of_dirs 158 # module_dirs = thirpartypackage.loader:func_to_get_list_of_dirs 159 # 160 # 161 # New way of defining entrypoints 162 # [options.entry_points] 163 # salt.loader= 164 # <this-name-does-not-matter> = thirpartypackage 165 # <this-name-does-not-matter> = thirpartypackage:callable 166 # 167 # We try and see if the thirpartypackage has a `ext_type` sub module, and if so, 168 # we append it to loaded_entry_point_paths. 169 # If the entry-point is in the form of `thirpartypackage:callable`, the return of that 170 # callable must be a dictionary where the keys are the `ext_type`'s and the values must be 171 # lists of paths. 172 173 # We could feed the paths we load directly to `ext_type_types`, but we would not 174 # check for duplicates 175 loaded_entry_point_paths = set() 176 177 if isinstance(loaded_entry_point, types.FunctionType): 178 # If the entry point object is a function, we have two scenarios 179 # 1: It returns a list; This is an old style entry entry_point 180 # 2: It returns a dictionary; This is a new style entry point 181 with catch_entry_points_exception(entry_point) as ctx: 182 loaded_entry_point_value = loaded_entry_point() 183 if ctx.exception_caught: 184 continue 185 186 if isinstance(loaded_entry_point_value, dict): 187 # This is new style entry-point and it returns a dictionary. 188 # It MUST contain `ext_type` in it's keys to be considered 189 if ext_type not in loaded_entry_point_value: 190 continue 191 with catch_entry_points_exception(entry_point) as ctx: 192 if isinstance(loaded_entry_point_value[ext_type], str): 193 # No strings please! 194 raise ValueError( 195 "The callable must return an iterable of strings. " 196 "A single string is not supported." 197 ) 198 for path in loaded_entry_point_value[ext_type]: 199 loaded_entry_point_paths.add(path) 200 else: 201 # This is old style entry-point, and, as such, the entry point name MUST 202 # match the value of `ext_type_dirs 203 if entry_point.name != ext_type_dirs: 204 continue 205 for path in loaded_entry_point_value: 206 loaded_entry_point_paths.add(path) 207 elif isinstance(loaded_entry_point, types.ModuleType): 208 # This is a new style entry points definition which just points us to a package 209 # 210 # We try and see if the thirpartypackage has a `ext_type` sub module, and if so, 211 # we append it to loaded_entry_point_paths. 212 for loaded_entry_point_path in loaded_entry_point.__path__: 213 with catch_entry_points_exception(entry_point) as ctx: 214 entry_point_ext_type_package_path = os.path.join( 215 loaded_entry_point_path, ext_type 216 ) 217 if not os.path.exists(entry_point_ext_type_package_path): 218 continue 219 if ctx.exception_caught: 220 continue 221 loaded_entry_point_paths.add(entry_point_ext_type_package_path) 222 else: 223 with catch_entry_points_exception(entry_point): 224 raise ValueError( 225 "Don't know how to load a salt extension from {}".format( 226 loaded_entry_point 227 ) 228 ) 229 230 # Finally, we check all paths that we collected to see if they exist 231 for path in loaded_entry_point_paths: 232 if os.path.exists(path): 233 ext_type_types.append(path) 234 235 cli_module_dirs = [] 236 # The dirs can be any module dir, or a in-tree _{ext_type} dir 237 for _dir in opts.get("module_dirs", []): 238 # Prepend to the list to match cli argument ordering 239 maybe_dir = os.path.join(_dir, ext_type) 240 if os.path.isdir(maybe_dir): 241 cli_module_dirs.insert(0, maybe_dir) 242 continue 243 244 maybe_dir = os.path.join(_dir, "_{}".format(ext_type)) 245 if os.path.isdir(maybe_dir): 246 cli_module_dirs.insert(0, maybe_dir) 247 248 return cli_module_dirs + ext_type_types + [ext_types, sys_types] 249 250 251def minion_mods( 252 opts, 253 context=None, 254 utils=None, 255 whitelist=None, 256 initial_load=False, 257 loaded_base_name=None, 258 notify=False, 259 static_modules=None, 260 proxy=None, 261): 262 """ 263 Load execution modules 264 265 Returns a dictionary of execution modules appropriate for the current 266 system by evaluating the __virtual__() function in each module. 267 268 :param dict opts: The Salt options dictionary 269 270 :param dict context: A Salt context that should be made present inside 271 generated modules in __context__ 272 273 :param dict utils: Utility functions which should be made available to 274 Salt modules in __utils__. See `utils_dirs` in 275 salt.config for additional information about 276 configuration. 277 278 :param list whitelist: A list of modules which should be whitelisted. 279 :param bool initial_load: Deprecated flag! Unused. 280 :param str loaded_base_name: A string marker for the loaded base name. 281 :param bool notify: Flag indicating that an event should be fired upon 282 completion of module loading. 283 284 .. code-block:: python 285 286 import salt.config 287 import salt.loader 288 289 __opts__ = salt.config.minion_config('/etc/salt/minion') 290 __grains__ = salt.loader.grains(__opts__) 291 __opts__['grains'] = __grains__ 292 __utils__ = salt.loader.utils(__opts__) 293 __salt__ = salt.loader.minion_mods(__opts__, utils=__utils__) 294 __salt__['test.ping']() 295 """ 296 # TODO Publish documentation for module whitelisting 297 if not whitelist: 298 whitelist = opts.get("whitelist_modules", None) 299 ret = LazyLoader( 300 _module_dirs(opts, "modules", "module"), 301 opts, 302 tag="module", 303 pack={"__context__": context, "__utils__": utils, "__proxy__": proxy}, 304 whitelist=whitelist, 305 loaded_base_name=loaded_base_name, 306 static_modules=static_modules, 307 extra_module_dirs=utils.module_dirs if utils else None, 308 pack_self="__salt__", 309 ) 310 311 # Load any provider overrides from the configuration file providers option 312 # Note: Providers can be pkg, service, user or group - not to be confused 313 # with cloud providers. 314 providers = opts.get("providers", False) 315 if providers and isinstance(providers, dict): 316 for mod in providers: 317 # sometimes providers opts is not to diverge modules but 318 # for other configuration 319 try: 320 funcs = raw_mod(opts, providers[mod], ret) 321 except TypeError: 322 break 323 else: 324 if funcs: 325 for func in funcs: 326 f_key = "{}{}".format(mod, func[func.rindex(".") :]) 327 ret[f_key] = funcs[func] 328 329 if notify: 330 with salt.utils.event.get_event("minion", opts=opts, listen=False) as evt: 331 evt.fire_event( 332 {"complete": True}, tag=salt.defaults.events.MINION_MOD_REFRESH_COMPLETE 333 ) 334 335 return ret 336 337 338def raw_mod(opts, name, functions, mod="modules"): 339 """ 340 Returns a single module loaded raw and bypassing the __virtual__ function 341 342 .. code-block:: python 343 344 import salt.config 345 import salt.loader 346 347 __opts__ = salt.config.minion_config('/etc/salt/minion') 348 testmod = salt.loader.raw_mod(__opts__, 'test', None) 349 testmod['test.ping']() 350 """ 351 loader = LazyLoader( 352 _module_dirs(opts, mod, "module"), 353 opts, 354 tag="rawmodule", 355 virtual_enable=False, 356 pack={"__salt__": functions}, 357 ) 358 # if we don't have the module, return an empty dict 359 if name not in loader.file_mapping: 360 return {} 361 362 # load a single module (the one passed in) 363 loader._load_module(name) 364 # return a copy of *just* the funcs for `name` 365 return dict({x: loader[x] for x in loader._dict}) 366 367 368def metaproxy(opts, loaded_base_name=None): 369 """ 370 Return functions used in the meta proxy 371 """ 372 return LazyLoader( 373 _module_dirs(opts, "metaproxy"), 374 opts, 375 tag="metaproxy", 376 loaded_base_name=loaded_base_name, 377 ) 378 379 380def matchers(opts): 381 """ 382 Return the matcher services plugins 383 """ 384 return LazyLoader(_module_dirs(opts, "matchers"), opts, tag="matchers") 385 386 387def engines(opts, functions, runners, utils, proxy=None): 388 """ 389 Return the master services plugins 390 """ 391 pack = { 392 "__salt__": functions, 393 "__runners__": runners, 394 "__proxy__": proxy, 395 "__utils__": utils, 396 } 397 return LazyLoader( 398 _module_dirs(opts, "engines"), 399 opts, 400 tag="engines", 401 pack=pack, 402 extra_module_dirs=utils.module_dirs if utils else None, 403 ) 404 405 406def proxy( 407 opts, 408 functions=None, 409 returners=None, 410 whitelist=None, 411 utils=None, 412 context=None, 413 pack_self="__proxy__", 414): 415 """ 416 Returns the proxy module for this salt-proxy-minion 417 """ 418 return LazyLoader( 419 _module_dirs(opts, "proxy"), 420 opts, 421 tag="proxy", 422 pack={ 423 "__salt__": functions, 424 "__ret__": returners, 425 "__utils__": utils, 426 "__context__": context, 427 }, 428 extra_module_dirs=utils.module_dirs if utils else None, 429 pack_self=pack_self, 430 ) 431 432 433def returners(opts, functions, whitelist=None, context=None, proxy=None): 434 """ 435 Returns the returner modules 436 """ 437 return LazyLoader( 438 _module_dirs(opts, "returners", "returner"), 439 opts, 440 tag="returner", 441 whitelist=whitelist, 442 pack={"__salt__": functions, "__context__": context, "__proxy__": proxy or {}}, 443 ) 444 445 446def utils(opts, whitelist=None, context=None, proxy=None, pack_self=None): 447 """ 448 Returns the utility modules 449 """ 450 return LazyLoader( 451 _module_dirs(opts, "utils", ext_type_dirs="utils_dirs", load_extensions=False), 452 opts, 453 tag="utils", 454 whitelist=whitelist, 455 pack={"__context__": context, "__proxy__": proxy or {}}, 456 pack_self=pack_self, 457 ) 458 459 460def pillars(opts, functions, context=None): 461 """ 462 Returns the pillars modules 463 """ 464 _utils = utils(opts) 465 ret = LazyLoader( 466 _module_dirs(opts, "pillar"), 467 opts, 468 tag="pillar", 469 pack={"__salt__": functions, "__context__": context, "__utils__": _utils}, 470 extra_module_dirs=_utils.module_dirs, 471 pack_self="__ext_pillar__", 472 ) 473 return FilterDictWrapper(ret, ".ext_pillar") 474 475 476def tops(opts): 477 """ 478 Returns the tops modules 479 """ 480 if "master_tops" not in opts: 481 return {} 482 whitelist = list(opts["master_tops"].keys()) 483 ret = LazyLoader( 484 _module_dirs(opts, "tops", "top"), 485 opts, 486 tag="top", 487 whitelist=whitelist, 488 ) 489 return FilterDictWrapper(ret, ".top") 490 491 492def wheels(opts, whitelist=None, context=None): 493 """ 494 Returns the wheels modules 495 """ 496 if context is None: 497 context = {} 498 return LazyLoader( 499 _module_dirs(opts, "wheel"), 500 opts, 501 tag="wheel", 502 whitelist=whitelist, 503 pack={"__context__": context}, 504 ) 505 506 507def outputters(opts): 508 """ 509 Returns the outputters modules 510 511 :param dict opts: The Salt options dictionary 512 :returns: LazyLoader instance, with only outputters present in the keyspace 513 """ 514 ret = LazyLoader( 515 _module_dirs(opts, "output", ext_type_dirs="outputter_dirs"), 516 opts, 517 tag="output", 518 ) 519 wrapped_ret = FilterDictWrapper(ret, ".output") 520 # TODO: this name seems terrible... __salt__ should always be execution mods 521 ret.pack["__salt__"] = wrapped_ret 522 return wrapped_ret 523 524 525def serializers(opts): 526 """ 527 Returns the serializers modules 528 :param dict opts: The Salt options dictionary 529 :returns: LazyLoader instance, with only serializers present in the keyspace 530 """ 531 return LazyLoader( 532 _module_dirs(opts, "serializers"), 533 opts, 534 tag="serializers", 535 ) 536 537 538def eauth_tokens(opts): 539 """ 540 Returns the tokens modules 541 :param dict opts: The Salt options dictionary 542 :returns: LazyLoader instance, with only token backends present in the keyspace 543 """ 544 return LazyLoader( 545 _module_dirs(opts, "tokens"), 546 opts, 547 tag="tokens", 548 ) 549 550 551def auth(opts, whitelist=None): 552 """ 553 Returns the auth modules 554 555 :param dict opts: The Salt options dictionary 556 :returns: LazyLoader 557 """ 558 return LazyLoader( 559 _module_dirs(opts, "auth"), 560 opts, 561 tag="auth", 562 whitelist=whitelist, 563 pack={"__salt__": minion_mods(opts)}, 564 ) 565 566 567def fileserver(opts, backends): 568 """ 569 Returns the file server modules 570 """ 571 _utils = utils(opts) 572 573 if backends is not None: 574 if not isinstance(backends, list): 575 backends = [backends] 576 577 # If backend is a VCS, add both the '-fs' and non '-fs' versions to the list. 578 # Use a set to keep them unique 579 backend_set = set() 580 vcs_re = re.compile("^(git|svn|hg)(?:fs)?$") 581 for backend in backends: 582 match = vcs_re.match(backend) 583 if match: 584 backend_set.add(match.group(1)) 585 backend_set.add(match.group(1) + "fs") 586 else: 587 backend_set.add(backend) 588 backends = list(backend_set) 589 590 return LazyLoader( 591 _module_dirs(opts, "fileserver"), 592 opts, 593 tag="fileserver", 594 whitelist=backends, 595 pack={"__utils__": _utils}, 596 extra_module_dirs=_utils.module_dirs, 597 ) 598 599 600def roster(opts, runner=None, utils=None, whitelist=None): 601 """ 602 Returns the roster modules 603 """ 604 return LazyLoader( 605 _module_dirs(opts, "roster"), 606 opts, 607 tag="roster", 608 whitelist=whitelist, 609 pack={"__runner__": runner, "__utils__": utils}, 610 extra_module_dirs=utils.module_dirs if utils else None, 611 ) 612 613 614def thorium(opts, functions, runners): 615 """ 616 Load the thorium runtime modules 617 """ 618 pack = {"__salt__": functions, "__runner__": runners, "__context__": {}} 619 ret = LazyLoader(_module_dirs(opts, "thorium"), opts, tag="thorium", pack=pack) 620 ret.pack["__thorium__"] = ret 621 return ret 622 623 624def states( 625 opts, functions, utils, serializers, whitelist=None, proxy=None, context=None 626): 627 """ 628 Returns the state modules 629 630 :param dict opts: The Salt options dictionary 631 :param dict functions: A dictionary of minion modules, with module names as 632 keys and funcs as values. 633 634 .. code-block:: python 635 636 import salt.config 637 import salt.loader 638 639 __opts__ = salt.config.minion_config('/etc/salt/minion') 640 statemods = salt.loader.states(__opts__, None, None) 641 """ 642 if context is None: 643 context = {} 644 645 return LazyLoader( 646 _module_dirs(opts, "states"), 647 opts, 648 tag="states", 649 pack={ 650 "__salt__": functions, 651 "__proxy__": proxy or {}, 652 "__utils__": utils, 653 "__serializers__": serializers, 654 "__context__": context, 655 }, 656 whitelist=whitelist, 657 extra_module_dirs=utils.module_dirs if utils else None, 658 pack_self="__states__", 659 ) 660 661 662def beacons(opts, functions, context=None, proxy=None): 663 """ 664 Load the beacon modules 665 666 :param dict opts: The Salt options dictionary 667 :param dict functions: A dictionary of minion modules, with module names as 668 keys and funcs as values. 669 """ 670 return LazyLoader( 671 _module_dirs(opts, "beacons"), 672 opts, 673 tag="beacons", 674 pack={"__context__": context, "__salt__": functions, "__proxy__": proxy or {}}, 675 virtual_funcs=[], 676 ) 677 678 679def log_handlers(opts): 680 """ 681 Returns the custom logging handler modules 682 683 :param dict opts: The Salt options dictionary 684 """ 685 ret = LazyLoader( 686 _module_dirs( 687 opts, 688 "log_handlers", 689 int_type="handlers", 690 base_path=str(SALT_BASE_PATH / "log"), 691 ), 692 opts, 693 tag="log_handlers", 694 ) 695 return FilterDictWrapper(ret, ".setup_handlers") 696 697 698def ssh_wrapper(opts, functions=None, context=None): 699 """ 700 Returns the custom logging handler modules 701 """ 702 return LazyLoader( 703 _module_dirs( 704 opts, 705 "wrapper", 706 base_path=str(SALT_BASE_PATH / "client" / "ssh"), 707 ), 708 opts, 709 tag="wrapper", 710 pack={"__salt__": functions, "__context__": context}, 711 ) 712 713 714def render(opts, functions, states=None, proxy=None, context=None): 715 """ 716 Returns the render modules 717 """ 718 if context is None: 719 context = {} 720 721 pack = { 722 "__salt__": functions, 723 "__grains__": opts.get("grains", {}), 724 "__context__": context, 725 } 726 727 if states: 728 pack["__states__"] = states 729 730 if proxy is None: 731 proxy = {} 732 pack["__proxy__"] = proxy 733 734 ret = LazyLoader( 735 _module_dirs( 736 opts, 737 "renderers", 738 "render", 739 ext_type_dirs="render_dirs", 740 ), 741 opts, 742 tag="render", 743 pack=pack, 744 ) 745 rend = FilterDictWrapper(ret, ".render") 746 747 if not check_render_pipe_str( 748 opts["renderer"], rend, opts["renderer_blacklist"], opts["renderer_whitelist"] 749 ): 750 err = ( 751 "The renderer {} is unavailable, this error is often because " 752 "the needed software is unavailable".format(opts["renderer"]) 753 ) 754 log.critical(err) 755 raise LoaderError(err) 756 return rend 757 758 759def grain_funcs(opts, proxy=None, context=None): 760 """ 761 Returns the grain functions 762 763 .. code-block:: python 764 765 import salt.config 766 import salt.loader 767 768 __opts__ = salt.config.minion_config('/etc/salt/minion') 769 grainfuncs = salt.loader.grain_funcs(__opts__) 770 """ 771 _utils = utils(opts, proxy=proxy) 772 pack = {"__utils__": utils(opts, proxy=proxy), "__context__": context} 773 ret = LazyLoader( 774 _module_dirs( 775 opts, 776 "grains", 777 "grain", 778 ext_type_dirs="grains_dirs", 779 ), 780 opts, 781 tag="grains", 782 extra_module_dirs=_utils.module_dirs, 783 pack=pack, 784 ) 785 ret.pack["__utils__"] = _utils 786 return ret 787 788 789def _format_cached_grains(cached_grains): 790 """ 791 Returns cached grains with fixed types, like tuples. 792 """ 793 if cached_grains.get("osrelease_info"): 794 osrelease_info = cached_grains["osrelease_info"] 795 if isinstance(osrelease_info, list): 796 cached_grains["osrelease_info"] = tuple(osrelease_info) 797 return cached_grains 798 799 800def _load_cached_grains(opts, cfn): 801 """ 802 Returns the grains cached in cfn, or None if the cache is too old or is 803 corrupted. 804 """ 805 if not os.path.isfile(cfn): 806 log.debug("Grains cache file does not exist.") 807 return None 808 809 grains_cache_age = int(time.time() - os.path.getmtime(cfn)) 810 if grains_cache_age > opts.get("grains_cache_expiration", 300): 811 log.debug( 812 "Grains cache last modified %s seconds ago and cache " 813 "expiration is set to %s. Grains cache expired. " 814 "Refreshing.", 815 grains_cache_age, 816 opts.get("grains_cache_expiration", 300), 817 ) 818 return None 819 820 if opts.get("refresh_grains_cache", False): 821 log.debug("refresh_grains_cache requested, Refreshing.") 822 return None 823 824 log.debug("Retrieving grains from cache") 825 try: 826 with salt.utils.files.fopen(cfn, "rb") as fp_: 827 cached_grains = salt.utils.data.decode( 828 salt.payload.load(fp_), preserve_tuples=True 829 ) 830 if not cached_grains: 831 log.debug("Cached grains are empty, cache might be corrupted. Refreshing.") 832 return None 833 834 return _format_cached_grains(cached_grains) 835 except OSError: 836 return None 837 838 839def grains(opts, force_refresh=False, proxy=None, context=None): 840 """ 841 Return the functions for the dynamic grains and the values for the static 842 grains. 843 844 Since grains are computed early in the startup process, grains functions 845 do not have __salt__ or __proxy__ available. At proxy-minion startup, 846 this function is called with the proxymodule LazyLoader object so grains 847 functions can communicate with their controlled device. 848 849 .. code-block:: python 850 851 import salt.config 852 import salt.loader 853 854 __opts__ = salt.config.minion_config('/etc/salt/minion') 855 __grains__ = salt.loader.grains(__opts__) 856 print __grains__['id'] 857 """ 858 # Need to re-import salt.config, somehow it got lost when a minion is starting 859 import salt.config 860 861 # if we have no grains, lets try loading from disk (TODO: move to decorator?) 862 cfn = os.path.join(opts["cachedir"], "grains.cache.p") 863 if not force_refresh and opts.get("grains_cache", False): 864 cached_grains = _load_cached_grains(opts, cfn) 865 if cached_grains: 866 return cached_grains 867 else: 868 log.debug("Grains refresh requested. Refreshing grains.") 869 870 if opts.get("skip_grains", False): 871 return {} 872 grains_deep_merge = opts.get("grains_deep_merge", False) is True 873 if "conf_file" in opts: 874 pre_opts = {} 875 pre_opts.update( 876 salt.config.load_config( 877 opts["conf_file"], 878 "SALT_MINION_CONFIG", 879 salt.config.DEFAULT_MINION_OPTS["conf_file"], 880 ) 881 ) 882 default_include = pre_opts.get("default_include", opts["default_include"]) 883 include = pre_opts.get("include", []) 884 pre_opts.update( 885 salt.config.include_config( 886 default_include, opts["conf_file"], verbose=False 887 ) 888 ) 889 pre_opts.update( 890 salt.config.include_config(include, opts["conf_file"], verbose=True) 891 ) 892 if "grains" in pre_opts: 893 opts["grains"] = pre_opts["grains"] 894 else: 895 opts["grains"] = {} 896 else: 897 opts["grains"] = {} 898 899 grains_data = {} 900 blist = opts.get("grains_blacklist", []) 901 funcs = grain_funcs(opts, proxy=proxy, context=context or {}) 902 if force_refresh: # if we refresh, lets reload grain modules 903 funcs.clear() 904 # Run core grains 905 for key in funcs: 906 if not key.startswith("core."): 907 continue 908 log.trace("Loading %s grain", key) 909 ret = funcs[key]() 910 if not isinstance(ret, dict): 911 continue 912 if blist: 913 for key in list(ret): 914 for block in blist: 915 if salt.utils.stringutils.expr_match(key, block): 916 del ret[key] 917 log.trace("Filtering %s grain", key) 918 if not ret: 919 continue 920 if grains_deep_merge: 921 salt.utils.dictupdate.update(grains_data, ret) 922 else: 923 grains_data.update(ret) 924 925 # Run the rest of the grains 926 for key in funcs: 927 if key.startswith("core.") or key == "_errors": 928 continue 929 try: 930 # Grains are loaded too early to take advantage of the injected 931 # __proxy__ variable. Pass an instance of that LazyLoader 932 # here instead to grains functions if the grains functions take 933 # one parameter. Then the grains can have access to the 934 # proxymodule for retrieving information from the connected 935 # device. 936 log.trace("Loading %s grain", key) 937 parameters = salt.utils.args.get_function_argspec(funcs[key]).args 938 kwargs = {} 939 if "proxy" in parameters: 940 kwargs["proxy"] = proxy 941 if "grains" in parameters: 942 kwargs["grains"] = grains_data 943 ret = funcs[key](**kwargs) 944 except Exception: # pylint: disable=broad-except 945 if salt.utils.platform.is_proxy(): 946 log.info( 947 "The following CRITICAL message may not be an error; the proxy may not be completely established yet." 948 ) 949 log.critical( 950 "Failed to load grains defined in grain file %s in " 951 "function %s, error:\n", 952 key, 953 funcs[key], 954 exc_info=True, 955 ) 956 continue 957 if not isinstance(ret, dict): 958 continue 959 if blist: 960 for key in list(ret): 961 for block in blist: 962 if salt.utils.stringutils.expr_match(key, block): 963 del ret[key] 964 log.trace("Filtering %s grain", key) 965 if not ret: 966 continue 967 if grains_deep_merge: 968 salt.utils.dictupdate.update(grains_data, ret) 969 else: 970 grains_data.update(ret) 971 972 if opts.get("proxy_merge_grains_in_module", True) and proxy: 973 try: 974 proxytype = proxy.opts["proxy"]["proxytype"] 975 if proxytype + ".grains" in proxy: 976 if ( 977 proxytype + ".initialized" in proxy 978 and proxy[proxytype + ".initialized"]() 979 ): 980 try: 981 proxytype = proxy.opts["proxy"]["proxytype"] 982 ret = proxy[proxytype + ".grains"]() 983 if grains_deep_merge: 984 salt.utils.dictupdate.update(grains_data, ret) 985 else: 986 grains_data.update(ret) 987 except Exception: # pylint: disable=broad-except 988 log.critical( 989 "Failed to run proxy's grains function!", exc_info=True 990 ) 991 except KeyError: 992 pass 993 994 grains_data.update(opts["grains"]) 995 # Write cache if enabled 996 if opts.get("grains_cache", False): 997 with salt.utils.files.set_umask(0o077): 998 try: 999 if salt.utils.platform.is_windows(): 1000 # Late import 1001 import salt.modules.cmdmod 1002 1003 # Make sure cache file isn't read-only 1004 salt.modules.cmdmod._run_quiet('attrib -R "{}"'.format(cfn)) 1005 with salt.utils.files.fopen(cfn, "w+b") as fp_: 1006 try: 1007 salt.payload.dump(grains_data, fp_) 1008 except TypeError as e: 1009 log.error("Failed to serialize grains cache: %s", e) 1010 raise # re-throw for cleanup 1011 except Exception as e: # pylint: disable=broad-except 1012 log.error("Unable to write to grains cache file %s: %s", cfn, e) 1013 # Based on the original exception, the file may or may not have been 1014 # created. If it was, we will remove it now, as the exception means 1015 # the serialized data is not to be trusted, no matter what the 1016 # exception is. 1017 if os.path.isfile(cfn): 1018 os.unlink(cfn) 1019 1020 if grains_deep_merge: 1021 salt.utils.dictupdate.update(grains_data, opts["grains"]) 1022 else: 1023 grains_data.update(opts["grains"]) 1024 return salt.utils.data.decode(grains_data, preserve_tuples=True) 1025 1026 1027# TODO: get rid of? Does anyone use this? You should use raw() instead 1028def call(fun, **kwargs): 1029 """ 1030 Directly call a function inside a loader directory 1031 """ 1032 args = kwargs.get("args", []) 1033 dirs = kwargs.get("dirs", []) 1034 1035 funcs = LazyLoader( 1036 [str(SALT_BASE_PATH / "modules")] + dirs, 1037 None, 1038 tag="modules", 1039 virtual_enable=False, 1040 ) 1041 return funcs[fun](*args) 1042 1043 1044def runner(opts, utils=None, context=None, whitelist=None): 1045 """ 1046 Directly call a function inside a loader directory 1047 """ 1048 if utils is None: 1049 utils = {} 1050 if context is None: 1051 context = {} 1052 return LazyLoader( 1053 _module_dirs(opts, "runners", "runner", ext_type_dirs="runner_dirs"), 1054 opts, 1055 tag="runners", 1056 pack={"__utils__": utils, "__context__": context}, 1057 whitelist=whitelist, 1058 extra_module_dirs=utils.module_dirs if utils else None, 1059 # TODO: change from __salt__ to something else, we overload __salt__ too much 1060 pack_self="__salt__", 1061 ) 1062 1063 1064def queues(opts): 1065 """ 1066 Directly call a function inside a loader directory 1067 """ 1068 return LazyLoader( 1069 _module_dirs(opts, "queues", "queue", ext_type_dirs="queue_dirs"), 1070 opts, 1071 tag="queues", 1072 ) 1073 1074 1075def sdb(opts, functions=None, whitelist=None, utils=None): 1076 """ 1077 Make a very small database call 1078 """ 1079 if utils is None: 1080 utils = {} 1081 1082 return LazyLoader( 1083 _module_dirs(opts, "sdb"), 1084 opts, 1085 tag="sdb", 1086 pack={ 1087 "__sdb__": functions, 1088 "__utils__": utils, 1089 "__salt__": minion_mods(opts, utils=utils), 1090 }, 1091 whitelist=whitelist, 1092 extra_module_dirs=utils.module_dirs if utils else None, 1093 ) 1094 1095 1096def pkgdb(opts): 1097 """ 1098 Return modules for SPM's package database 1099 1100 .. versionadded:: 2015.8.0 1101 """ 1102 return LazyLoader( 1103 _module_dirs(opts, "pkgdb", base_path=str(SALT_BASE_PATH / "spm")), 1104 opts, 1105 tag="pkgdb", 1106 ) 1107 1108 1109def pkgfiles(opts): 1110 """ 1111 Return modules for SPM's file handling 1112 1113 .. versionadded:: 2015.8.0 1114 """ 1115 return LazyLoader( 1116 _module_dirs(opts, "pkgfiles", base_path=str(SALT_BASE_PATH / "spm")), 1117 opts, 1118 tag="pkgfiles", 1119 ) 1120 1121 1122def clouds(opts): 1123 """ 1124 Return the cloud functions 1125 """ 1126 _utils = utils(opts) 1127 # Let's bring __active_provider_name__, defaulting to None, to all cloud 1128 # drivers. This will get temporarily updated/overridden with a context 1129 # manager when needed. 1130 functions = LazyLoader( 1131 _module_dirs( 1132 opts, 1133 "clouds", 1134 "cloud", 1135 base_path=str(SALT_BASE_PATH / "cloud"), 1136 int_type="clouds", 1137 ), 1138 opts, 1139 tag="clouds", 1140 pack={"__utils__": _utils, "__active_provider_name__": None}, 1141 extra_module_dirs=_utils.module_dirs, 1142 ) 1143 for funcname in LIBCLOUD_FUNCS_NOT_SUPPORTED: 1144 log.trace( 1145 "'%s' has been marked as not supported. Removing from the " 1146 "list of supported cloud functions", 1147 funcname, 1148 ) 1149 functions.pop(funcname, None) 1150 return functions 1151 1152 1153def netapi(opts): 1154 """ 1155 Return the network api functions 1156 """ 1157 return LazyLoader( 1158 _module_dirs(opts, "netapi"), 1159 opts, 1160 tag="netapi", 1161 ) 1162 1163 1164def executors(opts, functions=None, context=None, proxy=None): 1165 """ 1166 Returns the executor modules 1167 """ 1168 if proxy is None: 1169 proxy = {} 1170 if context is None: 1171 context = {} 1172 return LazyLoader( 1173 _module_dirs(opts, "executors", "executor"), 1174 opts, 1175 tag="executor", 1176 pack={"__salt__": functions, "__context__": context, "__proxy__": proxy}, 1177 pack_self="__executors__", 1178 ) 1179 1180 1181def cache(opts): 1182 """ 1183 Returns the returner modules 1184 """ 1185 return LazyLoader( 1186 _module_dirs(opts, "cache", "cache"), 1187 opts, 1188 tag="cache", 1189 ) 1190 1191 1192@contextlib.contextmanager 1193def catch_entry_points_exception(entry_point): 1194 context = types.SimpleNamespace(exception_caught=False) 1195 try: 1196 yield context 1197 except Exception as exc: # pylint: disable=broad-except 1198 context.exception_caught = True 1199 entry_point_details = entrypoints.name_and_version_from_entry_point(entry_point) 1200 log.error( 1201 "Error processing Salt Extension %s(version: %s): %s", 1202 entry_point_details.name, 1203 entry_point_details.version, 1204 exc, 1205 exc_info_on_loglevel=logging.DEBUG, 1206 ) 1207