1""" 2tests.support.sminion 3~~~~~~~~~~~~~~~~~~~~~ 4 5SMinion's support functions 6""" 7 8import fnmatch 9import hashlib 10import logging 11import os 12import shutil 13import sys 14 15import salt.minion 16import salt.utils.path 17import salt.utils.stringutils 18from tests.support.runtests import RUNTIME_VARS 19 20log = logging.getLogger(__name__) 21 22DEFAULT_SMINION_ID = "pytest-internal-sminion" 23 24 25def build_minion_opts( 26 minion_id=None, 27 root_dir=None, 28 initial_conf_file=None, 29 minion_opts_overrides=None, 30 skip_cached_opts=False, 31 cache_opts=True, 32 minion_role=None, 33): 34 if minion_id is None: 35 minion_id = DEFAULT_SMINION_ID 36 if skip_cached_opts is False: 37 try: 38 opts_cache = build_minion_opts.__cached_opts__ 39 except AttributeError: 40 opts_cache = build_minion_opts.__cached_opts__ = {} 41 cached_opts = opts_cache.get(minion_id) 42 if cached_opts: 43 return cached_opts 44 45 log.info("Generating testing minion %r configuration...", minion_id) 46 if root_dir is None: 47 hashed_minion_id = hashlib.sha1() 48 hashed_minion_id.update(salt.utils.stringutils.to_bytes(minion_id)) 49 root_dir = os.path.join( 50 RUNTIME_VARS.TMP_ROOT_DIR, hashed_minion_id.hexdigest()[:6] 51 ) 52 53 if initial_conf_file is not None: 54 minion_opts = salt.config._read_conf_file( 55 initial_conf_file 56 ) # pylint: disable=protected-access 57 else: 58 minion_opts = {} 59 60 conf_dir = os.path.join(root_dir, "conf") 61 conf_file = os.path.join(conf_dir, "minion") 62 63 minion_opts["id"] = minion_id 64 minion_opts["conf_file"] = conf_file 65 minion_opts["root_dir"] = root_dir 66 minion_opts["cachedir"] = "cache" 67 minion_opts["user"] = RUNTIME_VARS.RUNNING_TESTS_USER 68 minion_opts["pki_dir"] = "pki" 69 minion_opts["hosts.file"] = os.path.join(RUNTIME_VARS.TMP_ROOT_DIR, "hosts") 70 minion_opts["aliases.file"] = os.path.join(RUNTIME_VARS.TMP_ROOT_DIR, "aliases") 71 minion_opts["file_client"] = "local" 72 minion_opts["server_id_use_crc"] = "adler32" 73 minion_opts["pillar_roots"] = {"base": [RUNTIME_VARS.TMP_PILLAR_TREE]} 74 minion_opts["file_roots"] = { 75 "base": [ 76 # Let's support runtime created files that can be used like: 77 # salt://my-temp-file.txt 78 RUNTIME_VARS.TMP_STATE_TREE 79 ], 80 # Alternate root to test __env__ choices 81 "prod": [ 82 os.path.join(RUNTIME_VARS.FILES, "file", "prod"), 83 RUNTIME_VARS.TMP_PRODENV_STATE_TREE, 84 ], 85 } 86 if initial_conf_file and initial_conf_file.startswith(RUNTIME_VARS.FILES): 87 # We assume we were passed a minion configuration file defined fo testing and, as such 88 # we define the file and pillar roots to include the testing states/pillar trees 89 minion_opts["pillar_roots"]["base"].append( 90 os.path.join(RUNTIME_VARS.FILES, "pillar", "base"), 91 ) 92 minion_opts["file_roots"]["base"].append( 93 os.path.join(RUNTIME_VARS.FILES, "file", "base"), 94 ) 95 minion_opts["file_roots"]["prod"].append( 96 os.path.join(RUNTIME_VARS.FILES, "file", "prod"), 97 ) 98 99 # We need to copy the extension modules into the new master root_dir or 100 # it will be prefixed by it 101 extension_modules_path = os.path.join(root_dir, "extension_modules") 102 if not os.path.exists(extension_modules_path): 103 shutil.copytree( 104 os.path.join(RUNTIME_VARS.FILES, "extension_modules"), 105 extension_modules_path, 106 ) 107 minion_opts["extension_modules"] = extension_modules_path 108 109 # Custom grains 110 if "grains" not in minion_opts: 111 minion_opts["grains"] = {} 112 if minion_role is not None: 113 minion_opts["grains"]["role"] = minion_role 114 115 # Under windows we can't seem to properly create a virtualenv off of another 116 # virtualenv, we can on linux but we will still point to the virtualenv binary 117 # outside the virtualenv running the test suite, if that's the case. 118 try: 119 real_prefix = sys.real_prefix 120 # The above attribute exists, this is a virtualenv 121 if salt.utils.platform.is_windows(): 122 virtualenv_binary = os.path.join(real_prefix, "Scripts", "virtualenv.exe") 123 else: 124 # We need to remove the virtualenv from PATH or we'll get the virtualenv binary 125 # from within the virtualenv, we don't want that 126 path = os.environ.get("PATH") 127 if path is not None: 128 path_items = path.split(os.pathsep) 129 for item in path_items[:]: 130 if item.startswith(sys.base_prefix): 131 path_items.remove(item) 132 os.environ["PATH"] = os.pathsep.join(path_items) 133 virtualenv_binary = salt.utils.path.which("virtualenv") 134 if path is not None: 135 # Restore previous environ PATH 136 os.environ["PATH"] = path 137 if not virtualenv_binary.startswith(real_prefix): 138 virtualenv_binary = None 139 if virtualenv_binary and not os.path.exists(virtualenv_binary): 140 # It doesn't exist?! 141 virtualenv_binary = None 142 except AttributeError: 143 # We're not running inside a virtualenv 144 virtualenv_binary = None 145 if virtualenv_binary: 146 minion_opts["venv_bin"] = virtualenv_binary 147 148 # Override minion_opts with minion_opts_overrides 149 if minion_opts_overrides: 150 minion_opts.update(minion_opts_overrides) 151 152 if not os.path.exists(conf_dir): 153 os.makedirs(conf_dir) 154 155 with salt.utils.files.fopen(conf_file, "w") as fp_: 156 salt.utils.yaml.safe_dump(minion_opts, fp_, default_flow_style=False) 157 158 log.info("Generating testing minion %r configuration completed.", minion_id) 159 minion_opts = salt.config.minion_config( 160 conf_file, minion_id=minion_id, cache_minion_id=True 161 ) 162 salt.utils.verify.verify_env( 163 [ 164 os.path.join(minion_opts["pki_dir"], "accepted"), 165 os.path.join(minion_opts["pki_dir"], "rejected"), 166 os.path.join(minion_opts["pki_dir"], "pending"), 167 os.path.dirname(minion_opts["log_file"]), 168 minion_opts["extension_modules"], 169 minion_opts["cachedir"], 170 minion_opts["sock_dir"], 171 RUNTIME_VARS.TMP_STATE_TREE, 172 RUNTIME_VARS.TMP_PILLAR_TREE, 173 RUNTIME_VARS.TMP_PRODENV_STATE_TREE, 174 RUNTIME_VARS.TMP, 175 ], 176 RUNTIME_VARS.RUNNING_TESTS_USER, 177 root_dir=root_dir, 178 ) 179 if cache_opts: 180 try: 181 opts_cache = build_minion_opts.__cached_opts__ 182 except AttributeError: 183 opts_cache = build_minion_opts.__cached_opts__ = {} 184 opts_cache[minion_id] = minion_opts 185 return minion_opts 186 187 188def create_sminion( 189 minion_id=None, 190 root_dir=None, 191 initial_conf_file=None, 192 sminion_cls=salt.minion.SMinion, 193 minion_opts_overrides=None, 194 skip_cached_minion=False, 195 cache_sminion=True, 196): 197 if minion_id is None: 198 minion_id = DEFAULT_SMINION_ID 199 if skip_cached_minion is False: 200 try: 201 minions_cache = create_sminion.__cached_minions__ 202 except AttributeError: 203 create_sminion.__cached_minions__ = {} 204 cached_minion = create_sminion.__cached_minions__.get(minion_id) 205 if cached_minion: 206 return cached_minion 207 minion_opts = build_minion_opts( 208 minion_id=minion_id, 209 root_dir=root_dir, 210 initial_conf_file=initial_conf_file, 211 minion_opts_overrides=minion_opts_overrides, 212 skip_cached_opts=skip_cached_minion, 213 cache_opts=cache_sminion, 214 ) 215 log.info("Instantiating a testing %s(%s)", sminion_cls.__name__, minion_id) 216 sminion = sminion_cls(minion_opts) 217 if cache_sminion: 218 try: 219 minions_cache = create_sminion.__cached_minions__ 220 except AttributeError: 221 minions_cache = create_sminion.__cached_minions__ = {} 222 minions_cache[minion_id] = sminion 223 return sminion 224 225 226def check_required_sminion_attributes(sminion_attr, required_items): 227 """ 228 :param sminion_attr: The name of the sminion attribute to check, such as 'functions' or 'states' 229 :param required_items: The items that must be part of the designated sminion attribute for the decorated test 230 :return The packages that are not available 231 """ 232 required_salt_items = set(required_items) 233 sminion = create_sminion(minion_id=DEFAULT_SMINION_ID) 234 available_items = list(getattr(sminion, sminion_attr)) 235 not_available_items = set() 236 237 name = "__not_available_{items}s__".format(items=sminion_attr) 238 if not hasattr(sminion, name): 239 setattr(sminion, name, set()) 240 241 cached_not_available_items = getattr(sminion, name) 242 243 for not_available_item in cached_not_available_items: 244 if not_available_item in required_salt_items: 245 not_available_items.add(not_available_item) 246 required_salt_items.remove(not_available_item) 247 248 for required_item_name in required_salt_items: 249 search_name = required_item_name 250 if "." not in search_name: 251 search_name += ".*" 252 if not fnmatch.filter(available_items, search_name): 253 not_available_items.add(required_item_name) 254 cached_not_available_items.add(required_item_name) 255 256 return not_available_items 257