1"""
2tests.support.sminion
3~~~~~~~~~~~~~~~~~~~~~
4
5SMinion's support functions
6"""
7
8import fnmatch
9import hashlib
10import logging
11import os
12import shutil
13import sys
14
15import salt.minion
16import salt.utils.path
17import salt.utils.stringutils
18from tests.support.runtests import RUNTIME_VARS
19
20log = logging.getLogger(__name__)
21
22DEFAULT_SMINION_ID = "pytest-internal-sminion"
23
24
25def build_minion_opts(
26    minion_id=None,
27    root_dir=None,
28    initial_conf_file=None,
29    minion_opts_overrides=None,
30    skip_cached_opts=False,
31    cache_opts=True,
32    minion_role=None,
33):
34    if minion_id is None:
35        minion_id = DEFAULT_SMINION_ID
36    if skip_cached_opts is False:
37        try:
38            opts_cache = build_minion_opts.__cached_opts__
39        except AttributeError:
40            opts_cache = build_minion_opts.__cached_opts__ = {}
41        cached_opts = opts_cache.get(minion_id)
42        if cached_opts:
43            return cached_opts
44
45    log.info("Generating testing minion %r configuration...", minion_id)
46    if root_dir is None:
47        hashed_minion_id = hashlib.sha1()
48        hashed_minion_id.update(salt.utils.stringutils.to_bytes(minion_id))
49        root_dir = os.path.join(
50            RUNTIME_VARS.TMP_ROOT_DIR, hashed_minion_id.hexdigest()[:6]
51        )
52
53    if initial_conf_file is not None:
54        minion_opts = salt.config._read_conf_file(
55            initial_conf_file
56        )  # pylint: disable=protected-access
57    else:
58        minion_opts = {}
59
60    conf_dir = os.path.join(root_dir, "conf")
61    conf_file = os.path.join(conf_dir, "minion")
62
63    minion_opts["id"] = minion_id
64    minion_opts["conf_file"] = conf_file
65    minion_opts["root_dir"] = root_dir
66    minion_opts["cachedir"] = "cache"
67    minion_opts["user"] = RUNTIME_VARS.RUNNING_TESTS_USER
68    minion_opts["pki_dir"] = "pki"
69    minion_opts["hosts.file"] = os.path.join(RUNTIME_VARS.TMP_ROOT_DIR, "hosts")
70    minion_opts["aliases.file"] = os.path.join(RUNTIME_VARS.TMP_ROOT_DIR, "aliases")
71    minion_opts["file_client"] = "local"
72    minion_opts["server_id_use_crc"] = "adler32"
73    minion_opts["pillar_roots"] = {"base": [RUNTIME_VARS.TMP_PILLAR_TREE]}
74    minion_opts["file_roots"] = {
75        "base": [
76            # Let's support runtime created files that can be used like:
77            #   salt://my-temp-file.txt
78            RUNTIME_VARS.TMP_STATE_TREE
79        ],
80        # Alternate root to test __env__ choices
81        "prod": [
82            os.path.join(RUNTIME_VARS.FILES, "file", "prod"),
83            RUNTIME_VARS.TMP_PRODENV_STATE_TREE,
84        ],
85    }
86    if initial_conf_file and initial_conf_file.startswith(RUNTIME_VARS.FILES):
87        # We assume we were passed a minion configuration file defined fo testing and, as such
88        # we define the file and pillar roots to include the testing states/pillar trees
89        minion_opts["pillar_roots"]["base"].append(
90            os.path.join(RUNTIME_VARS.FILES, "pillar", "base"),
91        )
92        minion_opts["file_roots"]["base"].append(
93            os.path.join(RUNTIME_VARS.FILES, "file", "base"),
94        )
95        minion_opts["file_roots"]["prod"].append(
96            os.path.join(RUNTIME_VARS.FILES, "file", "prod"),
97        )
98
99    # We need to copy the extension modules into the new master root_dir or
100    # it will be prefixed by it
101    extension_modules_path = os.path.join(root_dir, "extension_modules")
102    if not os.path.exists(extension_modules_path):
103        shutil.copytree(
104            os.path.join(RUNTIME_VARS.FILES, "extension_modules"),
105            extension_modules_path,
106        )
107    minion_opts["extension_modules"] = extension_modules_path
108
109    # Custom grains
110    if "grains" not in minion_opts:
111        minion_opts["grains"] = {}
112    if minion_role is not None:
113        minion_opts["grains"]["role"] = minion_role
114
115    # Under windows we can't seem to properly create a virtualenv off of another
116    # virtualenv, we can on linux but we will still point to the virtualenv binary
117    # outside the virtualenv running the test suite, if that's the case.
118    try:
119        real_prefix = sys.real_prefix
120        # The above attribute exists, this is a virtualenv
121        if salt.utils.platform.is_windows():
122            virtualenv_binary = os.path.join(real_prefix, "Scripts", "virtualenv.exe")
123        else:
124            # We need to remove the virtualenv from PATH or we'll get the virtualenv binary
125            # from within the virtualenv, we don't want that
126            path = os.environ.get("PATH")
127            if path is not None:
128                path_items = path.split(os.pathsep)
129                for item in path_items[:]:
130                    if item.startswith(sys.base_prefix):
131                        path_items.remove(item)
132                os.environ["PATH"] = os.pathsep.join(path_items)
133            virtualenv_binary = salt.utils.path.which("virtualenv")
134            if path is not None:
135                # Restore previous environ PATH
136                os.environ["PATH"] = path
137            if not virtualenv_binary.startswith(real_prefix):
138                virtualenv_binary = None
139        if virtualenv_binary and not os.path.exists(virtualenv_binary):
140            # It doesn't exist?!
141            virtualenv_binary = None
142    except AttributeError:
143        # We're not running inside a virtualenv
144        virtualenv_binary = None
145    if virtualenv_binary:
146        minion_opts["venv_bin"] = virtualenv_binary
147
148    # Override minion_opts with minion_opts_overrides
149    if minion_opts_overrides:
150        minion_opts.update(minion_opts_overrides)
151
152    if not os.path.exists(conf_dir):
153        os.makedirs(conf_dir)
154
155    with salt.utils.files.fopen(conf_file, "w") as fp_:
156        salt.utils.yaml.safe_dump(minion_opts, fp_, default_flow_style=False)
157
158    log.info("Generating testing minion %r configuration completed.", minion_id)
159    minion_opts = salt.config.minion_config(
160        conf_file, minion_id=minion_id, cache_minion_id=True
161    )
162    salt.utils.verify.verify_env(
163        [
164            os.path.join(minion_opts["pki_dir"], "accepted"),
165            os.path.join(minion_opts["pki_dir"], "rejected"),
166            os.path.join(minion_opts["pki_dir"], "pending"),
167            os.path.dirname(minion_opts["log_file"]),
168            minion_opts["extension_modules"],
169            minion_opts["cachedir"],
170            minion_opts["sock_dir"],
171            RUNTIME_VARS.TMP_STATE_TREE,
172            RUNTIME_VARS.TMP_PILLAR_TREE,
173            RUNTIME_VARS.TMP_PRODENV_STATE_TREE,
174            RUNTIME_VARS.TMP,
175        ],
176        RUNTIME_VARS.RUNNING_TESTS_USER,
177        root_dir=root_dir,
178    )
179    if cache_opts:
180        try:
181            opts_cache = build_minion_opts.__cached_opts__
182        except AttributeError:
183            opts_cache = build_minion_opts.__cached_opts__ = {}
184        opts_cache[minion_id] = minion_opts
185    return minion_opts
186
187
188def create_sminion(
189    minion_id=None,
190    root_dir=None,
191    initial_conf_file=None,
192    sminion_cls=salt.minion.SMinion,
193    minion_opts_overrides=None,
194    skip_cached_minion=False,
195    cache_sminion=True,
196):
197    if minion_id is None:
198        minion_id = DEFAULT_SMINION_ID
199    if skip_cached_minion is False:
200        try:
201            minions_cache = create_sminion.__cached_minions__
202        except AttributeError:
203            create_sminion.__cached_minions__ = {}
204        cached_minion = create_sminion.__cached_minions__.get(minion_id)
205        if cached_minion:
206            return cached_minion
207    minion_opts = build_minion_opts(
208        minion_id=minion_id,
209        root_dir=root_dir,
210        initial_conf_file=initial_conf_file,
211        minion_opts_overrides=minion_opts_overrides,
212        skip_cached_opts=skip_cached_minion,
213        cache_opts=cache_sminion,
214    )
215    log.info("Instantiating a testing %s(%s)", sminion_cls.__name__, minion_id)
216    sminion = sminion_cls(minion_opts)
217    if cache_sminion:
218        try:
219            minions_cache = create_sminion.__cached_minions__
220        except AttributeError:
221            minions_cache = create_sminion.__cached_minions__ = {}
222        minions_cache[minion_id] = sminion
223    return sminion
224
225
226def check_required_sminion_attributes(sminion_attr, required_items):
227    """
228    :param sminion_attr: The name of the sminion attribute to check, such as 'functions' or 'states'
229    :param required_items: The items that must be part of the designated sminion attribute for the decorated test
230    :return The packages that are not available
231    """
232    required_salt_items = set(required_items)
233    sminion = create_sminion(minion_id=DEFAULT_SMINION_ID)
234    available_items = list(getattr(sminion, sminion_attr))
235    not_available_items = set()
236
237    name = "__not_available_{items}s__".format(items=sminion_attr)
238    if not hasattr(sminion, name):
239        setattr(sminion, name, set())
240
241    cached_not_available_items = getattr(sminion, name)
242
243    for not_available_item in cached_not_available_items:
244        if not_available_item in required_salt_items:
245            not_available_items.add(not_available_item)
246            required_salt_items.remove(not_available_item)
247
248    for required_item_name in required_salt_items:
249        search_name = required_item_name
250        if "." not in search_name:
251            search_name += ".*"
252        if not fnmatch.filter(available_items, search_name):
253            not_available_items.add(required_item_name)
254            cached_not_available_items.add(required_item_name)
255
256    return not_available_items
257