1import glob
2import logging
3import os
4import re
5
6import salt.utils.path
7import salt.utils.yaml
8from jinja2 import Environment, FileSystemLoader
9
10log = logging.getLogger(__name__)
11
12
13# Renders jinja from a template file
14def render_jinja(_file, salt_data):
15    j_env = Environment(loader=FileSystemLoader(os.path.dirname(_file)))
16    j_env.globals.update(
17        {
18            "__opts__": salt_data["__opts__"],
19            "__salt__": salt_data["__salt__"],
20            "__grains__": salt_data["__grains__"],
21            "__pillar__": salt_data["__pillar__"],
22            "minion_id": salt_data["minion_id"],
23        }
24    )
25    j_render = j_env.get_template(os.path.basename(_file)).render()
26    return j_render
27
28
29# Renders yaml from rendered jinja
30def render_yaml(_file, salt_data):
31    return salt.utils.yaml.safe_load(render_jinja(_file, salt_data))
32
33
34# Returns a dict from a class yaml definition
35def get_class(_class, salt_data):
36    l_files = []
37    saltclass_path = salt_data["path"]
38
39    straight, sub_init, sub_straight = get_class_paths(_class, saltclass_path)
40
41    for root, dirs, files in salt.utils.path.os_walk(
42        os.path.join(saltclass_path, "classes"), followlinks=True
43    ):
44        for l_file in files:
45            l_files.append(os.path.join(root, l_file))
46
47    if straight in l_files:
48        return render_yaml(straight, salt_data)
49
50    if sub_straight in l_files:
51        return render_yaml(sub_straight, salt_data)
52
53    if sub_init in l_files:
54        return render_yaml(sub_init, salt_data)
55
56    log.warning("%s: Class definition not found", _class)
57    return {}
58
59
60def get_class_paths(_class, saltclass_path):
61    """
62    Converts the dotted notation of a saltclass class to its possible file counterparts.
63
64    :param str _class: Dotted notation of the class
65    :param str saltclass_path: Root to saltclass storage
66    :return: 3-tuple of possible file counterparts
67    :rtype: tuple(str)
68    """
69    straight = os.path.join(saltclass_path, "classes", "{}.yml".format(_class))
70    sub_straight = os.path.join(
71        saltclass_path, "classes", "{}.yml".format(_class.replace(".", os.sep))
72    )
73    sub_init = os.path.join(
74        saltclass_path, "classes", _class.replace(".", os.sep), "init.yml"
75    )
76    return straight, sub_init, sub_straight
77
78
79def get_class_from_file(_file, saltclass_path):
80    """
81    Converts the absolute path to a saltclass file back to the dotted notation.
82
83    .. code-block:: python
84
85       print(get_class_from_file('/srv/saltclass/classes/services/nginx/init.yml', '/srv/saltclass'))
86       # services.nginx
87
88    :param str _file: Absolute path to file
89    :param str saltclass_path: Root to saltclass storage
90    :return: class name in dotted notation
91    :rtype: str
92    """
93    # remove classes path prefix
94    _file = _file[len(os.path.join(saltclass_path, "classes")) + len(os.sep) :]
95    # remove .yml extension
96    _file = _file[:-4]
97    # revert to dotted notation
98    _file = _file.replace(os.sep, ".")
99    # remove tailing init
100    if _file.endswith(".init"):
101        _file = _file[:-5]
102    return _file
103
104
105# Return environment
106def get_env_from_dict(exp_dict_list):
107    environment = ""
108    for s_class in exp_dict_list:
109        if "environment" in s_class:
110            environment = s_class["environment"]
111    return environment
112
113
114# Merge dict b into a
115def dict_merge(a, b, path=None):
116    if path is None:
117        path = []
118
119    for key in b:
120        if key in a:
121            if isinstance(a[key], list) and isinstance(b[key], list):
122                if b[key][0] == "^":
123                    b[key].pop(0)
124                    a[key] = b[key]
125                else:
126                    a[key].extend(b[key])
127            elif isinstance(a[key], dict) and isinstance(b[key], dict):
128                dict_merge(a[key], b[key], path + [str(key)])
129            elif a[key] == b[key]:
130                pass
131            else:
132                a[key] = b[key]
133        else:
134            a[key] = b[key]
135    return a
136
137
138# Recursive search and replace in a dict
139def dict_search_and_replace(d, old, new, expanded):
140    for (k, v) in d.items():
141        if isinstance(v, dict):
142            dict_search_and_replace(d[k], old, new, expanded)
143
144        if isinstance(v, list):
145            x = 0
146            for i in v:
147                if isinstance(i, dict):
148                    dict_search_and_replace(v[x], old, new, expanded)
149                if isinstance(i, str):
150                    if i == old:
151                        v[x] = new
152                x = x + 1
153
154        if v == old:
155            d[k] = new
156
157    return d
158
159
160# Retrieve original value from ${xx:yy:zz} to be expanded
161def find_value_to_expand(x, v):
162    a = x
163    for i in v[2:-1].split(":"):
164        if a is None:
165            return v
166        if i in a:
167            a = a.get(i)
168        else:
169            return v
170    return a
171
172
173# Look for regexes and expand them
174def find_and_process_re(_str, v, k, b, expanded):
175    vre = re.finditer(r"(^|.)\$\{.*?\}", _str)
176    if vre:
177        for re_v in vre:
178            re_str = str(re_v.group())
179            if re_str.startswith("\\"):
180                v_new = _str.replace(re_str, re_str.lstrip("\\"))
181                b = dict_search_and_replace(b, _str, v_new, expanded)
182                expanded.append(k)
183            elif not re_str.startswith("$"):
184                v_expanded = find_value_to_expand(b, re_str[1:])
185                v_new = _str.replace(re_str[1:], v_expanded)
186                b = dict_search_and_replace(b, _str, v_new, expanded)
187                _str = v_new
188                expanded.append(k)
189            else:
190                v_expanded = find_value_to_expand(b, re_str)
191                if isinstance(v, str):
192                    v_new = v.replace(re_str, v_expanded)
193                else:
194                    v_new = _str.replace(re_str, v_expanded)
195                b = dict_search_and_replace(b, _str, v_new, expanded)
196                _str = v_new
197                v = v_new
198                expanded.append(k)
199    return b
200
201
202# Return a dict that contains expanded variables if found
203def expand_variables(a, b, expanded, path=None):
204    if path is None:
205        b = a.copy()
206        path = []
207
208    for (k, v) in a.items():
209        if isinstance(v, dict):
210            expand_variables(v, b, expanded, path + [str(k)])
211        else:
212            if isinstance(v, list):
213                for i in v:
214                    if isinstance(i, dict):
215                        expand_variables(i, b, expanded, path + [str(k)])
216                    if isinstance(i, str):
217                        b = find_and_process_re(i, v, k, b, expanded)
218
219            if isinstance(v, str):
220                b = find_and_process_re(v, v, k, b, expanded)
221    return b
222
223
224def match_class_glob(_class, saltclass_path):
225    """
226    Takes a class name possibly including `*` or `?` wildcards (or any other wildcards supportet by `glob.glob`) and
227    returns a list of expanded class names without wildcards.
228
229    .. code-block:: python
230
231       classes = match_class_glob('services.*', '/srv/saltclass')
232       print(classes)
233       # services.mariadb
234       # services.nginx...
235
236
237    :param str _class: dotted class name, globbing allowed.
238    :param str saltclass_path: path to the saltclass root directory.
239
240    :return: The list of expanded class matches.
241    :rtype: list(str)
242    """
243    straight, sub_init, sub_straight = get_class_paths(_class, saltclass_path)
244    classes = []
245    matches = []
246    matches.extend(glob.glob(straight))
247    matches.extend(glob.glob(sub_straight))
248    matches.extend(glob.glob(sub_init))
249    if not matches:
250        log.warning("%s: Class globbing did not yield any results", _class)
251    for match in matches:
252        classes.append(get_class_from_file(match, saltclass_path))
253    return classes
254
255
256def expand_classes_glob(classes, salt_data):
257    """
258    Expand the list of `classes` to no longer include any globbing.
259
260    :param iterable(str) classes: Iterable of classes
261    :param dict salt_data: configuration data
262    :return: Expanded list of classes with resolved globbing
263    :rtype: list(str)
264    """
265    all_classes = []
266    expanded_classes = []
267    saltclass_path = salt_data["path"]
268
269    for _class in classes:
270        all_classes.extend(match_class_glob(_class, saltclass_path))
271
272    for _class in all_classes:
273        if _class not in expanded_classes:
274            expanded_classes.append(_class)
275
276    return expanded_classes
277
278
279def expand_classes_in_order(
280    minion_dict, salt_data, seen_classes, expanded_classes, classes_to_expand
281):
282    # Get classes to expand from minion dictionary
283    if not classes_to_expand and "classes" in minion_dict:
284        classes_to_expand = minion_dict["classes"]
285
286    classes_to_expand = expand_classes_glob(classes_to_expand, salt_data)
287
288    # Now loop on list to recursively expand them
289    for klass in classes_to_expand:
290        if klass not in seen_classes:
291            seen_classes.append(klass)
292            expanded_classes[klass] = get_class(klass, salt_data)
293            # Fix corner case where class is loaded but doesn't contain anything
294            if expanded_classes[klass] is None:
295                expanded_classes[klass] = {}
296
297            # Merge newly found pillars into existing ones
298            new_pillars = expanded_classes[klass].get("pillars", {})
299            if new_pillars:
300                dict_merge(salt_data["__pillar__"], new_pillars)
301
302            # Now replace class element in classes_to_expand by expansion
303            if expanded_classes[klass].get("classes"):
304                l_id = classes_to_expand.index(klass)
305                classes_to_expand[l_id:l_id] = expanded_classes[klass]["classes"]
306                expand_classes_in_order(
307                    minion_dict,
308                    salt_data,
309                    seen_classes,
310                    expanded_classes,
311                    classes_to_expand,
312                )
313            else:
314                expand_classes_in_order(
315                    minion_dict,
316                    salt_data,
317                    seen_classes,
318                    expanded_classes,
319                    classes_to_expand,
320                )
321
322    # We may have duplicates here and we want to remove them
323    tmp = []
324    for t_element in classes_to_expand:
325        if t_element not in tmp:
326            tmp.append(t_element)
327
328    classes_to_expand = tmp
329
330    # Now that we've retrieved every class in order,
331    # let's return an ordered list of dicts
332    ord_expanded_classes = []
333    ord_expanded_states = []
334    for ord_klass in classes_to_expand:
335        ord_expanded_classes.append(expanded_classes[ord_klass])
336        # And be smart and sort out states list
337        # Address the corner case where states is empty in a class definition
338        if (
339            "states" in expanded_classes[ord_klass]
340            and expanded_classes[ord_klass]["states"] is None
341        ):
342            expanded_classes[ord_klass]["states"] = {}
343
344        if "states" in expanded_classes[ord_klass]:
345            ord_expanded_states.extend(expanded_classes[ord_klass]["states"])
346
347    # Add our minion dict as final element but check if we have states to process
348    if "states" in minion_dict and minion_dict["states"] is None:
349        minion_dict["states"] = []
350
351    if "states" in minion_dict:
352        ord_expanded_states.extend(minion_dict["states"])
353
354    ord_expanded_classes.append(minion_dict)
355
356    return ord_expanded_classes, classes_to_expand, ord_expanded_states
357
358
359def expanded_dict_from_minion(minion_id, salt_data):
360    _file = ""
361    saltclass_path = salt_data["path"]
362    # Start
363    for root, dirs, files in salt.utils.path.os_walk(
364        os.path.join(saltclass_path, "nodes"), followlinks=True
365    ):
366        for minion_file in files:
367            if minion_file == "{}.yml".format(minion_id):
368                _file = os.path.join(root, minion_file)
369
370    # Load the minion_id definition if existing, else an empty dict
371    node_dict = {}
372    if _file:
373        node_dict[minion_id] = render_yaml(_file, salt_data)
374    else:
375        log.warning("%s: Node definition not found", minion_id)
376        node_dict[minion_id] = {}
377
378    # Merge newly found pillars into existing ones
379    dict_merge(salt_data["__pillar__"], node_dict[minion_id].get("pillars", {}))
380
381    # Get 2 ordered lists:
382    # expanded_classes: A list of all the dicts
383    # classes_list: List of all the classes
384    expanded_classes, classes_list, states_list = expand_classes_in_order(
385        node_dict[minion_id], salt_data, [], {}, []
386    )
387
388    # Here merge the pillars together
389    pillars_dict = {}
390    for exp_dict in expanded_classes:
391        if "pillars" in exp_dict:
392            dict_merge(pillars_dict, exp_dict)
393
394    return expanded_classes, pillars_dict, classes_list, states_list
395
396
397def get_pillars(minion_id, salt_data):
398    # Get 2 dicts and 2 lists
399    # expanded_classes: Full list of expanded dicts
400    # pillars_dict: dict containing merged pillars in order
401    # classes_list: All classes processed in order
402    # states_list: All states listed in order
403    (
404        expanded_classes,
405        pillars_dict,
406        classes_list,
407        states_list,
408    ) = expanded_dict_from_minion(minion_id, salt_data)
409
410    # Retrieve environment
411    environment = get_env_from_dict(expanded_classes)
412
413    # Expand ${} variables in merged dict
414    # pillars key shouldn't exist if we haven't found any minion_id ref
415    if "pillars" in pillars_dict:
416        pillars_dict_expanded = expand_variables(pillars_dict["pillars"], {}, [])
417    else:
418        pillars_dict_expanded = expand_variables({}, {}, [])
419
420    # Build the final pillars dict
421    pillars_dict = {}
422    pillars_dict["__saltclass__"] = {}
423    pillars_dict["__saltclass__"]["states"] = states_list
424    pillars_dict["__saltclass__"]["classes"] = classes_list
425    pillars_dict["__saltclass__"]["environment"] = environment
426    pillars_dict["__saltclass__"]["nodename"] = minion_id
427    pillars_dict.update(pillars_dict_expanded)
428
429    return pillars_dict
430
431
432def get_tops(minion_id, salt_data):
433    # Get 2 dicts and 2 lists
434    # expanded_classes: Full list of expanded dicts
435    # pillars_dict: dict containing merged pillars in order
436    # classes_list: All classes processed in order
437    # states_list: All states listed in order
438    (
439        expanded_classes,
440        pillars_dict,
441        classes_list,
442        states_list,
443    ) = expanded_dict_from_minion(minion_id, salt_data)
444
445    # Retrieve environment
446    environment = get_env_from_dict(expanded_classes)
447
448    # Build final top dict
449    tops_dict = {}
450    tops_dict[environment] = states_list
451
452    return tops_dict
453