1import glob 2import logging 3import os 4import re 5 6import salt.utils.path 7import salt.utils.yaml 8from jinja2 import Environment, FileSystemLoader 9 10log = logging.getLogger(__name__) 11 12 13# Renders jinja from a template file 14def render_jinja(_file, salt_data): 15 j_env = Environment(loader=FileSystemLoader(os.path.dirname(_file))) 16 j_env.globals.update( 17 { 18 "__opts__": salt_data["__opts__"], 19 "__salt__": salt_data["__salt__"], 20 "__grains__": salt_data["__grains__"], 21 "__pillar__": salt_data["__pillar__"], 22 "minion_id": salt_data["minion_id"], 23 } 24 ) 25 j_render = j_env.get_template(os.path.basename(_file)).render() 26 return j_render 27 28 29# Renders yaml from rendered jinja 30def render_yaml(_file, salt_data): 31 return salt.utils.yaml.safe_load(render_jinja(_file, salt_data)) 32 33 34# Returns a dict from a class yaml definition 35def get_class(_class, salt_data): 36 l_files = [] 37 saltclass_path = salt_data["path"] 38 39 straight, sub_init, sub_straight = get_class_paths(_class, saltclass_path) 40 41 for root, dirs, files in salt.utils.path.os_walk( 42 os.path.join(saltclass_path, "classes"), followlinks=True 43 ): 44 for l_file in files: 45 l_files.append(os.path.join(root, l_file)) 46 47 if straight in l_files: 48 return render_yaml(straight, salt_data) 49 50 if sub_straight in l_files: 51 return render_yaml(sub_straight, salt_data) 52 53 if sub_init in l_files: 54 return render_yaml(sub_init, salt_data) 55 56 log.warning("%s: Class definition not found", _class) 57 return {} 58 59 60def get_class_paths(_class, saltclass_path): 61 """ 62 Converts the dotted notation of a saltclass class to its possible file counterparts. 63 64 :param str _class: Dotted notation of the class 65 :param str saltclass_path: Root to saltclass storage 66 :return: 3-tuple of possible file counterparts 67 :rtype: tuple(str) 68 """ 69 straight = os.path.join(saltclass_path, "classes", "{}.yml".format(_class)) 70 sub_straight = os.path.join( 71 saltclass_path, "classes", "{}.yml".format(_class.replace(".", os.sep)) 72 ) 73 sub_init = os.path.join( 74 saltclass_path, "classes", _class.replace(".", os.sep), "init.yml" 75 ) 76 return straight, sub_init, sub_straight 77 78 79def get_class_from_file(_file, saltclass_path): 80 """ 81 Converts the absolute path to a saltclass file back to the dotted notation. 82 83 .. code-block:: python 84 85 print(get_class_from_file('/srv/saltclass/classes/services/nginx/init.yml', '/srv/saltclass')) 86 # services.nginx 87 88 :param str _file: Absolute path to file 89 :param str saltclass_path: Root to saltclass storage 90 :return: class name in dotted notation 91 :rtype: str 92 """ 93 # remove classes path prefix 94 _file = _file[len(os.path.join(saltclass_path, "classes")) + len(os.sep) :] 95 # remove .yml extension 96 _file = _file[:-4] 97 # revert to dotted notation 98 _file = _file.replace(os.sep, ".") 99 # remove tailing init 100 if _file.endswith(".init"): 101 _file = _file[:-5] 102 return _file 103 104 105# Return environment 106def get_env_from_dict(exp_dict_list): 107 environment = "" 108 for s_class in exp_dict_list: 109 if "environment" in s_class: 110 environment = s_class["environment"] 111 return environment 112 113 114# Merge dict b into a 115def dict_merge(a, b, path=None): 116 if path is None: 117 path = [] 118 119 for key in b: 120 if key in a: 121 if isinstance(a[key], list) and isinstance(b[key], list): 122 if b[key][0] == "^": 123 b[key].pop(0) 124 a[key] = b[key] 125 else: 126 a[key].extend(b[key]) 127 elif isinstance(a[key], dict) and isinstance(b[key], dict): 128 dict_merge(a[key], b[key], path + [str(key)]) 129 elif a[key] == b[key]: 130 pass 131 else: 132 a[key] = b[key] 133 else: 134 a[key] = b[key] 135 return a 136 137 138# Recursive search and replace in a dict 139def dict_search_and_replace(d, old, new, expanded): 140 for (k, v) in d.items(): 141 if isinstance(v, dict): 142 dict_search_and_replace(d[k], old, new, expanded) 143 144 if isinstance(v, list): 145 x = 0 146 for i in v: 147 if isinstance(i, dict): 148 dict_search_and_replace(v[x], old, new, expanded) 149 if isinstance(i, str): 150 if i == old: 151 v[x] = new 152 x = x + 1 153 154 if v == old: 155 d[k] = new 156 157 return d 158 159 160# Retrieve original value from ${xx:yy:zz} to be expanded 161def find_value_to_expand(x, v): 162 a = x 163 for i in v[2:-1].split(":"): 164 if a is None: 165 return v 166 if i in a: 167 a = a.get(i) 168 else: 169 return v 170 return a 171 172 173# Look for regexes and expand them 174def find_and_process_re(_str, v, k, b, expanded): 175 vre = re.finditer(r"(^|.)\$\{.*?\}", _str) 176 if vre: 177 for re_v in vre: 178 re_str = str(re_v.group()) 179 if re_str.startswith("\\"): 180 v_new = _str.replace(re_str, re_str.lstrip("\\")) 181 b = dict_search_and_replace(b, _str, v_new, expanded) 182 expanded.append(k) 183 elif not re_str.startswith("$"): 184 v_expanded = find_value_to_expand(b, re_str[1:]) 185 v_new = _str.replace(re_str[1:], v_expanded) 186 b = dict_search_and_replace(b, _str, v_new, expanded) 187 _str = v_new 188 expanded.append(k) 189 else: 190 v_expanded = find_value_to_expand(b, re_str) 191 if isinstance(v, str): 192 v_new = v.replace(re_str, v_expanded) 193 else: 194 v_new = _str.replace(re_str, v_expanded) 195 b = dict_search_and_replace(b, _str, v_new, expanded) 196 _str = v_new 197 v = v_new 198 expanded.append(k) 199 return b 200 201 202# Return a dict that contains expanded variables if found 203def expand_variables(a, b, expanded, path=None): 204 if path is None: 205 b = a.copy() 206 path = [] 207 208 for (k, v) in a.items(): 209 if isinstance(v, dict): 210 expand_variables(v, b, expanded, path + [str(k)]) 211 else: 212 if isinstance(v, list): 213 for i in v: 214 if isinstance(i, dict): 215 expand_variables(i, b, expanded, path + [str(k)]) 216 if isinstance(i, str): 217 b = find_and_process_re(i, v, k, b, expanded) 218 219 if isinstance(v, str): 220 b = find_and_process_re(v, v, k, b, expanded) 221 return b 222 223 224def match_class_glob(_class, saltclass_path): 225 """ 226 Takes a class name possibly including `*` or `?` wildcards (or any other wildcards supportet by `glob.glob`) and 227 returns a list of expanded class names without wildcards. 228 229 .. code-block:: python 230 231 classes = match_class_glob('services.*', '/srv/saltclass') 232 print(classes) 233 # services.mariadb 234 # services.nginx... 235 236 237 :param str _class: dotted class name, globbing allowed. 238 :param str saltclass_path: path to the saltclass root directory. 239 240 :return: The list of expanded class matches. 241 :rtype: list(str) 242 """ 243 straight, sub_init, sub_straight = get_class_paths(_class, saltclass_path) 244 classes = [] 245 matches = [] 246 matches.extend(glob.glob(straight)) 247 matches.extend(glob.glob(sub_straight)) 248 matches.extend(glob.glob(sub_init)) 249 if not matches: 250 log.warning("%s: Class globbing did not yield any results", _class) 251 for match in matches: 252 classes.append(get_class_from_file(match, saltclass_path)) 253 return classes 254 255 256def expand_classes_glob(classes, salt_data): 257 """ 258 Expand the list of `classes` to no longer include any globbing. 259 260 :param iterable(str) classes: Iterable of classes 261 :param dict salt_data: configuration data 262 :return: Expanded list of classes with resolved globbing 263 :rtype: list(str) 264 """ 265 all_classes = [] 266 expanded_classes = [] 267 saltclass_path = salt_data["path"] 268 269 for _class in classes: 270 all_classes.extend(match_class_glob(_class, saltclass_path)) 271 272 for _class in all_classes: 273 if _class not in expanded_classes: 274 expanded_classes.append(_class) 275 276 return expanded_classes 277 278 279def expand_classes_in_order( 280 minion_dict, salt_data, seen_classes, expanded_classes, classes_to_expand 281): 282 # Get classes to expand from minion dictionary 283 if not classes_to_expand and "classes" in minion_dict: 284 classes_to_expand = minion_dict["classes"] 285 286 classes_to_expand = expand_classes_glob(classes_to_expand, salt_data) 287 288 # Now loop on list to recursively expand them 289 for klass in classes_to_expand: 290 if klass not in seen_classes: 291 seen_classes.append(klass) 292 expanded_classes[klass] = get_class(klass, salt_data) 293 # Fix corner case where class is loaded but doesn't contain anything 294 if expanded_classes[klass] is None: 295 expanded_classes[klass] = {} 296 297 # Merge newly found pillars into existing ones 298 new_pillars = expanded_classes[klass].get("pillars", {}) 299 if new_pillars: 300 dict_merge(salt_data["__pillar__"], new_pillars) 301 302 # Now replace class element in classes_to_expand by expansion 303 if expanded_classes[klass].get("classes"): 304 l_id = classes_to_expand.index(klass) 305 classes_to_expand[l_id:l_id] = expanded_classes[klass]["classes"] 306 expand_classes_in_order( 307 minion_dict, 308 salt_data, 309 seen_classes, 310 expanded_classes, 311 classes_to_expand, 312 ) 313 else: 314 expand_classes_in_order( 315 minion_dict, 316 salt_data, 317 seen_classes, 318 expanded_classes, 319 classes_to_expand, 320 ) 321 322 # We may have duplicates here and we want to remove them 323 tmp = [] 324 for t_element in classes_to_expand: 325 if t_element not in tmp: 326 tmp.append(t_element) 327 328 classes_to_expand = tmp 329 330 # Now that we've retrieved every class in order, 331 # let's return an ordered list of dicts 332 ord_expanded_classes = [] 333 ord_expanded_states = [] 334 for ord_klass in classes_to_expand: 335 ord_expanded_classes.append(expanded_classes[ord_klass]) 336 # And be smart and sort out states list 337 # Address the corner case where states is empty in a class definition 338 if ( 339 "states" in expanded_classes[ord_klass] 340 and expanded_classes[ord_klass]["states"] is None 341 ): 342 expanded_classes[ord_klass]["states"] = {} 343 344 if "states" in expanded_classes[ord_klass]: 345 ord_expanded_states.extend(expanded_classes[ord_klass]["states"]) 346 347 # Add our minion dict as final element but check if we have states to process 348 if "states" in minion_dict and minion_dict["states"] is None: 349 minion_dict["states"] = [] 350 351 if "states" in minion_dict: 352 ord_expanded_states.extend(minion_dict["states"]) 353 354 ord_expanded_classes.append(minion_dict) 355 356 return ord_expanded_classes, classes_to_expand, ord_expanded_states 357 358 359def expanded_dict_from_minion(minion_id, salt_data): 360 _file = "" 361 saltclass_path = salt_data["path"] 362 # Start 363 for root, dirs, files in salt.utils.path.os_walk( 364 os.path.join(saltclass_path, "nodes"), followlinks=True 365 ): 366 for minion_file in files: 367 if minion_file == "{}.yml".format(minion_id): 368 _file = os.path.join(root, minion_file) 369 370 # Load the minion_id definition if existing, else an empty dict 371 node_dict = {} 372 if _file: 373 node_dict[minion_id] = render_yaml(_file, salt_data) 374 else: 375 log.warning("%s: Node definition not found", minion_id) 376 node_dict[minion_id] = {} 377 378 # Merge newly found pillars into existing ones 379 dict_merge(salt_data["__pillar__"], node_dict[minion_id].get("pillars", {})) 380 381 # Get 2 ordered lists: 382 # expanded_classes: A list of all the dicts 383 # classes_list: List of all the classes 384 expanded_classes, classes_list, states_list = expand_classes_in_order( 385 node_dict[minion_id], salt_data, [], {}, [] 386 ) 387 388 # Here merge the pillars together 389 pillars_dict = {} 390 for exp_dict in expanded_classes: 391 if "pillars" in exp_dict: 392 dict_merge(pillars_dict, exp_dict) 393 394 return expanded_classes, pillars_dict, classes_list, states_list 395 396 397def get_pillars(minion_id, salt_data): 398 # Get 2 dicts and 2 lists 399 # expanded_classes: Full list of expanded dicts 400 # pillars_dict: dict containing merged pillars in order 401 # classes_list: All classes processed in order 402 # states_list: All states listed in order 403 ( 404 expanded_classes, 405 pillars_dict, 406 classes_list, 407 states_list, 408 ) = expanded_dict_from_minion(minion_id, salt_data) 409 410 # Retrieve environment 411 environment = get_env_from_dict(expanded_classes) 412 413 # Expand ${} variables in merged dict 414 # pillars key shouldn't exist if we haven't found any minion_id ref 415 if "pillars" in pillars_dict: 416 pillars_dict_expanded = expand_variables(pillars_dict["pillars"], {}, []) 417 else: 418 pillars_dict_expanded = expand_variables({}, {}, []) 419 420 # Build the final pillars dict 421 pillars_dict = {} 422 pillars_dict["__saltclass__"] = {} 423 pillars_dict["__saltclass__"]["states"] = states_list 424 pillars_dict["__saltclass__"]["classes"] = classes_list 425 pillars_dict["__saltclass__"]["environment"] = environment 426 pillars_dict["__saltclass__"]["nodename"] = minion_id 427 pillars_dict.update(pillars_dict_expanded) 428 429 return pillars_dict 430 431 432def get_tops(minion_id, salt_data): 433 # Get 2 dicts and 2 lists 434 # expanded_classes: Full list of expanded dicts 435 # pillars_dict: dict containing merged pillars in order 436 # classes_list: All classes processed in order 437 # states_list: All states listed in order 438 ( 439 expanded_classes, 440 pillars_dict, 441 classes_list, 442 states_list, 443 ) = expanded_dict_from_minion(minion_id, salt_data) 444 445 # Retrieve environment 446 environment = get_env_from_dict(expanded_classes) 447 448 # Build final top dict 449 tops_dict = {} 450 tops_dict[environment] = states_list 451 452 return tops_dict 453