1""" 2Jinja loading utils to enable a more powerful backend for jinja templates 3""" 4 5 6import atexit 7import logging 8import os.path 9import pipes 10import pprint 11import re 12import time 13import uuid 14import warnings 15from collections.abc import Hashable 16from functools import wraps 17from xml.dom import minidom 18from xml.etree.ElementTree import Element, SubElement, tostring 19 20import jinja2 21import salt.fileclient 22import salt.utils.data 23import salt.utils.files 24import salt.utils.json 25import salt.utils.stringutils 26import salt.utils.url 27import salt.utils.yaml 28from jinja2 import BaseLoader, Markup, TemplateNotFound, nodes 29from jinja2.environment import TemplateModule 30from jinja2.exceptions import TemplateRuntimeError 31from jinja2.ext import Extension 32from salt.exceptions import TemplateError 33from salt.utils.decorators.jinja import jinja_filter, jinja_global, jinja_test 34from salt.utils.odict import OrderedDict 35from salt.utils.versions import LooseVersion 36 37log = logging.getLogger(__name__) 38 39__all__ = ["SaltCacheLoader", "SerializerExtension"] 40 41GLOBAL_UUID = uuid.UUID("91633EBF-1C86-5E33-935A-28061F4B480E") 42JINJA_VERSION = LooseVersion(jinja2.__version__) 43 44 45class SaltCacheLoader(BaseLoader): 46 """ 47 A special jinja Template Loader for salt. 48 Requested templates are always fetched from the server 49 to guarantee that the file is up to date. 50 Templates are cached like regular salt states 51 and only loaded once per loader instance. 52 """ 53 54 _cached_pillar_client = None 55 _cached_client = None 56 57 @classmethod 58 def shutdown(cls): 59 for attr in ("_cached_client", "_cached_pillar_client"): 60 client = getattr(cls, attr, None) 61 if client is not None: 62 # PillarClient and LocalClient objects do not have a destroy method 63 if hasattr(client, "destroy"): 64 client.destroy() 65 setattr(cls, attr, None) 66 67 def __init__( 68 self, 69 opts, 70 saltenv="base", 71 encoding="utf-8", 72 pillar_rend=False, 73 _file_client=None, 74 ): 75 self.opts = opts 76 self.saltenv = saltenv 77 self.encoding = encoding 78 self.pillar_rend = pillar_rend 79 if self.pillar_rend: 80 if saltenv not in self.opts["pillar_roots"]: 81 self.searchpath = [] 82 else: 83 self.searchpath = opts["pillar_roots"][saltenv] 84 else: 85 self.searchpath = [os.path.join(opts["cachedir"], "files", saltenv)] 86 log.debug("Jinja search path: %s", self.searchpath) 87 self.cached = [] 88 self._file_client = _file_client 89 # Instantiate the fileclient 90 self.file_client() 91 92 def file_client(self): 93 """ 94 Return a file client. Instantiates on first call. 95 """ 96 # If there was no file_client passed to the class, create a cache_client 97 # and use that. This avoids opening a new file_client every time this 98 # class is instantiated 99 if self._file_client is None: 100 attr = "_cached_pillar_client" if self.pillar_rend else "_cached_client" 101 cached_client = getattr(self, attr, None) 102 if cached_client is None: 103 cached_client = salt.fileclient.get_file_client( 104 self.opts, self.pillar_rend 105 ) 106 setattr(SaltCacheLoader, attr, cached_client) 107 self._file_client = cached_client 108 return self._file_client 109 110 def cache_file(self, template): 111 """ 112 Cache a file from the salt master 113 """ 114 saltpath = salt.utils.url.create(template) 115 self.file_client().get_file(saltpath, "", True, self.saltenv) 116 117 def check_cache(self, template): 118 """ 119 Cache a file only once 120 """ 121 if template not in self.cached: 122 self.cache_file(template) 123 self.cached.append(template) 124 125 def get_source(self, environment, template): 126 """ 127 Salt-specific loader to find imported jinja files. 128 129 Jinja imports will be interpreted as originating from the top 130 of each of the directories in the searchpath when the template 131 name does not begin with './' or '../'. When a template name 132 begins with './' or '../' then the import will be relative to 133 the importing file. 134 135 """ 136 # FIXME: somewhere do seprataor replacement: '\\' => '/' 137 _template = template 138 if template.split("/", 1)[0] in ("..", "."): 139 is_relative = True 140 else: 141 is_relative = False 142 # checks for relative '..' paths that step-out of file_roots 143 if is_relative: 144 # Starts with a relative path indicator 145 146 if not environment or "tpldir" not in environment.globals: 147 log.warning( 148 'Relative path "%s" cannot be resolved without an environment', 149 template, 150 ) 151 raise TemplateNotFound(template) 152 base_path = environment.globals["tpldir"] 153 _template = os.path.normpath("/".join((base_path, _template))) 154 if _template.split("/", 1)[0] == "..": 155 log.warning( 156 'Discarded template path "%s": attempts to' 157 " ascend outside of salt://", 158 template, 159 ) 160 raise TemplateNotFound(template) 161 162 self.check_cache(_template) 163 164 if environment and template: 165 tpldir = os.path.dirname(_template).replace("\\", "/") 166 tplfile = _template 167 if is_relative: 168 tpldir = environment.globals.get("tpldir", tpldir) 169 tplfile = template 170 tpldata = { 171 "tplfile": tplfile, 172 "tpldir": "." if tpldir == "" else tpldir, 173 "tpldot": tpldir.replace("/", "."), 174 } 175 environment.globals.update(tpldata) 176 177 # pylint: disable=cell-var-from-loop 178 for spath in self.searchpath: 179 filepath = os.path.join(spath, _template) 180 try: 181 with salt.utils.files.fopen(filepath, "rb") as ifile: 182 contents = ifile.read().decode(self.encoding) 183 mtime = os.path.getmtime(filepath) 184 185 def uptodate(): 186 try: 187 return os.path.getmtime(filepath) == mtime 188 except OSError: 189 return False 190 191 return contents, filepath, uptodate 192 except OSError: 193 # there is no file under current path 194 continue 195 # pylint: enable=cell-var-from-loop 196 197 # there is no template file within searchpaths 198 raise TemplateNotFound(template) 199 200 201atexit.register(SaltCacheLoader.shutdown) 202 203 204class PrintableDict(OrderedDict): 205 """ 206 Ensures that dict str() and repr() are YAML friendly. 207 208 .. code-block:: python 209 210 mapping = OrderedDict([('a', 'b'), ('c', None)]) 211 print mapping 212 # OrderedDict([('a', 'b'), ('c', None)]) 213 214 decorated = PrintableDict(mapping) 215 print decorated 216 # {'a': 'b', 'c': None} 217 """ 218 219 def __str__(self): 220 output = [] 221 for key, value in self.items(): 222 if isinstance(value, str): 223 # keeps quotes around strings 224 # pylint: disable=repr-flag-used-in-string 225 output.append("{!r}: {!r}".format(key, value)) 226 # pylint: enable=repr-flag-used-in-string 227 else: 228 # let default output 229 output.append("{!r}: {!s}".format(key, value)) 230 return "{" + ", ".join(output) + "}" 231 232 def __repr__(self): # pylint: disable=W0221 233 output = [] 234 for key, value in self.items(): 235 # Raw string formatter required here because this is a repr 236 # function. 237 # pylint: disable=repr-flag-used-in-string 238 output.append("{!r}: {!r}".format(key, value)) 239 # pylint: enable=repr-flag-used-in-string 240 return "{" + ", ".join(output) + "}" 241 242 243# Additional globals 244@jinja_global("raise") 245def jinja_raise(msg): 246 raise TemplateError(msg) 247 248 249# Additional tests 250@jinja_test("match") 251def test_match(txt, rgx, ignorecase=False, multiline=False): 252 """Returns true if a sequence of chars matches a pattern.""" 253 flag = 0 254 if ignorecase: 255 flag |= re.I 256 if multiline: 257 flag |= re.M 258 compiled_rgx = re.compile(rgx, flag) 259 return True if compiled_rgx.match(txt) else False 260 261 262@jinja_test("equalto") 263def test_equalto(value, other): 264 """Returns true if two values are equal.""" 265 return value == other 266 267 268# Additional filters 269@jinja_filter("skip") 270def skip_filter(data): 271 """ 272 Suppress data output 273 274 .. code-block:: yaml 275 276 {% my_string = "foo" %} 277 278 {{ my_string|skip }} 279 280 will be rendered as empty string, 281 282 """ 283 return "" 284 285 286@jinja_filter("sequence") 287def ensure_sequence_filter(data): 288 """ 289 Ensure sequenced data. 290 291 **sequence** 292 293 ensure that parsed data is a sequence 294 295 .. code-block:: jinja 296 297 {% set my_string = "foo" %} 298 {% set my_list = ["bar", ] %} 299 {% set my_dict = {"baz": "qux"} %} 300 301 {{ my_string|sequence|first }} 302 {{ my_list|sequence|first }} 303 {{ my_dict|sequence|first }} 304 305 306 will be rendered as: 307 308 .. code-block:: yaml 309 310 foo 311 bar 312 baz 313 """ 314 if not isinstance(data, (list, tuple, set, dict)): 315 return [data] 316 return data 317 318 319@jinja_filter("to_bool") 320def to_bool(val): 321 """ 322 Returns the logical value. 323 324 .. code-block:: jinja 325 326 {{ 'yes' | to_bool }} 327 328 will be rendered as: 329 330 .. code-block:: text 331 332 True 333 """ 334 if val is None: 335 return False 336 if isinstance(val, bool): 337 return val 338 if isinstance(val, (str, (str,))): 339 return val.lower() in ("yes", "1", "true") 340 if isinstance(val, int): 341 return val > 0 342 if not isinstance(val, Hashable): 343 return len(val) > 0 344 return False 345 346 347@jinja_filter("indent") 348def indent(s, width=4, first=False, blank=False, indentfirst=None): 349 """ 350 A ported version of the "indent" filter containing a fix for indenting Markup 351 objects. If the minion has Jinja version 2.11 or newer, the "indent" filter 352 from upstream will be used, and this one will be ignored. 353 """ 354 if indentfirst is not None: 355 warnings.warn( 356 "The 'indentfirst' argument is renamed to 'first' and will" 357 " be removed in Jinja 3.0.", 358 DeprecationWarning, 359 stacklevel=2, 360 ) 361 first = indentfirst 362 363 indention = " " * width 364 newline = "\n" 365 366 if isinstance(s, Markup): 367 indention = Markup(indention) 368 newline = Markup(newline) 369 370 s += newline # this quirk is necessary for splitlines method 371 372 if blank: 373 rv = (newline + indention).join(s.splitlines()) 374 else: 375 lines = s.splitlines() 376 rv = lines.pop(0) 377 378 if lines: 379 rv += newline + newline.join( 380 indention + line if line else line for line in lines 381 ) 382 383 if first: 384 rv = indention + rv 385 386 return rv 387 388 389@jinja_filter("tojson") 390def tojson(val, indent=None, **options): 391 """ 392 Implementation of tojson filter (only present in Jinja 2.9 and later). 393 Unlike the Jinja built-in filter, this allows arbitrary options to be 394 passed in to the underlying JSON library. 395 """ 396 options.setdefault("ensure_ascii", True) 397 if indent is not None: 398 options["indent"] = indent 399 return ( 400 salt.utils.json.dumps(val, **options) 401 .replace("<", "\\u003c") 402 .replace(">", "\\u003e") 403 .replace("&", "\\u0026") 404 .replace("'", "\\u0027") 405 ) 406 407 408@jinja_filter("quote") 409def quote(txt): 410 """ 411 Wraps a text around quotes. 412 413 .. code-block:: jinja 414 415 {% set my_text = 'my_text' %} 416 {{ my_text | quote }} 417 418 will be rendered as: 419 420 .. code-block:: text 421 422 'my_text' 423 """ 424 return pipes.quote(txt) 425 426 427@jinja_filter() 428def regex_escape(value): 429 return re.escape(value) 430 431 432@jinja_filter("regex_search") 433def regex_search(txt, rgx, ignorecase=False, multiline=False): 434 """ 435 Searches for a pattern in the text. 436 437 .. code-block:: jinja 438 439 {% set my_text = 'abcd' %} 440 {{ my_text | regex_search('^(.*)BC(.*)$', ignorecase=True) }} 441 442 will be rendered as: 443 444 .. code-block:: text 445 446 ('a', 'd') 447 """ 448 flag = 0 449 if ignorecase: 450 flag |= re.I 451 if multiline: 452 flag |= re.M 453 obj = re.search(rgx, txt, flag) 454 if not obj: 455 return 456 return obj.groups() 457 458 459@jinja_filter("regex_match") 460def regex_match(txt, rgx, ignorecase=False, multiline=False): 461 """ 462 Searches for a pattern in the text. 463 464 .. code-block:: jinja 465 466 {% set my_text = 'abcd' %} 467 {{ my_text | regex_match('^(.*)BC(.*)$', ignorecase=True) }} 468 469 will be rendered as: 470 471 .. code-block:: text 472 473 ('a', 'd') 474 """ 475 flag = 0 476 if ignorecase: 477 flag |= re.I 478 if multiline: 479 flag |= re.M 480 obj = re.match(rgx, txt, flag) 481 if not obj: 482 return 483 return obj.groups() 484 485 486@jinja_filter("regex_replace") 487def regex_replace(txt, rgx, val, ignorecase=False, multiline=False): 488 r""" 489 Searches for a pattern and replaces with a sequence of characters. 490 491 .. code-block:: jinja 492 493 {% set my_text = 'lets replace spaces' %} 494 {{ my_text | regex_replace('\s+', '__') }} 495 496 will be rendered as: 497 498 .. code-block:: text 499 500 lets__replace__spaces 501 """ 502 flag = 0 503 if ignorecase: 504 flag |= re.I 505 if multiline: 506 flag |= re.M 507 compiled_rgx = re.compile(rgx, flag) 508 return compiled_rgx.sub(val, txt) 509 510 511@jinja_filter("uuid") 512def uuid_(val): 513 """ 514 Returns a UUID corresponding to the value passed as argument. 515 516 .. code-block:: jinja 517 518 {{ 'example' | uuid }} 519 520 will be rendered as: 521 522 .. code-block:: text 523 524 f4efeff8-c219-578a-bad7-3dc280612ec8 525 """ 526 return str(uuid.uuid5(GLOBAL_UUID, salt.utils.stringutils.to_str(val))) 527 528 529### List-related filters 530 531 532@jinja_filter() 533def unique(values): 534 """ 535 Removes duplicates from a list. 536 537 .. code-block:: jinja 538 539 {% set my_list = ['a', 'b', 'c', 'a', 'b'] -%} 540 {{ my_list | unique }} 541 542 will be rendered as: 543 544 .. code-block:: text 545 546 ['a', 'b', 'c'] 547 """ 548 ret = None 549 if isinstance(values, Hashable): 550 ret = set(values) 551 else: 552 ret = [] 553 for value in values: 554 if value not in ret: 555 ret.append(value) 556 return ret 557 558 559@jinja_filter("min") 560def lst_min(obj): 561 """ 562 Returns the min value. 563 564 .. code-block:: jinja 565 566 {% set my_list = [1,2,3,4] -%} 567 {{ my_list | min }} 568 569 will be rendered as: 570 571 .. code-block:: text 572 573 1 574 """ 575 return min(obj) 576 577 578@jinja_filter("max") 579def lst_max(obj): 580 """ 581 Returns the max value. 582 583 .. code-block:: jinja 584 585 {% my_list = [1,2,3,4] -%} 586 {{ set my_list | max }} 587 588 will be rendered as: 589 590 .. code-block:: text 591 592 4 593 """ 594 return max(obj) 595 596 597@jinja_filter("avg") 598def lst_avg(lst): 599 """ 600 Returns the average value of a list. 601 602 .. code-block:: jinja 603 604 {% my_list = [1,2,3,4] -%} 605 {{ set my_list | avg }} 606 607 will be rendered as: 608 609 .. code-block:: yaml 610 611 2.5 612 """ 613 if not isinstance(lst, Hashable): 614 return float(sum(lst) / len(lst)) 615 return float(lst) 616 617 618@jinja_filter("union") 619def union(lst1, lst2): 620 """ 621 Returns the union of two lists. 622 623 .. code-block:: jinja 624 625 {% my_list = [1,2,3,4] -%} 626 {{ set my_list | union([2, 4, 6]) }} 627 628 will be rendered as: 629 630 .. code-block:: text 631 632 [1, 2, 3, 4, 6] 633 """ 634 if isinstance(lst1, Hashable) and isinstance(lst2, Hashable): 635 return set(lst1) | set(lst2) 636 return unique(lst1 + lst2) 637 638 639@jinja_filter("intersect") 640def intersect(lst1, lst2): 641 """ 642 Returns the intersection of two lists. 643 644 .. code-block:: jinja 645 646 {% my_list = [1,2,3,4] -%} 647 {{ set my_list | intersect([2, 4, 6]) }} 648 649 will be rendered as: 650 651 .. code-block:: text 652 653 [2, 4] 654 """ 655 if isinstance(lst1, Hashable) and isinstance(lst2, Hashable): 656 return set(lst1) & set(lst2) 657 return unique([ele for ele in lst1 if ele in lst2]) 658 659 660@jinja_filter("difference") 661def difference(lst1, lst2): 662 """ 663 Returns the difference of two lists. 664 665 .. code-block:: jinja 666 667 {% my_list = [1,2,3,4] -%} 668 {{ set my_list | difference([2, 4, 6]) }} 669 670 will be rendered as: 671 672 .. code-block:: text 673 674 [1, 3, 6] 675 """ 676 if isinstance(lst1, Hashable) and isinstance(lst2, Hashable): 677 return set(lst1) - set(lst2) 678 return unique([ele for ele in lst1 if ele not in lst2]) 679 680 681@jinja_filter("symmetric_difference") 682def symmetric_difference(lst1, lst2): 683 """ 684 Returns the symmetric difference of two lists. 685 686 .. code-block:: jinja 687 688 {% my_list = [1,2,3,4] -%} 689 {{ set my_list | symmetric_difference([2, 4, 6]) }} 690 691 will be rendered as: 692 693 .. code-block:: text 694 695 [1, 3] 696 """ 697 if isinstance(lst1, Hashable) and isinstance(lst2, Hashable): 698 return set(lst1) ^ set(lst2) 699 return unique( 700 [ele for ele in union(lst1, lst2) if ele not in intersect(lst1, lst2)] 701 ) 702 703 704@jinja_filter("method_call") 705def method_call(obj, f_name, *f_args, **f_kwargs): 706 return getattr(obj, f_name, lambda *args, **kwargs: None)(*f_args, **f_kwargs) 707 708 709@jinja2.contextfunction 710def show_full_context(ctx): 711 return salt.utils.data.simple_types_filter( 712 {key: value for key, value in ctx.items()} 713 ) 714 715 716class SerializerExtension(Extension): 717 ''' 718 Yaml and Json manipulation. 719 720 **Format filters** 721 722 Allows jsonifying or yamlifying any data structure. For example, this dataset: 723 724 .. code-block:: python 725 726 data = { 727 'foo': True, 728 'bar': 42, 729 'baz': [1, 2, 3], 730 'qux': 2.0 731 } 732 733 .. code-block:: jinja 734 735 yaml = {{ data|yaml }} 736 json = {{ data|json }} 737 python = {{ data|python }} 738 xml = {{ {'root_node': data}|xml }} 739 740 will be rendered as:: 741 742 yaml = {bar: 42, baz: [1, 2, 3], foo: true, qux: 2.0} 743 json = {"baz": [1, 2, 3], "foo": true, "bar": 42, "qux": 2.0} 744 python = {'bar': 42, 'baz': [1, 2, 3], 'foo': True, 'qux': 2.0} 745 xml = """<<?xml version="1.0" ?> 746 <root_node bar="42" foo="True" qux="2.0"> 747 <baz>1</baz> 748 <baz>2</baz> 749 <baz>3</baz> 750 </root_node>""" 751 752 The yaml filter takes an optional flow_style parameter to control the 753 default-flow-style parameter of the YAML dumper. 754 755 .. code-block:: jinja 756 757 {{ data|yaml(False) }} 758 759 will be rendered as: 760 761 .. code-block:: yaml 762 763 bar: 42 764 baz: 765 - 1 766 - 2 767 - 3 768 foo: true 769 qux: 2.0 770 771 **Load filters** 772 773 Strings and variables can be deserialized with **load_yaml** and 774 **load_json** tags and filters. It allows one to manipulate data directly 775 in templates, easily: 776 777 .. code-block:: jinja 778 779 {%- set yaml_src = "{foo: it works}"|load_yaml %} 780 {%- set json_src = "{'bar': 'for real'}"|load_json %} 781 Dude, {{ yaml_src.foo }} {{ json_src.bar }}! 782 783 will be rendered as:: 784 785 Dude, it works for real! 786 787 **Load tags** 788 789 Salt implements ``load_yaml`` and ``load_json`` tags. They work like 790 the `import tag`_, except that the document is also deserialized. 791 792 Syntaxes are ``{% load_yaml as [VARIABLE] %}[YOUR DATA]{% endload %}`` 793 and ``{% load_json as [VARIABLE] %}[YOUR DATA]{% endload %}`` 794 795 For example: 796 797 .. code-block:: jinja 798 799 {% load_yaml as yaml_src %} 800 foo: it works 801 {% endload %} 802 {% load_json as json_src %} 803 { 804 "bar": "for real" 805 } 806 {% endload %} 807 Dude, {{ yaml_src.foo }} {{ json_src.bar }}! 808 809 will be rendered as:: 810 811 Dude, it works for real! 812 813 **Import tags** 814 815 External files can be imported and made available as a Jinja variable. 816 817 .. code-block:: jinja 818 819 {% import_yaml "myfile.yml" as myfile %} 820 {% import_json "defaults.json" as defaults %} 821 {% import_text "completeworksofshakespeare.txt" as poems %} 822 823 **Catalog** 824 825 ``import_*`` and ``load_*`` tags will automatically expose their 826 target variable to import. This feature makes catalog of data to 827 handle. 828 829 for example: 830 831 .. code-block:: jinja 832 833 # doc1.sls 834 {% load_yaml as var1 %} 835 foo: it works 836 {% endload %} 837 {% load_yaml as var2 %} 838 bar: for real 839 {% endload %} 840 841 .. code-block:: jinja 842 843 # doc2.sls 844 {% from "doc1.sls" import var1, var2 as local2 %} 845 {{ var1.foo }} {{ local2.bar }} 846 847 ** Escape Filters ** 848 849 .. versionadded:: 2017.7.0 850 851 Allows escaping of strings so they can be interpreted literally by another 852 function. 853 854 For example: 855 856 .. code-block:: jinja 857 858 regex_escape = {{ 'https://example.com?foo=bar%20baz' | regex_escape }} 859 860 will be rendered as:: 861 862 regex_escape = https\\:\\/\\/example\\.com\\?foo\\=bar\\%20baz 863 864 ** Set Theory Filters ** 865 866 .. versionadded:: 2017.7.0 867 868 Performs set math using Jinja filters. 869 870 For example: 871 872 .. code-block:: jinja 873 874 unique = {{ ['foo', 'foo', 'bar'] | unique }} 875 876 will be rendered as:: 877 878 unique = ['foo', 'bar'] 879 880 .. _`import tag`: https://jinja.palletsprojects.com/en/2.11.x/templates/#import 881 ''' 882 883 tags = { 884 "load_yaml", 885 "load_json", 886 "import_yaml", 887 "import_json", 888 "load_text", 889 "import_text", 890 "profile", 891 } 892 893 def __init__(self, environment): 894 super().__init__(environment) 895 self.environment.filters.update( 896 { 897 "yaml": self.format_yaml, 898 "json": self.format_json, 899 "xml": self.format_xml, 900 "python": self.format_python, 901 "load_yaml": self.load_yaml, 902 "load_json": self.load_json, 903 "load_text": self.load_text, 904 } 905 ) 906 907 if self.environment.finalize is None: 908 self.environment.finalize = self.finalizer 909 else: 910 finalizer = self.environment.finalize 911 912 @wraps(finalizer) 913 def wrapper(self, data): 914 return finalizer(self.finalizer(data)) 915 916 self.environment.finalize = wrapper 917 918 def finalizer(self, data): 919 """ 920 Ensure that printed mappings are YAML friendly. 921 """ 922 923 def explore(data): 924 if isinstance(data, (dict, OrderedDict)): 925 return PrintableDict( 926 [(key, explore(value)) for key, value in data.items()] 927 ) 928 elif isinstance(data, (list, tuple, set)): 929 return data.__class__([explore(value) for value in data]) 930 return data 931 932 return explore(data) 933 934 def format_json(self, value, sort_keys=True, indent=None): 935 json_txt = salt.utils.json.dumps( 936 value, sort_keys=sort_keys, indent=indent 937 ).strip() 938 try: 939 return Markup(json_txt) 940 except UnicodeDecodeError: 941 return Markup(salt.utils.stringutils.to_unicode(json_txt)) 942 943 def format_yaml(self, value, flow_style=True): 944 yaml_txt = salt.utils.yaml.safe_dump( 945 value, default_flow_style=flow_style 946 ).strip() 947 if yaml_txt.endswith("\n..."): 948 yaml_txt = yaml_txt[: len(yaml_txt) - 4] 949 try: 950 return Markup(yaml_txt) 951 except UnicodeDecodeError: 952 return Markup(salt.utils.stringutils.to_unicode(yaml_txt)) 953 954 def format_xml(self, value): 955 """Render a formatted multi-line XML string from a complex Python 956 data structure. Supports tag attributes and nested dicts/lists. 957 958 :param value: Complex data structure representing XML contents 959 :returns: Formatted XML string rendered with newlines and indentation 960 :rtype: str 961 """ 962 963 def normalize_iter(value): 964 if isinstance(value, (list, tuple)): 965 if isinstance(value[0], str): 966 xmlval = value 967 else: 968 xmlval = [] 969 elif isinstance(value, dict): 970 xmlval = list(value.items()) 971 else: 972 raise TemplateRuntimeError( 973 "Value is not a dict or list. Cannot render as XML" 974 ) 975 return xmlval 976 977 def recurse_tree(xmliter, element=None): 978 sub = None 979 for tag, attrs in xmliter: 980 if isinstance(attrs, list): 981 for attr in attrs: 982 recurse_tree(((tag, attr),), element) 983 elif element is not None: 984 sub = SubElement(element, tag) 985 else: 986 sub = Element(tag) 987 if isinstance(attrs, (str, int, bool, float)): 988 sub.text = str(attrs) 989 continue 990 if isinstance(attrs, dict): 991 sub.attrib = { 992 attr: str(val) 993 for attr, val in attrs.items() 994 if not isinstance(val, (dict, list)) 995 } 996 for tag, val in [ 997 item 998 for item in normalize_iter(attrs) 999 if isinstance(item[1], (dict, list)) 1000 ]: 1001 recurse_tree(((tag, val),), sub) 1002 return sub 1003 1004 return Markup( 1005 minidom.parseString( 1006 tostring(recurse_tree(normalize_iter(value))) 1007 ).toprettyxml(indent=" ") 1008 ) 1009 1010 def format_python(self, value): 1011 return Markup(pprint.pformat(value).strip()) 1012 1013 def load_yaml(self, value): 1014 if isinstance(value, TemplateModule): 1015 value = str(value) 1016 try: 1017 return salt.utils.data.decode(salt.utils.yaml.safe_load(value)) 1018 except salt.utils.yaml.YAMLError as exc: 1019 msg = "Encountered error loading yaml: " 1020 try: 1021 # Reported line is off by one, add 1 to correct it 1022 line = exc.problem_mark.line + 1 1023 buf = exc.problem_mark.buffer 1024 problem = exc.problem 1025 except AttributeError: 1026 # No context information available in the exception, fall back 1027 # to the stringified version of the exception. 1028 msg += str(exc) 1029 else: 1030 msg += "{}\n".format(problem) 1031 msg += salt.utils.stringutils.get_context( 1032 buf, line, marker=" <======================" 1033 ) 1034 raise TemplateRuntimeError(msg) 1035 except AttributeError: 1036 raise TemplateRuntimeError("Unable to load yaml from {}".format(value)) 1037 1038 def load_json(self, value): 1039 if isinstance(value, TemplateModule): 1040 value = str(value) 1041 try: 1042 return salt.utils.json.loads(value) 1043 except (ValueError, TypeError, AttributeError): 1044 raise TemplateRuntimeError("Unable to load json from {}".format(value)) 1045 1046 def load_text(self, value): 1047 if isinstance(value, TemplateModule): 1048 value = str(value) 1049 1050 return value 1051 1052 _load_parsers = {"load_yaml", "load_json", "load_text"} 1053 _import_parsers = {"import_yaml", "import_json", "import_text"} 1054 1055 def parse(self, parser): 1056 if parser.stream.current.value in self._load_parsers: 1057 return self.parse_load(parser) 1058 elif parser.stream.current.value in self._import_parsers: 1059 return self.parse_import( 1060 parser, parser.stream.current.value.split("_", 1)[1] 1061 ) 1062 elif parser.stream.current.value == "profile": 1063 return self.parse_profile(parser) 1064 1065 parser.fail( 1066 "Unknown format " + parser.stream.current.value, 1067 parser.stream.current.lineno, 1068 ) 1069 1070 # pylint: disable=E1120,E1121 1071 def parse_profile(self, parser): 1072 lineno = next(parser.stream).lineno 1073 parser.stream.expect("name:as") 1074 label = parser.parse_expression() 1075 body = parser.parse_statements(["name:endprofile"], drop_needle=True) 1076 return self._parse_profile_block(parser, label, "profile block", body, lineno) 1077 1078 def _create_profile_id(self, parser): 1079 return "_salt_profile_{}".format(parser.free_identifier().name) 1080 1081 def _profile_start(self, label, source): 1082 return (label, source, time.time()) 1083 1084 def _profile_end(self, label, source, previous_time): 1085 log.profile( 1086 "Time (in seconds) to render %s '%s': %s", 1087 source, 1088 label, 1089 time.time() - previous_time, 1090 ) 1091 1092 def _parse_profile_block(self, parser, label, source, body, lineno): 1093 profile_id = self._create_profile_id(parser) 1094 ret = ( 1095 [ 1096 nodes.Assign( 1097 nodes.Name(profile_id, "store").set_lineno(lineno), 1098 self.call_method( 1099 "_profile_start", 1100 dyn_args=nodes.List([label, nodes.Const(source)]).set_lineno( 1101 lineno 1102 ), 1103 ).set_lineno(lineno), 1104 ).set_lineno(lineno), 1105 ] 1106 + body 1107 + [ 1108 nodes.ExprStmt( 1109 self.call_method( 1110 "_profile_end", dyn_args=nodes.Name(profile_id, "load") 1111 ), 1112 ).set_lineno(lineno), 1113 ] 1114 ) 1115 return ret 1116 1117 def parse_load(self, parser): 1118 filter_name = parser.stream.current.value 1119 lineno = next(parser.stream).lineno 1120 if filter_name not in self.environment.filters: 1121 parser.fail("Unable to parse {}".format(filter_name), lineno) 1122 1123 parser.stream.expect("name:as") 1124 target = parser.parse_assign_target() 1125 macro_name = "_" + parser.free_identifier().name 1126 macro_body = parser.parse_statements(("name:endload",), drop_needle=True) 1127 1128 return [ 1129 nodes.Macro(macro_name, [], [], macro_body).set_lineno(lineno), 1130 nodes.Assign( 1131 target, 1132 nodes.Filter( 1133 nodes.Call( 1134 nodes.Name(macro_name, "load").set_lineno(lineno), 1135 [], 1136 [], 1137 None, 1138 None, 1139 ).set_lineno(lineno), 1140 filter_name, 1141 [], 1142 [], 1143 None, 1144 None, 1145 ).set_lineno(lineno), 1146 ).set_lineno(lineno), 1147 ] 1148 1149 def parse_import(self, parser, converter): 1150 import_node = parser.parse_import() 1151 target = import_node.target 1152 lineno = import_node.lineno 1153 1154 body = [ 1155 import_node, 1156 nodes.Assign( 1157 nodes.Name(target, "store").set_lineno(lineno), 1158 nodes.Filter( 1159 nodes.Name(target, "load").set_lineno(lineno), 1160 "load_{}".format(converter), 1161 [], 1162 [], 1163 None, 1164 None, 1165 ).set_lineno(lineno), 1166 ).set_lineno(lineno), 1167 ] 1168 return self._parse_profile_block( 1169 parser, import_node.template, "import_{}".format(converter), body, lineno 1170 ) 1171 1172 # pylint: enable=E1120,E1121 1173