1# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> 2# 3# This file is part of Ansible 4# 5# Ansible is free software: you can redistribute it and/or modify 6# it under the terms of the GNU General Public License as published by 7# the Free Software Foundation, either version 3 of the License, or 8# (at your option) any later version. 9# 10# Ansible is distributed in the hope that it will be useful, 11# but WITHOUT ANY WARRANTY; without even the implied warranty of 12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13# GNU General Public License for more details. 14# 15# You should have received a copy of the GNU General Public License 16# along with Ansible. If not, see <http://www.gnu.org/licenses/>. 17 18# Make coding more python3-ish 19from __future__ import (absolute_import, division, print_function) 20__metaclass__ = type 21 22import ast 23import datetime 24import os 25import pkgutil 26import pwd 27import re 28import time 29 30from contextlib import contextmanager 31from distutils.version import LooseVersion 32from numbers import Number 33from traceback import format_exc 34 35try: 36 from hashlib import sha1 37except ImportError: 38 from sha import sha as sha1 39 40from jinja2.exceptions import TemplateSyntaxError, UndefinedError 41from jinja2.loaders import FileSystemLoader 42from jinja2.runtime import Context, StrictUndefined 43 44from ansible import constants as C 45from ansible.errors import AnsibleError, AnsibleFilterError, AnsiblePluginRemovedError, AnsibleUndefinedVariable, AnsibleAssertionError 46from ansible.module_utils.six import iteritems, string_types, text_type 47from ansible.module_utils.six.moves import range 48from ansible.module_utils._text import to_native, to_text, to_bytes 49from ansible.module_utils.common._collections_compat import Iterator, Sequence, Mapping, MappingView, MutableMapping 50from ansible.module_utils.common.collections import is_sequence 51from ansible.module_utils.compat.importlib import import_module 52from ansible.plugins.loader import filter_loader, lookup_loader, test_loader 53from ansible.template.safe_eval import safe_eval 54from ansible.template.template import AnsibleJ2Template 55from ansible.template.vars import AnsibleJ2Vars 56from ansible.utils.collection_loader import AnsibleCollectionRef 57from ansible.utils.display import Display 58from ansible.utils.collection_loader._collection_finder import _get_collection_metadata 59from ansible.utils.unsafe_proxy import wrap_var 60 61display = Display() 62 63 64__all__ = ['Templar', 'generate_ansible_template_vars'] 65 66# A regex for checking to see if a variable we're trying to 67# expand is just a single variable name. 68 69# Primitive Types which we don't want Jinja to convert to strings. 70NON_TEMPLATED_TYPES = (bool, Number) 71 72JINJA2_OVERRIDE = '#jinja2:' 73 74from jinja2 import __version__ as j2_version 75 76USE_JINJA2_NATIVE = False 77if C.DEFAULT_JINJA2_NATIVE: 78 try: 79 from jinja2.nativetypes import NativeEnvironment as Environment 80 from ansible.template.native_helpers import ansible_native_concat as j2_concat 81 from ansible.template.native_helpers import NativeJinjaText 82 USE_JINJA2_NATIVE = True 83 except ImportError: 84 from jinja2 import Environment 85 from jinja2.utils import concat as j2_concat 86 display.warning( 87 'jinja2_native requires Jinja 2.10 and above. ' 88 'Version detected: %s. Falling back to default.' % j2_version 89 ) 90else: 91 from jinja2 import Environment 92 from jinja2.utils import concat as j2_concat 93 94 95JINJA2_BEGIN_TOKENS = frozenset(('variable_begin', 'block_begin', 'comment_begin', 'raw_begin')) 96JINJA2_END_TOKENS = frozenset(('variable_end', 'block_end', 'comment_end', 'raw_end')) 97 98 99RANGE_TYPE = type(range(0)) 100 101 102def generate_ansible_template_vars(path, fullpath=None, dest_path=None): 103 104 if fullpath is None: 105 b_path = to_bytes(path) 106 else: 107 b_path = to_bytes(fullpath) 108 109 try: 110 template_uid = pwd.getpwuid(os.stat(b_path).st_uid).pw_name 111 except (KeyError, TypeError): 112 template_uid = os.stat(b_path).st_uid 113 114 temp_vars = { 115 'template_host': to_text(os.uname()[1]), 116 'template_path': path, 117 'template_mtime': datetime.datetime.fromtimestamp(os.path.getmtime(b_path)), 118 'template_uid': to_text(template_uid), 119 'template_run_date': datetime.datetime.now(), 120 'template_destpath': to_native(dest_path) if dest_path else None, 121 } 122 123 if fullpath is None: 124 temp_vars['template_fullpath'] = os.path.abspath(path) 125 else: 126 temp_vars['template_fullpath'] = fullpath 127 128 managed_default = C.DEFAULT_MANAGED_STR 129 managed_str = managed_default.format( 130 host=temp_vars['template_host'], 131 uid=temp_vars['template_uid'], 132 file=temp_vars['template_path'], 133 ) 134 temp_vars['ansible_managed'] = to_text(time.strftime(to_native(managed_str), time.localtime(os.path.getmtime(b_path)))) 135 136 return temp_vars 137 138 139def _escape_backslashes(data, jinja_env): 140 """Double backslashes within jinja2 expressions 141 142 A user may enter something like this in a playbook:: 143 144 debug: 145 msg: "Test Case 1\\3; {{ test1_name | regex_replace('^(.*)_name$', '\\1')}}" 146 147 The string inside of the {{ gets interpreted multiple times First by yaml. 148 Then by python. And finally by jinja2 as part of it's variable. Because 149 it is processed by both python and jinja2, the backslash escaped 150 characters get unescaped twice. This means that we'd normally have to use 151 four backslashes to escape that. This is painful for playbook authors as 152 they have to remember different rules for inside vs outside of a jinja2 153 expression (The backslashes outside of the "{{ }}" only get processed by 154 yaml and python. So they only need to be escaped once). The following 155 code fixes this by automatically performing the extra quoting of 156 backslashes inside of a jinja2 expression. 157 158 """ 159 if '\\' in data and '{{' in data: 160 new_data = [] 161 d2 = jinja_env.preprocess(data) 162 in_var = False 163 164 for token in jinja_env.lex(d2): 165 if token[1] == 'variable_begin': 166 in_var = True 167 new_data.append(token[2]) 168 elif token[1] == 'variable_end': 169 in_var = False 170 new_data.append(token[2]) 171 elif in_var and token[1] == 'string': 172 # Double backslashes only if we're inside of a jinja2 variable 173 new_data.append(token[2].replace('\\', '\\\\')) 174 else: 175 new_data.append(token[2]) 176 177 data = ''.join(new_data) 178 179 return data 180 181 182def is_template(data, jinja_env): 183 """This function attempts to quickly detect whether a value is a jinja2 184 template. To do so, we look for the first 2 matching jinja2 tokens for 185 start and end delimiters. 186 """ 187 found = None 188 start = True 189 comment = False 190 d2 = jinja_env.preprocess(data) 191 192 # This wraps a lot of code, but this is due to lex returning a generator 193 # so we may get an exception at any part of the loop 194 try: 195 for token in jinja_env.lex(d2): 196 if token[1] in JINJA2_BEGIN_TOKENS: 197 if start and token[1] == 'comment_begin': 198 # Comments can wrap other token types 199 comment = True 200 start = False 201 # Example: variable_end -> variable 202 found = token[1].split('_')[0] 203 elif token[1] in JINJA2_END_TOKENS: 204 if token[1].split('_')[0] == found: 205 return True 206 elif comment: 207 continue 208 return False 209 except TemplateSyntaxError: 210 return False 211 212 return False 213 214 215def _count_newlines_from_end(in_str): 216 ''' 217 Counts the number of newlines at the end of a string. This is used during 218 the jinja2 templating to ensure the count matches the input, since some newlines 219 may be thrown away during the templating. 220 ''' 221 222 try: 223 i = len(in_str) 224 j = i - 1 225 while in_str[j] == '\n': 226 j -= 1 227 return i - 1 - j 228 except IndexError: 229 # Uncommon cases: zero length string and string containing only newlines 230 return i 231 232 233def recursive_check_defined(item): 234 from jinja2.runtime import Undefined 235 236 if isinstance(item, MutableMapping): 237 for key in item: 238 recursive_check_defined(item[key]) 239 elif isinstance(item, list): 240 for i in item: 241 recursive_check_defined(i) 242 else: 243 if isinstance(item, Undefined): 244 raise AnsibleFilterError("{0} is undefined".format(item)) 245 246 247def _is_rolled(value): 248 """Helper method to determine if something is an unrolled generator, 249 iterator, or similar object 250 """ 251 return ( 252 isinstance(value, Iterator) or 253 isinstance(value, MappingView) or 254 isinstance(value, RANGE_TYPE) 255 ) 256 257 258def _unroll_iterator(func): 259 """Wrapper function, that intercepts the result of a filter 260 and auto unrolls a generator, so that users are not required to 261 explicitly use ``|list`` to unroll. 262 """ 263 def wrapper(*args, **kwargs): 264 ret = func(*args, **kwargs) 265 if _is_rolled(ret): 266 return list(ret) 267 return ret 268 269 wrapper.__UNROLLED__ = True 270 return _update_wrapper(wrapper, func) 271 272 273def _update_wrapper(wrapper, func): 274 # This code is duplicated from ``functools.update_wrapper`` from Py3.7. 275 # ``functools.update_wrapper`` was failing when the func was ``functools.partial`` 276 for attr in ('__module__', '__name__', '__qualname__', '__doc__', '__annotations__'): 277 try: 278 value = getattr(func, attr) 279 except AttributeError: 280 pass 281 else: 282 setattr(wrapper, attr, value) 283 for attr in ('__dict__',): 284 getattr(wrapper, attr).update(getattr(func, attr, {})) 285 wrapper.__wrapped__ = func 286 return wrapper 287 288 289def _wrap_native_text(func): 290 """Wrapper function, that intercepts the result of a filter 291 and wraps it into NativeJinjaText which is then used 292 in ``ansible_native_concat`` to indicate that it is a text 293 which should not be passed into ``literal_eval``. 294 """ 295 def wrapper(*args, **kwargs): 296 ret = func(*args, **kwargs) 297 return NativeJinjaText(ret) 298 299 return _update_wrapper(wrapper, func) 300 301 302class AnsibleUndefined(StrictUndefined): 303 ''' 304 A custom Undefined class, which returns further Undefined objects on access, 305 rather than throwing an exception. 306 ''' 307 def __getattr__(self, name): 308 if name == '__UNSAFE__': 309 # AnsibleUndefined should never be assumed to be unsafe 310 # This prevents ``hasattr(val, '__UNSAFE__')`` from evaluating to ``True`` 311 raise AttributeError(name) 312 # Return original Undefined object to preserve the first failure context 313 return self 314 315 def __getitem__(self, key): 316 # Return original Undefined object to preserve the first failure context 317 return self 318 319 def __repr__(self): 320 return 'AnsibleUndefined' 321 322 def __contains__(self, item): 323 # Return original Undefined object to preserve the first failure context 324 return self 325 326 327class AnsibleContext(Context): 328 ''' 329 A custom context, which intercepts resolve() calls and sets a flag 330 internally if any variable lookup returns an AnsibleUnsafe value. This 331 flag is checked post-templating, and (when set) will result in the 332 final templated result being wrapped in AnsibleUnsafe. 333 ''' 334 def __init__(self, *args, **kwargs): 335 super(AnsibleContext, self).__init__(*args, **kwargs) 336 self.unsafe = False 337 338 def _is_unsafe(self, val): 339 ''' 340 Our helper function, which will also recursively check dict and 341 list entries due to the fact that they may be repr'd and contain 342 a key or value which contains jinja2 syntax and would otherwise 343 lose the AnsibleUnsafe value. 344 ''' 345 if isinstance(val, dict): 346 for key in val.keys(): 347 if self._is_unsafe(val[key]): 348 return True 349 elif isinstance(val, list): 350 for item in val: 351 if self._is_unsafe(item): 352 return True 353 elif getattr(val, '__UNSAFE__', False) is True: 354 return True 355 return False 356 357 def _update_unsafe(self, val): 358 if val is not None and not self.unsafe and self._is_unsafe(val): 359 self.unsafe = True 360 361 def resolve(self, key): 362 ''' 363 The intercepted resolve(), which uses the helper above to set the 364 internal flag whenever an unsafe variable value is returned. 365 ''' 366 val = super(AnsibleContext, self).resolve(key) 367 self._update_unsafe(val) 368 return val 369 370 def resolve_or_missing(self, key): 371 val = super(AnsibleContext, self).resolve_or_missing(key) 372 self._update_unsafe(val) 373 return val 374 375 def get_all(self): 376 """Return the complete context as a dict including the exported 377 variables. For optimizations reasons this might not return an 378 actual copy so be careful with using it. 379 380 This is to prevent from running ``AnsibleJ2Vars`` through dict(): 381 382 ``dict(self.parent, **self.vars)`` 383 384 In Ansible this means that ALL variables would be templated in the 385 process of re-creating the parent because ``AnsibleJ2Vars`` templates 386 each variable in its ``__getitem__`` method. Instead we re-create the 387 parent via ``AnsibleJ2Vars.add_locals`` that creates a new 388 ``AnsibleJ2Vars`` copy without templating each variable. 389 390 This will prevent unnecessarily templating unused variables in cases 391 like setting a local variable and passing it to {% include %} 392 in a template. 393 394 Also see ``AnsibleJ2Template``and 395 https://github.com/pallets/jinja/commit/d67f0fd4cc2a4af08f51f4466150d49da7798729 396 """ 397 if LooseVersion(j2_version) >= LooseVersion('2.9'): 398 if not self.vars: 399 return self.parent 400 if not self.parent: 401 return self.vars 402 403 if isinstance(self.parent, AnsibleJ2Vars): 404 return self.parent.add_locals(self.vars) 405 else: 406 # can this happen in Ansible? 407 return dict(self.parent, **self.vars) 408 409 410class JinjaPluginIntercept(MutableMapping): 411 def __init__(self, delegatee, pluginloader, *args, **kwargs): 412 super(JinjaPluginIntercept, self).__init__(*args, **kwargs) 413 self._delegatee = delegatee 414 self._pluginloader = pluginloader 415 416 if self._pluginloader.class_name == 'FilterModule': 417 self._method_map_name = 'filters' 418 self._dirname = 'filter' 419 elif self._pluginloader.class_name == 'TestModule': 420 self._method_map_name = 'tests' 421 self._dirname = 'test' 422 423 self._collection_jinja_func_cache = {} 424 425 # FUTURE: we can cache FQ filter/test calls for the entire duration of a run, since a given collection's impl's 426 # aren't supposed to change during a run 427 def __getitem__(self, key): 428 try: 429 if not isinstance(key, string_types): 430 raise ValueError('key must be a string') 431 432 key = to_native(key) 433 434 if '.' not in key: # might be a built-in or legacy, check the delegatee dict first, then try for a last-chance base redirect 435 func = self._delegatee.get(key) 436 437 if func: 438 return func 439 440 # didn't find it in the pre-built Jinja env, assume it's a former builtin and follow the normal routing path 441 leaf_key = key 442 key = 'ansible.builtin.' + key 443 else: 444 leaf_key = key.split('.')[-1] 445 446 acr = AnsibleCollectionRef.try_parse_fqcr(key, self._dirname) 447 448 if not acr: 449 raise KeyError('invalid plugin name: {0}'.format(key)) 450 451 ts = _get_collection_metadata(acr.collection) 452 453 # TODO: implement support for collection-backed redirect (currently only builtin) 454 # TODO: implement cycle detection (unified across collection redir as well) 455 456 routing_entry = ts.get('plugin_routing', {}).get(self._dirname, {}).get(leaf_key, {}) 457 458 deprecation_entry = routing_entry.get('deprecation') 459 if deprecation_entry: 460 warning_text = deprecation_entry.get('warning_text') 461 removal_date = deprecation_entry.get('removal_date') 462 removal_version = deprecation_entry.get('removal_version') 463 464 if not warning_text: 465 warning_text = '{0} "{1}" is deprecated'.format(self._dirname, key) 466 467 display.deprecated(warning_text, version=removal_version, date=removal_date, collection_name=acr.collection) 468 469 tombstone_entry = routing_entry.get('tombstone') 470 471 if tombstone_entry: 472 warning_text = tombstone_entry.get('warning_text') 473 removal_date = tombstone_entry.get('removal_date') 474 removal_version = tombstone_entry.get('removal_version') 475 476 if not warning_text: 477 warning_text = '{0} "{1}" has been removed'.format(self._dirname, key) 478 479 exc_msg = display.get_deprecation_message(warning_text, version=removal_version, date=removal_date, 480 collection_name=acr.collection, removed=True) 481 482 raise AnsiblePluginRemovedError(exc_msg) 483 484 redirect_fqcr = routing_entry.get('redirect', None) 485 if redirect_fqcr: 486 acr = AnsibleCollectionRef.from_fqcr(ref=redirect_fqcr, ref_type=self._dirname) 487 display.vvv('redirecting {0} {1} to {2}.{3}'.format(self._dirname, key, acr.collection, acr.resource)) 488 key = redirect_fqcr 489 # TODO: handle recursive forwarding (not necessary for builtin, but definitely for further collection redirs) 490 491 func = self._collection_jinja_func_cache.get(key) 492 493 if func: 494 return func 495 496 try: 497 pkg = import_module(acr.n_python_package_name) 498 except ImportError: 499 raise KeyError() 500 501 parent_prefix = acr.collection 502 503 if acr.subdirs: 504 parent_prefix = '{0}.{1}'.format(parent_prefix, acr.subdirs) 505 506 # TODO: implement collection-level redirect 507 508 for dummy, module_name, ispkg in pkgutil.iter_modules(pkg.__path__, prefix=parent_prefix + '.'): 509 if ispkg: 510 continue 511 512 try: 513 plugin_impl = self._pluginloader.get(module_name) 514 except Exception as e: 515 raise TemplateSyntaxError(to_native(e), 0) 516 517 method_map = getattr(plugin_impl, self._method_map_name) 518 519 for func_name, func in iteritems(method_map()): 520 fq_name = '.'.join((parent_prefix, func_name)) 521 # FIXME: detect/warn on intra-collection function name collisions 522 if USE_JINJA2_NATIVE and fq_name.startswith(('ansible.builtin.', 'ansible.legacy.')) and \ 523 func_name in C.STRING_TYPE_FILTERS: 524 self._collection_jinja_func_cache[fq_name] = _wrap_native_text(func) 525 else: 526 self._collection_jinja_func_cache[fq_name] = _unroll_iterator(func) 527 528 function_impl = self._collection_jinja_func_cache[key] 529 return function_impl 530 except AnsiblePluginRemovedError as apre: 531 raise TemplateSyntaxError(to_native(apre), 0) 532 except KeyError: 533 raise 534 except Exception as ex: 535 display.warning('an unexpected error occurred during Jinja2 environment setup: {0}'.format(to_native(ex))) 536 display.vvv('exception during Jinja2 environment setup: {0}'.format(format_exc())) 537 raise TemplateSyntaxError(to_native(ex), 0) 538 539 def __setitem__(self, key, value): 540 return self._delegatee.__setitem__(key, value) 541 542 def __delitem__(self, key): 543 raise NotImplementedError() 544 545 def __iter__(self): 546 # not strictly accurate since we're not counting dynamically-loaded values 547 return iter(self._delegatee) 548 549 def __len__(self): 550 # not strictly accurate since we're not counting dynamically-loaded values 551 return len(self._delegatee) 552 553 554class AnsibleEnvironment(Environment): 555 ''' 556 Our custom environment, which simply allows us to override the class-level 557 values for the Template and Context classes used by jinja2 internally. 558 ''' 559 context_class = AnsibleContext 560 template_class = AnsibleJ2Template 561 562 def __init__(self, *args, **kwargs): 563 super(AnsibleEnvironment, self).__init__(*args, **kwargs) 564 565 self.filters = JinjaPluginIntercept(self.filters, filter_loader) 566 self.tests = JinjaPluginIntercept(self.tests, test_loader) 567 568 569class Templar: 570 ''' 571 The main class for templating, with the main entry-point of template(). 572 ''' 573 574 def __init__(self, loader, shared_loader_obj=None, variables=None): 575 variables = {} if variables is None else variables 576 577 self._loader = loader 578 self._filters = None 579 self._tests = None 580 self._available_variables = variables 581 self._cached_result = {} 582 583 if loader: 584 self._basedir = loader.get_basedir() 585 else: 586 self._basedir = './' 587 588 if shared_loader_obj: 589 self._filter_loader = getattr(shared_loader_obj, 'filter_loader') 590 self._test_loader = getattr(shared_loader_obj, 'test_loader') 591 self._lookup_loader = getattr(shared_loader_obj, 'lookup_loader') 592 else: 593 self._filter_loader = filter_loader 594 self._test_loader = test_loader 595 self._lookup_loader = lookup_loader 596 597 # flags to determine whether certain failures during templating 598 # should result in fatal errors being raised 599 self._fail_on_lookup_errors = True 600 self._fail_on_filter_errors = True 601 self._fail_on_undefined_errors = C.DEFAULT_UNDEFINED_VAR_BEHAVIOR 602 603 self.environment = AnsibleEnvironment( 604 trim_blocks=True, 605 undefined=AnsibleUndefined, 606 extensions=self._get_extensions(), 607 finalize=self._finalize, 608 loader=FileSystemLoader(self._basedir), 609 ) 610 611 # jinja2 global is inconsistent across versions, this normalizes them 612 self.environment.globals['dict'] = dict 613 614 # Custom globals 615 self.environment.globals['lookup'] = self._lookup 616 self.environment.globals['query'] = self.environment.globals['q'] = self._query_lookup 617 self.environment.globals['now'] = self._now_datetime 618 self.environment.globals['finalize'] = self._finalize 619 620 # the current rendering context under which the templar class is working 621 self.cur_context = None 622 623 self.SINGLE_VAR = re.compile(r"^%s\s*(\w*)\s*%s$" % (self.environment.variable_start_string, self.environment.variable_end_string)) 624 625 self._clean_regex = re.compile(r'(?:%s|%s|%s|%s)' % ( 626 self.environment.variable_start_string, 627 self.environment.block_start_string, 628 self.environment.block_end_string, 629 self.environment.variable_end_string 630 )) 631 self._no_type_regex = re.compile(r'.*?\|\s*(?:%s)(?:\([^\|]*\))?\s*\)?\s*(?:%s)' % 632 ('|'.join(C.STRING_TYPE_FILTERS), self.environment.variable_end_string)) 633 634 def _get_filters(self): 635 ''' 636 Returns filter plugins, after loading and caching them if need be 637 ''' 638 639 if self._filters is not None: 640 return self._filters.copy() 641 642 self._filters = dict() 643 644 for fp in self._filter_loader.all(): 645 self._filters.update(fp.filters()) 646 647 if USE_JINJA2_NATIVE: 648 for string_filter in C.STRING_TYPE_FILTERS: 649 try: 650 orig_filter = self._filters[string_filter] 651 except KeyError: 652 try: 653 orig_filter = self.environment.filters[string_filter] 654 except KeyError: 655 continue 656 self._filters[string_filter] = _wrap_native_text(orig_filter) 657 658 return self._filters.copy() 659 660 def _get_tests(self): 661 ''' 662 Returns tests plugins, after loading and caching them if need be 663 ''' 664 665 if self._tests is not None: 666 return self._tests.copy() 667 668 self._tests = dict() 669 for fp in self._test_loader.all(): 670 self._tests.update(fp.tests()) 671 672 return self._tests.copy() 673 674 def _get_extensions(self): 675 ''' 676 Return jinja2 extensions to load. 677 678 If some extensions are set via jinja_extensions in ansible.cfg, we try 679 to load them with the jinja environment. 680 ''' 681 682 jinja_exts = [] 683 if C.DEFAULT_JINJA2_EXTENSIONS: 684 # make sure the configuration directive doesn't contain spaces 685 # and split extensions in an array 686 jinja_exts = C.DEFAULT_JINJA2_EXTENSIONS.replace(" ", "").split(',') 687 688 return jinja_exts 689 690 @property 691 def available_variables(self): 692 return self._available_variables 693 694 @available_variables.setter 695 def available_variables(self, variables): 696 ''' 697 Sets the list of template variables this Templar instance will use 698 to template things, so we don't have to pass them around between 699 internal methods. We also clear the template cache here, as the variables 700 are being changed. 701 ''' 702 703 if not isinstance(variables, Mapping): 704 raise AnsibleAssertionError("the type of 'variables' should be a Mapping but was a %s" % (type(variables))) 705 self._available_variables = variables 706 self._cached_result = {} 707 708 def set_available_variables(self, variables): 709 display.deprecated( 710 'set_available_variables is being deprecated. Use "@available_variables.setter" instead.', 711 version='2.13', collection_name='ansible.builtin' 712 ) 713 self.available_variables = variables 714 715 @contextmanager 716 def set_temporary_context(self, **kwargs): 717 """Context manager used to set temporary templating context, without having to worry about resetting 718 original values afterward 719 720 Use a keyword that maps to the attr you are setting. Applies to ``self.environment`` by default, to 721 set context on another object, it must be in ``mapping``. 722 """ 723 mapping = { 724 'available_variables': self, 725 'searchpath': self.environment.loader, 726 } 727 original = {} 728 729 for key, value in kwargs.items(): 730 obj = mapping.get(key, self.environment) 731 try: 732 original[key] = getattr(obj, key) 733 if value is not None: 734 setattr(obj, key, value) 735 except AttributeError: 736 # Ignore invalid attrs, lstrip_blocks was added in jinja2==2.7 737 pass 738 739 yield 740 741 for key in original: 742 obj = mapping.get(key, self.environment) 743 setattr(obj, key, original[key]) 744 745 def template(self, variable, convert_bare=False, preserve_trailing_newlines=True, escape_backslashes=True, fail_on_undefined=None, overrides=None, 746 convert_data=True, static_vars=None, cache=True, disable_lookups=False): 747 ''' 748 Templates (possibly recursively) any given data as input. If convert_bare is 749 set to True, the given data will be wrapped as a jinja2 variable ('{{foo}}') 750 before being sent through the template engine. 751 ''' 752 static_vars = [''] if static_vars is None else static_vars 753 754 # Don't template unsafe variables, just return them. 755 if hasattr(variable, '__UNSAFE__'): 756 return variable 757 758 if fail_on_undefined is None: 759 fail_on_undefined = self._fail_on_undefined_errors 760 761 try: 762 if convert_bare: 763 variable = self._convert_bare_variable(variable) 764 765 if isinstance(variable, string_types): 766 result = variable 767 768 if self.is_possibly_template(variable): 769 # Check to see if the string we are trying to render is just referencing a single 770 # var. In this case we don't want to accidentally change the type of the variable 771 # to a string by using the jinja template renderer. We just want to pass it. 772 only_one = self.SINGLE_VAR.match(variable) 773 if only_one: 774 var_name = only_one.group(1) 775 if var_name in self._available_variables: 776 resolved_val = self._available_variables[var_name] 777 if isinstance(resolved_val, NON_TEMPLATED_TYPES): 778 return resolved_val 779 elif resolved_val is None: 780 return C.DEFAULT_NULL_REPRESENTATION 781 782 # Using a cache in order to prevent template calls with already templated variables 783 sha1_hash = None 784 if cache: 785 variable_hash = sha1(text_type(variable).encode('utf-8')) 786 options_hash = sha1( 787 ( 788 text_type(preserve_trailing_newlines) + 789 text_type(escape_backslashes) + 790 text_type(fail_on_undefined) + 791 text_type(overrides) 792 ).encode('utf-8') 793 ) 794 sha1_hash = variable_hash.hexdigest() + options_hash.hexdigest() 795 if cache and sha1_hash in self._cached_result: 796 result = self._cached_result[sha1_hash] 797 else: 798 result = self.do_template( 799 variable, 800 preserve_trailing_newlines=preserve_trailing_newlines, 801 escape_backslashes=escape_backslashes, 802 fail_on_undefined=fail_on_undefined, 803 overrides=overrides, 804 disable_lookups=disable_lookups, 805 ) 806 807 if not USE_JINJA2_NATIVE: 808 unsafe = hasattr(result, '__UNSAFE__') 809 if convert_data and not self._no_type_regex.match(variable): 810 # if this looks like a dictionary or list, convert it to such using the safe_eval method 811 if (result.startswith("{") and not result.startswith(self.environment.variable_start_string)) or \ 812 result.startswith("[") or result in ("True", "False"): 813 eval_results = safe_eval(result, include_exceptions=True) 814 if eval_results[1] is None: 815 result = eval_results[0] 816 if unsafe: 817 result = wrap_var(result) 818 else: 819 # FIXME: if the safe_eval raised an error, should we do something with it? 820 pass 821 822 # we only cache in the case where we have a single variable 823 # name, to make sure we're not putting things which may otherwise 824 # be dynamic in the cache (filters, lookups, etc.) 825 if cache and only_one: 826 self._cached_result[sha1_hash] = result 827 828 return result 829 830 elif is_sequence(variable): 831 return [self.template( 832 v, 833 preserve_trailing_newlines=preserve_trailing_newlines, 834 fail_on_undefined=fail_on_undefined, 835 overrides=overrides, 836 disable_lookups=disable_lookups, 837 ) for v in variable] 838 elif isinstance(variable, Mapping): 839 d = {} 840 # we don't use iteritems() here to avoid problems if the underlying dict 841 # changes sizes due to the templating, which can happen with hostvars 842 for k in variable.keys(): 843 if k not in static_vars: 844 d[k] = self.template( 845 variable[k], 846 preserve_trailing_newlines=preserve_trailing_newlines, 847 fail_on_undefined=fail_on_undefined, 848 overrides=overrides, 849 disable_lookups=disable_lookups, 850 ) 851 else: 852 d[k] = variable[k] 853 return d 854 else: 855 return variable 856 857 except AnsibleFilterError: 858 if self._fail_on_filter_errors: 859 raise 860 else: 861 return variable 862 863 def is_template(self, data): 864 '''lets us know if data has a template''' 865 if isinstance(data, string_types): 866 return is_template(data, self.environment) 867 elif isinstance(data, (list, tuple)): 868 for v in data: 869 if self.is_template(v): 870 return True 871 elif isinstance(data, dict): 872 for k in data: 873 if self.is_template(k) or self.is_template(data[k]): 874 return True 875 return False 876 877 templatable = is_template 878 879 def is_possibly_template(self, data): 880 '''Determines if a string looks like a template, by seeing if it 881 contains a jinja2 start delimiter. Does not guarantee that the string 882 is actually a template. 883 884 This is different than ``is_template`` which is more strict. 885 This method may return ``True`` on a string that is not templatable. 886 887 Useful when guarding passing a string for templating, but when 888 you want to allow the templating engine to make the final 889 assessment which may result in ``TemplateSyntaxError``. 890 ''' 891 env = self.environment 892 if isinstance(data, string_types): 893 for marker in (env.block_start_string, env.variable_start_string, env.comment_start_string): 894 if marker in data: 895 return True 896 return False 897 898 def _convert_bare_variable(self, variable): 899 ''' 900 Wraps a bare string, which may have an attribute portion (ie. foo.bar) 901 in jinja2 variable braces so that it is evaluated properly. 902 ''' 903 904 if isinstance(variable, string_types): 905 contains_filters = "|" in variable 906 first_part = variable.split("|")[0].split(".")[0].split("[")[0] 907 if (contains_filters or first_part in self._available_variables) and self.environment.variable_start_string not in variable: 908 return "%s%s%s" % (self.environment.variable_start_string, variable, self.environment.variable_end_string) 909 910 # the variable didn't meet the conditions to be converted, 911 # so just return it as-is 912 return variable 913 914 def _finalize(self, thing): 915 ''' 916 A custom finalize method for jinja2, which prevents None from being returned. This 917 avoids a string of ``"None"`` as ``None`` has no importance in YAML. 918 919 If using ANSIBLE_JINJA2_NATIVE we bypass this and return the actual value always 920 ''' 921 if _is_rolled(thing): 922 # Auto unroll a generator, so that users are not required to 923 # explicitly use ``|list`` to unroll 924 # This only affects the scenario where the final result of templating 925 # is a generator, and not where a filter creates a generator in the middle 926 # of a template. See ``_unroll_iterator`` for the other case. This is probably 927 # unncessary 928 return list(thing) 929 930 if USE_JINJA2_NATIVE: 931 return thing 932 933 return thing if thing is not None else '' 934 935 def _fail_lookup(self, name, *args, **kwargs): 936 raise AnsibleError("The lookup `%s` was found, however lookups were disabled from templating" % name) 937 938 def _now_datetime(self, utc=False, fmt=None): 939 '''jinja2 global function to return current datetime, potentially formatted via strftime''' 940 if utc: 941 now = datetime.datetime.utcnow() 942 else: 943 now = datetime.datetime.now() 944 945 if fmt: 946 return now.strftime(fmt) 947 948 return now 949 950 def _query_lookup(self, name, *args, **kwargs): 951 ''' wrapper for lookup, force wantlist true''' 952 kwargs['wantlist'] = True 953 return self._lookup(name, *args, **kwargs) 954 955 def _lookup(self, name, *args, **kwargs): 956 instance = self._lookup_loader.get(name, loader=self._loader, templar=self) 957 958 if instance is not None: 959 wantlist = kwargs.pop('wantlist', False) 960 allow_unsafe = kwargs.pop('allow_unsafe', C.DEFAULT_ALLOW_UNSAFE_LOOKUPS) 961 errors = kwargs.pop('errors', 'strict') 962 963 from ansible.utils.listify import listify_lookup_plugin_terms 964 loop_terms = listify_lookup_plugin_terms(terms=args, templar=self, loader=self._loader, fail_on_undefined=True, convert_bare=False) 965 # safely catch run failures per #5059 966 try: 967 ran = instance.run(loop_terms, variables=self._available_variables, **kwargs) 968 except (AnsibleUndefinedVariable, UndefinedError) as e: 969 raise AnsibleUndefinedVariable(e) 970 except Exception as e: 971 if self._fail_on_lookup_errors: 972 msg = u"An unhandled exception occurred while running the lookup plugin '%s'. Error was a %s, original message: %s" % \ 973 (name, type(e), to_text(e)) 974 if errors == 'warn': 975 display.warning(msg) 976 elif errors == 'ignore': 977 display.display(msg, log_only=True) 978 else: 979 raise AnsibleError(to_native(msg)) 980 ran = [] if wantlist else None 981 982 if ran and not allow_unsafe: 983 if wantlist: 984 ran = wrap_var(ran) 985 else: 986 try: 987 ran = wrap_var(",".join(ran)) 988 except TypeError: 989 # Lookup Plugins should always return lists. Throw an error if that's not 990 # the case: 991 if not isinstance(ran, Sequence): 992 raise AnsibleError("The lookup plugin '%s' did not return a list." 993 % name) 994 995 # The TypeError we can recover from is when the value *inside* of the list 996 # is not a string 997 if len(ran) == 1: 998 ran = wrap_var(ran[0]) 999 else: 1000 ran = wrap_var(ran) 1001 1002 if self.cur_context: 1003 self.cur_context.unsafe = True 1004 return ran 1005 else: 1006 raise AnsibleError("lookup plugin (%s) not found" % name) 1007 1008 def do_template(self, data, preserve_trailing_newlines=True, escape_backslashes=True, fail_on_undefined=None, overrides=None, disable_lookups=False): 1009 if USE_JINJA2_NATIVE and not isinstance(data, string_types): 1010 return data 1011 1012 # For preserving the number of input newlines in the output (used 1013 # later in this method) 1014 data_newlines = _count_newlines_from_end(data) 1015 1016 if fail_on_undefined is None: 1017 fail_on_undefined = self._fail_on_undefined_errors 1018 1019 try: 1020 # allows template header overrides to change jinja2 options. 1021 if overrides is None: 1022 myenv = self.environment.overlay() 1023 else: 1024 myenv = self.environment.overlay(overrides) 1025 1026 # Get jinja env overrides from template 1027 if hasattr(data, 'startswith') and data.startswith(JINJA2_OVERRIDE): 1028 eol = data.find('\n') 1029 line = data[len(JINJA2_OVERRIDE):eol] 1030 data = data[eol + 1:] 1031 for pair in line.split(','): 1032 (key, val) = pair.split(':') 1033 key = key.strip() 1034 setattr(myenv, key, ast.literal_eval(val.strip())) 1035 1036 # Adds Ansible custom filters and tests 1037 myenv.filters.update(self._get_filters()) 1038 for k in myenv.filters: 1039 if not getattr(myenv.filters[k], '__UNROLLED__', False): 1040 myenv.filters[k] = _unroll_iterator(myenv.filters[k]) 1041 myenv.tests.update(self._get_tests()) 1042 1043 if escape_backslashes: 1044 # Allow users to specify backslashes in playbooks as "\\" instead of as "\\\\". 1045 data = _escape_backslashes(data, myenv) 1046 1047 try: 1048 t = myenv.from_string(data) 1049 except TemplateSyntaxError as e: 1050 raise AnsibleError("template error while templating string: %s. String: %s" % (to_native(e), to_native(data))) 1051 except Exception as e: 1052 if 'recursion' in to_native(e): 1053 raise AnsibleError("recursive loop detected in template string: %s" % to_native(data)) 1054 else: 1055 return data 1056 1057 if disable_lookups: 1058 t.globals['query'] = t.globals['q'] = t.globals['lookup'] = self._fail_lookup 1059 1060 jvars = AnsibleJ2Vars(self, t.globals) 1061 1062 self.cur_context = new_context = t.new_context(jvars, shared=True) 1063 rf = t.root_render_func(new_context) 1064 1065 try: 1066 res = j2_concat(rf) 1067 unsafe = getattr(new_context, 'unsafe', False) 1068 if unsafe: 1069 res = wrap_var(res) 1070 except TypeError as te: 1071 if 'AnsibleUndefined' in to_native(te): 1072 errmsg = "Unable to look up a name or access an attribute in template string (%s).\n" % to_native(data) 1073 errmsg += "Make sure your variable name does not contain invalid characters like '-': %s" % to_native(te) 1074 raise AnsibleUndefinedVariable(errmsg) 1075 else: 1076 display.debug("failing because of a type error, template data is: %s" % to_text(data)) 1077 raise AnsibleError("Unexpected templating type error occurred on (%s): %s" % (to_native(data), to_native(te))) 1078 1079 if USE_JINJA2_NATIVE and not isinstance(res, string_types): 1080 return res 1081 1082 if preserve_trailing_newlines: 1083 # The low level calls above do not preserve the newline 1084 # characters at the end of the input data, so we use the 1085 # calculate the difference in newlines and append them 1086 # to the resulting output for parity 1087 # 1088 # jinja2 added a keep_trailing_newline option in 2.7 when 1089 # creating an Environment. That would let us make this code 1090 # better (remove a single newline if 1091 # preserve_trailing_newlines is False). Once we can depend on 1092 # that version being present, modify our code to set that when 1093 # initializing self.environment and remove a single trailing 1094 # newline here if preserve_newlines is False. 1095 res_newlines = _count_newlines_from_end(res) 1096 if data_newlines > res_newlines: 1097 res += self.environment.newline_sequence * (data_newlines - res_newlines) 1098 if unsafe: 1099 res = wrap_var(res) 1100 return res 1101 except (UndefinedError, AnsibleUndefinedVariable) as e: 1102 if fail_on_undefined: 1103 raise AnsibleUndefinedVariable(e) 1104 else: 1105 display.debug("Ignoring undefined failure: %s" % to_text(e)) 1106 return data 1107 1108 # for backwards compatibility in case anyone is using old private method directly 1109 _do_template = do_template 1110