1# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> 2# 3# This file is part of Ansible 4# 5# Ansible is free software: you can redistribute it and/or modify 6# it under the terms of the GNU General Public License as published by 7# the Free Software Foundation, either version 3 of the License, or 8# (at your option) any later version. 9# 10# Ansible is distributed in the hope that it will be useful, 11# but WITHOUT ANY WARRANTY; without even the implied warranty of 12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13# GNU General Public License for more details. 14# 15# You should have received a copy of the GNU General Public License 16# along with Ansible. If not, see <http://www.gnu.org/licenses/>. 17 18# Make coding more python3-ish 19from __future__ import (absolute_import, division, print_function) 20__metaclass__ = type 21 22import ast 23import datetime 24import os 25import pkgutil 26import pwd 27import re 28import time 29 30from contextlib import contextmanager 31from distutils.version import LooseVersion 32from numbers import Number 33from traceback import format_exc 34 35try: 36 from hashlib import sha1 37except ImportError: 38 from sha import sha as sha1 39 40from jinja2.exceptions import TemplateSyntaxError, UndefinedError 41from jinja2.loaders import FileSystemLoader 42from jinja2.runtime import Context, StrictUndefined 43 44from ansible import constants as C 45from ansible.errors import AnsibleError, AnsibleFilterError, AnsiblePluginRemovedError, AnsibleUndefinedVariable, AnsibleAssertionError 46from ansible.module_utils.six import iteritems, string_types, text_type 47from ansible.module_utils.six.moves import range 48from ansible.module_utils._text import to_native, to_text, to_bytes 49from ansible.module_utils.common._collections_compat import Iterator, Sequence, Mapping, MappingView, MutableMapping 50from ansible.module_utils.common.collections import is_sequence 51from ansible.module_utils.compat.importlib import import_module 52from ansible.plugins.loader import filter_loader, lookup_loader, test_loader 53from ansible.template.safe_eval import safe_eval 54from ansible.template.template import AnsibleJ2Template 55from ansible.template.vars import AnsibleJ2Vars 56from ansible.utils.collection_loader import AnsibleCollectionRef 57from ansible.utils.display import Display 58from ansible.utils.collection_loader._collection_finder import _get_collection_metadata 59from ansible.utils.listify import listify_lookup_plugin_terms 60from ansible.utils.unsafe_proxy import wrap_var 61 62display = Display() 63 64 65__all__ = ['Templar', 'generate_ansible_template_vars'] 66 67# A regex for checking to see if a variable we're trying to 68# expand is just a single variable name. 69 70# Primitive Types which we don't want Jinja to convert to strings. 71NON_TEMPLATED_TYPES = (bool, Number) 72 73JINJA2_OVERRIDE = '#jinja2:' 74 75from jinja2 import __version__ as j2_version 76from jinja2 import Environment 77from jinja2.utils import concat as j2_concat 78 79 80USE_JINJA2_NATIVE = False 81if C.DEFAULT_JINJA2_NATIVE: 82 try: 83 from jinja2.nativetypes import NativeEnvironment 84 from ansible.template.native_helpers import ansible_native_concat 85 from ansible.utils.native_jinja import NativeJinjaText 86 USE_JINJA2_NATIVE = True 87 except ImportError: 88 from jinja2 import Environment 89 from jinja2.utils import concat as j2_concat 90 display.warning( 91 'jinja2_native requires Jinja 2.10 and above. ' 92 'Version detected: %s. Falling back to default.' % j2_version 93 ) 94 95 96JINJA2_BEGIN_TOKENS = frozenset(('variable_begin', 'block_begin', 'comment_begin', 'raw_begin')) 97JINJA2_END_TOKENS = frozenset(('variable_end', 'block_end', 'comment_end', 'raw_end')) 98 99 100RANGE_TYPE = type(range(0)) 101 102 103def generate_ansible_template_vars(path, fullpath=None, dest_path=None): 104 105 if fullpath is None: 106 b_path = to_bytes(path) 107 else: 108 b_path = to_bytes(fullpath) 109 110 try: 111 template_uid = pwd.getpwuid(os.stat(b_path).st_uid).pw_name 112 except (KeyError, TypeError): 113 template_uid = os.stat(b_path).st_uid 114 115 temp_vars = { 116 'template_host': to_text(os.uname()[1]), 117 'template_path': path, 118 'template_mtime': datetime.datetime.fromtimestamp(os.path.getmtime(b_path)), 119 'template_uid': to_text(template_uid), 120 'template_run_date': datetime.datetime.now(), 121 'template_destpath': to_native(dest_path) if dest_path else None, 122 } 123 124 if fullpath is None: 125 temp_vars['template_fullpath'] = os.path.abspath(path) 126 else: 127 temp_vars['template_fullpath'] = fullpath 128 129 managed_default = C.DEFAULT_MANAGED_STR 130 managed_str = managed_default.format( 131 host=temp_vars['template_host'], 132 uid=temp_vars['template_uid'], 133 file=temp_vars['template_path'], 134 ) 135 temp_vars['ansible_managed'] = to_text(time.strftime(to_native(managed_str), time.localtime(os.path.getmtime(b_path)))) 136 137 return temp_vars 138 139 140def _escape_backslashes(data, jinja_env): 141 """Double backslashes within jinja2 expressions 142 143 A user may enter something like this in a playbook:: 144 145 debug: 146 msg: "Test Case 1\\3; {{ test1_name | regex_replace('^(.*)_name$', '\\1')}}" 147 148 The string inside of the {{ gets interpreted multiple times First by yaml. 149 Then by python. And finally by jinja2 as part of it's variable. Because 150 it is processed by both python and jinja2, the backslash escaped 151 characters get unescaped twice. This means that we'd normally have to use 152 four backslashes to escape that. This is painful for playbook authors as 153 they have to remember different rules for inside vs outside of a jinja2 154 expression (The backslashes outside of the "{{ }}" only get processed by 155 yaml and python. So they only need to be escaped once). The following 156 code fixes this by automatically performing the extra quoting of 157 backslashes inside of a jinja2 expression. 158 159 """ 160 if '\\' in data and '{{' in data: 161 new_data = [] 162 d2 = jinja_env.preprocess(data) 163 in_var = False 164 165 for token in jinja_env.lex(d2): 166 if token[1] == 'variable_begin': 167 in_var = True 168 new_data.append(token[2]) 169 elif token[1] == 'variable_end': 170 in_var = False 171 new_data.append(token[2]) 172 elif in_var and token[1] == 'string': 173 # Double backslashes only if we're inside of a jinja2 variable 174 new_data.append(token[2].replace('\\', '\\\\')) 175 else: 176 new_data.append(token[2]) 177 178 data = ''.join(new_data) 179 180 return data 181 182 183def is_template(data, jinja_env): 184 """This function attempts to quickly detect whether a value is a jinja2 185 template. To do so, we look for the first 2 matching jinja2 tokens for 186 start and end delimiters. 187 """ 188 found = None 189 start = True 190 comment = False 191 d2 = jinja_env.preprocess(data) 192 193 # This wraps a lot of code, but this is due to lex returning a generator 194 # so we may get an exception at any part of the loop 195 try: 196 for token in jinja_env.lex(d2): 197 if token[1] in JINJA2_BEGIN_TOKENS: 198 if start and token[1] == 'comment_begin': 199 # Comments can wrap other token types 200 comment = True 201 start = False 202 # Example: variable_end -> variable 203 found = token[1].split('_')[0] 204 elif token[1] in JINJA2_END_TOKENS: 205 if token[1].split('_')[0] == found: 206 return True 207 elif comment: 208 continue 209 return False 210 except TemplateSyntaxError: 211 return False 212 213 return False 214 215 216def _count_newlines_from_end(in_str): 217 ''' 218 Counts the number of newlines at the end of a string. This is used during 219 the jinja2 templating to ensure the count matches the input, since some newlines 220 may be thrown away during the templating. 221 ''' 222 223 try: 224 i = len(in_str) 225 j = i - 1 226 while in_str[j] == '\n': 227 j -= 1 228 return i - 1 - j 229 except IndexError: 230 # Uncommon cases: zero length string and string containing only newlines 231 return i 232 233 234def recursive_check_defined(item): 235 from jinja2.runtime import Undefined 236 237 if isinstance(item, MutableMapping): 238 for key in item: 239 recursive_check_defined(item[key]) 240 elif isinstance(item, list): 241 for i in item: 242 recursive_check_defined(i) 243 else: 244 if isinstance(item, Undefined): 245 raise AnsibleFilterError("{0} is undefined".format(item)) 246 247 248def _is_rolled(value): 249 """Helper method to determine if something is an unrolled generator, 250 iterator, or similar object 251 """ 252 return ( 253 isinstance(value, Iterator) or 254 isinstance(value, MappingView) or 255 isinstance(value, RANGE_TYPE) 256 ) 257 258 259def _unroll_iterator(func): 260 """Wrapper function, that intercepts the result of a filter 261 and auto unrolls a generator, so that users are not required to 262 explicitly use ``|list`` to unroll. 263 """ 264 def wrapper(*args, **kwargs): 265 ret = func(*args, **kwargs) 266 if _is_rolled(ret): 267 return list(ret) 268 return ret 269 270 return _update_wrapper(wrapper, func) 271 272 273def _update_wrapper(wrapper, func): 274 # This code is duplicated from ``functools.update_wrapper`` from Py3.7. 275 # ``functools.update_wrapper`` was failing when the func was ``functools.partial`` 276 for attr in ('__module__', '__name__', '__qualname__', '__doc__', '__annotations__'): 277 try: 278 value = getattr(func, attr) 279 except AttributeError: 280 pass 281 else: 282 setattr(wrapper, attr, value) 283 for attr in ('__dict__',): 284 getattr(wrapper, attr).update(getattr(func, attr, {})) 285 wrapper.__wrapped__ = func 286 return wrapper 287 288 289def _wrap_native_text(func): 290 """Wrapper function, that intercepts the result of a filter 291 and wraps it into NativeJinjaText which is then used 292 in ``ansible_native_concat`` to indicate that it is a text 293 which should not be passed into ``literal_eval``. 294 """ 295 def wrapper(*args, **kwargs): 296 ret = func(*args, **kwargs) 297 return NativeJinjaText(ret) 298 299 return _update_wrapper(wrapper, func) 300 301 302class AnsibleUndefined(StrictUndefined): 303 ''' 304 A custom Undefined class, which returns further Undefined objects on access, 305 rather than throwing an exception. 306 ''' 307 def __getattr__(self, name): 308 if name == '__UNSAFE__': 309 # AnsibleUndefined should never be assumed to be unsafe 310 # This prevents ``hasattr(val, '__UNSAFE__')`` from evaluating to ``True`` 311 raise AttributeError(name) 312 # Return original Undefined object to preserve the first failure context 313 return self 314 315 def __getitem__(self, key): 316 # Return original Undefined object to preserve the first failure context 317 return self 318 319 def __repr__(self): 320 return 'AnsibleUndefined' 321 322 def __contains__(self, item): 323 # Return original Undefined object to preserve the first failure context 324 return self 325 326 327class AnsibleContext(Context): 328 ''' 329 A custom context, which intercepts resolve() calls and sets a flag 330 internally if any variable lookup returns an AnsibleUnsafe value. This 331 flag is checked post-templating, and (when set) will result in the 332 final templated result being wrapped in AnsibleUnsafe. 333 ''' 334 def __init__(self, *args, **kwargs): 335 super(AnsibleContext, self).__init__(*args, **kwargs) 336 self.unsafe = False 337 338 def _is_unsafe(self, val): 339 ''' 340 Our helper function, which will also recursively check dict and 341 list entries due to the fact that they may be repr'd and contain 342 a key or value which contains jinja2 syntax and would otherwise 343 lose the AnsibleUnsafe value. 344 ''' 345 if isinstance(val, dict): 346 for key in val.keys(): 347 if self._is_unsafe(val[key]): 348 return True 349 elif isinstance(val, list): 350 for item in val: 351 if self._is_unsafe(item): 352 return True 353 elif getattr(val, '__UNSAFE__', False) is True: 354 return True 355 return False 356 357 def _update_unsafe(self, val): 358 if val is not None and not self.unsafe and self._is_unsafe(val): 359 self.unsafe = True 360 361 def resolve(self, key): 362 ''' 363 The intercepted resolve(), which uses the helper above to set the 364 internal flag whenever an unsafe variable value is returned. 365 ''' 366 val = super(AnsibleContext, self).resolve(key) 367 self._update_unsafe(val) 368 return val 369 370 def resolve_or_missing(self, key): 371 val = super(AnsibleContext, self).resolve_or_missing(key) 372 self._update_unsafe(val) 373 return val 374 375 def get_all(self): 376 """Return the complete context as a dict including the exported 377 variables. For optimizations reasons this might not return an 378 actual copy so be careful with using it. 379 380 This is to prevent from running ``AnsibleJ2Vars`` through dict(): 381 382 ``dict(self.parent, **self.vars)`` 383 384 In Ansible this means that ALL variables would be templated in the 385 process of re-creating the parent because ``AnsibleJ2Vars`` templates 386 each variable in its ``__getitem__`` method. Instead we re-create the 387 parent via ``AnsibleJ2Vars.add_locals`` that creates a new 388 ``AnsibleJ2Vars`` copy without templating each variable. 389 390 This will prevent unnecessarily templating unused variables in cases 391 like setting a local variable and passing it to {% include %} 392 in a template. 393 394 Also see ``AnsibleJ2Template``and 395 https://github.com/pallets/jinja/commit/d67f0fd4cc2a4af08f51f4466150d49da7798729 396 """ 397 if LooseVersion(j2_version) >= LooseVersion('2.9'): 398 if not self.vars: 399 return self.parent 400 if not self.parent: 401 return self.vars 402 403 if isinstance(self.parent, AnsibleJ2Vars): 404 return self.parent.add_locals(self.vars) 405 else: 406 # can this happen in Ansible? 407 return dict(self.parent, **self.vars) 408 409 410class JinjaPluginIntercept(MutableMapping): 411 def __init__(self, delegatee, pluginloader, jinja2_native, *args, **kwargs): 412 super(JinjaPluginIntercept, self).__init__(*args, **kwargs) 413 self._delegatee = delegatee 414 self._pluginloader = pluginloader 415 self._jinja2_native = jinja2_native 416 417 if self._pluginloader.class_name == 'FilterModule': 418 self._method_map_name = 'filters' 419 self._dirname = 'filter' 420 elif self._pluginloader.class_name == 'TestModule': 421 self._method_map_name = 'tests' 422 self._dirname = 'test' 423 424 self._collection_jinja_func_cache = {} 425 426 self._ansible_plugins_loaded = False 427 428 def _load_ansible_plugins(self): 429 if self._ansible_plugins_loaded: 430 return 431 432 for plugin in self._pluginloader.all(): 433 method_map = getattr(plugin, self._method_map_name) 434 self._delegatee.update(method_map()) 435 436 if self._pluginloader.class_name == 'FilterModule': 437 for plugin_name, plugin in self._delegatee.items(): 438 if self._jinja2_native and plugin_name in C.STRING_TYPE_FILTERS: 439 self._delegatee[plugin_name] = _wrap_native_text(plugin) 440 else: 441 self._delegatee[plugin_name] = _unroll_iterator(plugin) 442 443 self._ansible_plugins_loaded = True 444 445 # FUTURE: we can cache FQ filter/test calls for the entire duration of a run, since a given collection's impl's 446 # aren't supposed to change during a run 447 def __getitem__(self, key): 448 self._load_ansible_plugins() 449 450 try: 451 if not isinstance(key, string_types): 452 raise ValueError('key must be a string') 453 454 key = to_native(key) 455 456 if '.' not in key: # might be a built-in or legacy, check the delegatee dict first, then try for a last-chance base redirect 457 func = self._delegatee.get(key) 458 459 if func: 460 return func 461 462 # didn't find it in the pre-built Jinja env, assume it's a former builtin and follow the normal routing path 463 leaf_key = key 464 key = 'ansible.builtin.' + key 465 else: 466 leaf_key = key.split('.')[-1] 467 468 acr = AnsibleCollectionRef.try_parse_fqcr(key, self._dirname) 469 470 if not acr: 471 raise KeyError('invalid plugin name: {0}'.format(key)) 472 473 ts = _get_collection_metadata(acr.collection) 474 475 # TODO: implement support for collection-backed redirect (currently only builtin) 476 # TODO: implement cycle detection (unified across collection redir as well) 477 478 routing_entry = ts.get('plugin_routing', {}).get(self._dirname, {}).get(leaf_key, {}) 479 480 deprecation_entry = routing_entry.get('deprecation') 481 if deprecation_entry: 482 warning_text = deprecation_entry.get('warning_text') 483 removal_date = deprecation_entry.get('removal_date') 484 removal_version = deprecation_entry.get('removal_version') 485 486 if not warning_text: 487 warning_text = '{0} "{1}" is deprecated'.format(self._dirname, key) 488 489 display.deprecated(warning_text, version=removal_version, date=removal_date, collection_name=acr.collection) 490 491 tombstone_entry = routing_entry.get('tombstone') 492 493 if tombstone_entry: 494 warning_text = tombstone_entry.get('warning_text') 495 removal_date = tombstone_entry.get('removal_date') 496 removal_version = tombstone_entry.get('removal_version') 497 498 if not warning_text: 499 warning_text = '{0} "{1}" has been removed'.format(self._dirname, key) 500 501 exc_msg = display.get_deprecation_message(warning_text, version=removal_version, date=removal_date, 502 collection_name=acr.collection, removed=True) 503 504 raise AnsiblePluginRemovedError(exc_msg) 505 506 redirect_fqcr = routing_entry.get('redirect', None) 507 if redirect_fqcr: 508 acr = AnsibleCollectionRef.from_fqcr(ref=redirect_fqcr, ref_type=self._dirname) 509 display.vvv('redirecting {0} {1} to {2}.{3}'.format(self._dirname, key, acr.collection, acr.resource)) 510 key = redirect_fqcr 511 # TODO: handle recursive forwarding (not necessary for builtin, but definitely for further collection redirs) 512 513 func = self._collection_jinja_func_cache.get(key) 514 515 if func: 516 return func 517 518 try: 519 pkg = import_module(acr.n_python_package_name) 520 except ImportError: 521 raise KeyError() 522 523 parent_prefix = acr.collection 524 525 if acr.subdirs: 526 parent_prefix = '{0}.{1}'.format(parent_prefix, acr.subdirs) 527 528 # TODO: implement collection-level redirect 529 530 for dummy, module_name, ispkg in pkgutil.iter_modules(pkg.__path__, prefix=parent_prefix + '.'): 531 if ispkg: 532 continue 533 534 try: 535 plugin_impl = self._pluginloader.get(module_name) 536 except Exception as e: 537 raise TemplateSyntaxError(to_native(e), 0) 538 539 method_map = getattr(plugin_impl, self._method_map_name) 540 541 for func_name, func in iteritems(method_map()): 542 fq_name = '.'.join((parent_prefix, func_name)) 543 # FIXME: detect/warn on intra-collection function name collisions 544 if self._pluginloader.class_name == 'FilterModule': 545 if self._jinja2_native and fq_name.startswith(('ansible.builtin.', 'ansible.legacy.')) and \ 546 func_name in C.STRING_TYPE_FILTERS: 547 self._collection_jinja_func_cache[fq_name] = _wrap_native_text(func) 548 else: 549 self._collection_jinja_func_cache[fq_name] = _unroll_iterator(func) 550 else: 551 self._collection_jinja_func_cache[fq_name] = func 552 553 function_impl = self._collection_jinja_func_cache[key] 554 return function_impl 555 except AnsiblePluginRemovedError as apre: 556 raise TemplateSyntaxError(to_native(apre), 0) 557 except KeyError: 558 raise 559 except Exception as ex: 560 display.warning('an unexpected error occurred during Jinja2 environment setup: {0}'.format(to_native(ex))) 561 display.vvv('exception during Jinja2 environment setup: {0}'.format(format_exc())) 562 raise TemplateSyntaxError(to_native(ex), 0) 563 564 def __setitem__(self, key, value): 565 return self._delegatee.__setitem__(key, value) 566 567 def __delitem__(self, key): 568 raise NotImplementedError() 569 570 def __iter__(self): 571 # not strictly accurate since we're not counting dynamically-loaded values 572 return iter(self._delegatee) 573 574 def __len__(self): 575 # not strictly accurate since we're not counting dynamically-loaded values 576 return len(self._delegatee) 577 578 579class AnsibleEnvironment(Environment): 580 ''' 581 Our custom environment, which simply allows us to override the class-level 582 values for the Template and Context classes used by jinja2 internally. 583 584 NOTE: Any changes to this class must be reflected in 585 :class:`AnsibleNativeEnvironment` as well. 586 ''' 587 context_class = AnsibleContext 588 template_class = AnsibleJ2Template 589 590 def __init__(self, *args, **kwargs): 591 super(AnsibleEnvironment, self).__init__(*args, **kwargs) 592 593 self.filters = JinjaPluginIntercept(self.filters, filter_loader, jinja2_native=False) 594 self.tests = JinjaPluginIntercept(self.tests, test_loader, jinja2_native=False) 595 596 597if USE_JINJA2_NATIVE: 598 class AnsibleNativeEnvironment(NativeEnvironment): 599 ''' 600 Our custom environment, which simply allows us to override the class-level 601 values for the Template and Context classes used by jinja2 internally. 602 603 NOTE: Any changes to this class must be reflected in 604 :class:`AnsibleEnvironment` as well. 605 ''' 606 context_class = AnsibleContext 607 template_class = AnsibleJ2Template 608 609 def __init__(self, *args, **kwargs): 610 super(AnsibleNativeEnvironment, self).__init__(*args, **kwargs) 611 612 self.filters = JinjaPluginIntercept(self.filters, filter_loader, jinja2_native=True) 613 self.tests = JinjaPluginIntercept(self.tests, test_loader, jinja2_native=True) 614 615 616class Templar: 617 ''' 618 The main class for templating, with the main entry-point of template(). 619 ''' 620 621 def __init__(self, loader, shared_loader_obj=None, variables=None): 622 # NOTE shared_loader_obj is deprecated, ansible.plugins.loader is used 623 # directly. Keeping the arg for now in case 3rd party code "uses" it. 624 self._loader = loader 625 self._filters = None 626 self._tests = None 627 self._available_variables = {} if variables is None else variables 628 self._cached_result = {} 629 self._basedir = loader.get_basedir() if loader else './' 630 631 # flags to determine whether certain failures during templating 632 # should result in fatal errors being raised 633 self._fail_on_lookup_errors = True 634 self._fail_on_filter_errors = True 635 self._fail_on_undefined_errors = C.DEFAULT_UNDEFINED_VAR_BEHAVIOR 636 637 environment_class = AnsibleNativeEnvironment if USE_JINJA2_NATIVE else AnsibleEnvironment 638 639 self.environment = environment_class( 640 trim_blocks=True, 641 undefined=AnsibleUndefined, 642 extensions=self._get_extensions(), 643 finalize=self._finalize, 644 loader=FileSystemLoader(self._basedir), 645 ) 646 647 # jinja2 global is inconsistent across versions, this normalizes them 648 self.environment.globals['dict'] = dict 649 650 # Custom globals 651 self.environment.globals['lookup'] = self._lookup 652 self.environment.globals['query'] = self.environment.globals['q'] = self._query_lookup 653 self.environment.globals['now'] = self._now_datetime 654 self.environment.globals['finalize'] = self._finalize 655 656 # the current rendering context under which the templar class is working 657 self.cur_context = None 658 659 # FIXME these regular expressions should be re-compiled each time variable_start_string and variable_end_string are changed 660 self.SINGLE_VAR = re.compile(r"^%s\s*(\w*)\s*%s$" % (self.environment.variable_start_string, self.environment.variable_end_string)) 661 self._no_type_regex = re.compile(r'.*?\|\s*(?:%s)(?:\([^\|]*\))?\s*\)?\s*(?:%s)' % 662 ('|'.join(C.STRING_TYPE_FILTERS), self.environment.variable_end_string)) 663 664 @property 665 def jinja2_native(self): 666 return not isinstance(self.environment, AnsibleEnvironment) 667 668 def copy_with_new_env(self, environment_class=AnsibleEnvironment, **kwargs): 669 r"""Creates a new copy of Templar with a new environment. The new environment is based on 670 given environment class and kwargs. 671 672 :kwarg environment_class: Environment class used for creating a new environment. 673 :kwarg \*\*kwargs: Optional arguments for the new environment that override existing 674 environment attributes. 675 676 :returns: Copy of Templar with updated environment. 677 """ 678 # We need to use __new__ to skip __init__, mainly not to create a new 679 # environment there only to override it below 680 new_env = object.__new__(environment_class) 681 new_env.__dict__.update(self.environment.__dict__) 682 683 new_templar = object.__new__(Templar) 684 new_templar.__dict__.update(self.__dict__) 685 new_templar.environment = new_env 686 687 mapping = { 688 'available_variables': new_templar, 689 'searchpath': new_env.loader, 690 } 691 692 for key, value in kwargs.items(): 693 obj = mapping.get(key, new_env) 694 try: 695 if value is not None: 696 setattr(obj, key, value) 697 except AttributeError: 698 # Ignore invalid attrs, lstrip_blocks was added in jinja2==2.7 699 pass 700 701 return new_templar 702 703 def _get_extensions(self): 704 ''' 705 Return jinja2 extensions to load. 706 707 If some extensions are set via jinja_extensions in ansible.cfg, we try 708 to load them with the jinja environment. 709 ''' 710 711 jinja_exts = [] 712 if C.DEFAULT_JINJA2_EXTENSIONS: 713 # make sure the configuration directive doesn't contain spaces 714 # and split extensions in an array 715 jinja_exts = C.DEFAULT_JINJA2_EXTENSIONS.replace(" ", "").split(',') 716 717 return jinja_exts 718 719 @property 720 def available_variables(self): 721 return self._available_variables 722 723 @available_variables.setter 724 def available_variables(self, variables): 725 ''' 726 Sets the list of template variables this Templar instance will use 727 to template things, so we don't have to pass them around between 728 internal methods. We also clear the template cache here, as the variables 729 are being changed. 730 ''' 731 732 if not isinstance(variables, Mapping): 733 raise AnsibleAssertionError("the type of 'variables' should be a Mapping but was a %s" % (type(variables))) 734 self._available_variables = variables 735 self._cached_result = {} 736 737 def set_available_variables(self, variables): 738 display.deprecated( 739 'set_available_variables is being deprecated. Use "@available_variables.setter" instead.', 740 version='2.13', collection_name='ansible.builtin' 741 ) 742 self.available_variables = variables 743 744 @contextmanager 745 def set_temporary_context(self, **kwargs): 746 """Context manager used to set temporary templating context, without having to worry about resetting 747 original values afterward 748 749 Use a keyword that maps to the attr you are setting. Applies to ``self.environment`` by default, to 750 set context on another object, it must be in ``mapping``. 751 """ 752 mapping = { 753 'available_variables': self, 754 'searchpath': self.environment.loader, 755 } 756 original = {} 757 758 for key, value in kwargs.items(): 759 obj = mapping.get(key, self.environment) 760 try: 761 original[key] = getattr(obj, key) 762 if value is not None: 763 setattr(obj, key, value) 764 except AttributeError: 765 # Ignore invalid attrs, lstrip_blocks was added in jinja2==2.7 766 pass 767 768 yield 769 770 for key in original: 771 obj = mapping.get(key, self.environment) 772 setattr(obj, key, original[key]) 773 774 def template(self, variable, convert_bare=False, preserve_trailing_newlines=True, escape_backslashes=True, fail_on_undefined=None, overrides=None, 775 convert_data=True, static_vars=None, cache=True, disable_lookups=False): 776 ''' 777 Templates (possibly recursively) any given data as input. If convert_bare is 778 set to True, the given data will be wrapped as a jinja2 variable ('{{foo}}') 779 before being sent through the template engine. 780 ''' 781 static_vars = [] if static_vars is None else static_vars 782 783 # Don't template unsafe variables, just return them. 784 if hasattr(variable, '__UNSAFE__'): 785 return variable 786 787 if fail_on_undefined is None: 788 fail_on_undefined = self._fail_on_undefined_errors 789 790 try: 791 if convert_bare: 792 variable = self._convert_bare_variable(variable) 793 794 if isinstance(variable, string_types): 795 result = variable 796 797 if self.is_possibly_template(variable): 798 # Check to see if the string we are trying to render is just referencing a single 799 # var. In this case we don't want to accidentally change the type of the variable 800 # to a string by using the jinja template renderer. We just want to pass it. 801 only_one = self.SINGLE_VAR.match(variable) 802 if only_one: 803 var_name = only_one.group(1) 804 if var_name in self._available_variables: 805 resolved_val = self._available_variables[var_name] 806 if isinstance(resolved_val, NON_TEMPLATED_TYPES): 807 return resolved_val 808 elif resolved_val is None: 809 return C.DEFAULT_NULL_REPRESENTATION 810 811 # Using a cache in order to prevent template calls with already templated variables 812 sha1_hash = None 813 if cache: 814 variable_hash = sha1(text_type(variable).encode('utf-8')) 815 options_hash = sha1( 816 ( 817 text_type(preserve_trailing_newlines) + 818 text_type(escape_backslashes) + 819 text_type(fail_on_undefined) + 820 text_type(overrides) 821 ).encode('utf-8') 822 ) 823 sha1_hash = variable_hash.hexdigest() + options_hash.hexdigest() 824 if cache and sha1_hash in self._cached_result: 825 result = self._cached_result[sha1_hash] 826 else: 827 result = self.do_template( 828 variable, 829 preserve_trailing_newlines=preserve_trailing_newlines, 830 escape_backslashes=escape_backslashes, 831 fail_on_undefined=fail_on_undefined, 832 overrides=overrides, 833 disable_lookups=disable_lookups, 834 ) 835 836 if not self.jinja2_native: 837 unsafe = hasattr(result, '__UNSAFE__') 838 if convert_data and not self._no_type_regex.match(variable): 839 # if this looks like a dictionary or list, convert it to such using the safe_eval method 840 if (result.startswith("{") and not result.startswith(self.environment.variable_start_string)) or \ 841 result.startswith("[") or result in ("True", "False"): 842 eval_results = safe_eval(result, include_exceptions=True) 843 if eval_results[1] is None: 844 result = eval_results[0] 845 if unsafe: 846 result = wrap_var(result) 847 else: 848 # FIXME: if the safe_eval raised an error, should we do something with it? 849 pass 850 851 # we only cache in the case where we have a single variable 852 # name, to make sure we're not putting things which may otherwise 853 # be dynamic in the cache (filters, lookups, etc.) 854 if cache and only_one: 855 self._cached_result[sha1_hash] = result 856 857 return result 858 859 elif is_sequence(variable): 860 return [self.template( 861 v, 862 preserve_trailing_newlines=preserve_trailing_newlines, 863 fail_on_undefined=fail_on_undefined, 864 overrides=overrides, 865 disable_lookups=disable_lookups, 866 ) for v in variable] 867 elif isinstance(variable, Mapping): 868 d = {} 869 # we don't use iteritems() here to avoid problems if the underlying dict 870 # changes sizes due to the templating, which can happen with hostvars 871 for k in variable.keys(): 872 if k not in static_vars: 873 d[k] = self.template( 874 variable[k], 875 preserve_trailing_newlines=preserve_trailing_newlines, 876 fail_on_undefined=fail_on_undefined, 877 overrides=overrides, 878 disable_lookups=disable_lookups, 879 ) 880 else: 881 d[k] = variable[k] 882 return d 883 else: 884 return variable 885 886 except AnsibleFilterError: 887 if self._fail_on_filter_errors: 888 raise 889 else: 890 return variable 891 892 def is_template(self, data): 893 '''lets us know if data has a template''' 894 if isinstance(data, string_types): 895 return is_template(data, self.environment) 896 elif isinstance(data, (list, tuple)): 897 for v in data: 898 if self.is_template(v): 899 return True 900 elif isinstance(data, dict): 901 for k in data: 902 if self.is_template(k) or self.is_template(data[k]): 903 return True 904 return False 905 906 templatable = is_template 907 908 def is_possibly_template(self, data): 909 '''Determines if a string looks like a template, by seeing if it 910 contains a jinja2 start delimiter. Does not guarantee that the string 911 is actually a template. 912 913 This is different than ``is_template`` which is more strict. 914 This method may return ``True`` on a string that is not templatable. 915 916 Useful when guarding passing a string for templating, but when 917 you want to allow the templating engine to make the final 918 assessment which may result in ``TemplateSyntaxError``. 919 ''' 920 env = self.environment 921 if isinstance(data, string_types): 922 for marker in (env.block_start_string, env.variable_start_string, env.comment_start_string): 923 if marker in data: 924 return True 925 return False 926 927 def _convert_bare_variable(self, variable): 928 ''' 929 Wraps a bare string, which may have an attribute portion (ie. foo.bar) 930 in jinja2 variable braces so that it is evaluated properly. 931 ''' 932 933 if isinstance(variable, string_types): 934 contains_filters = "|" in variable 935 first_part = variable.split("|")[0].split(".")[0].split("[")[0] 936 if (contains_filters or first_part in self._available_variables) and self.environment.variable_start_string not in variable: 937 return "%s%s%s" % (self.environment.variable_start_string, variable, self.environment.variable_end_string) 938 939 # the variable didn't meet the conditions to be converted, 940 # so just return it as-is 941 return variable 942 943 def _finalize(self, thing): 944 ''' 945 A custom finalize method for jinja2, which prevents None from being returned. This 946 avoids a string of ``"None"`` as ``None`` has no importance in YAML. 947 948 If using ANSIBLE_JINJA2_NATIVE we bypass this and return the actual value always 949 ''' 950 if _is_rolled(thing): 951 # Auto unroll a generator, so that users are not required to 952 # explicitly use ``|list`` to unroll 953 # This only affects the scenario where the final result of templating 954 # is a generator, and not where a filter creates a generator in the middle 955 # of a template. See ``_unroll_iterator`` for the other case. This is probably 956 # unncessary 957 return list(thing) 958 959 if self.jinja2_native: 960 return thing 961 962 return thing if thing is not None else '' 963 964 def _fail_lookup(self, name, *args, **kwargs): 965 raise AnsibleError("The lookup `%s` was found, however lookups were disabled from templating" % name) 966 967 def _now_datetime(self, utc=False, fmt=None): 968 '''jinja2 global function to return current datetime, potentially formatted via strftime''' 969 if utc: 970 now = datetime.datetime.utcnow() 971 else: 972 now = datetime.datetime.now() 973 974 if fmt: 975 return now.strftime(fmt) 976 977 return now 978 979 def _query_lookup(self, name, *args, **kwargs): 980 ''' wrapper for lookup, force wantlist true''' 981 kwargs['wantlist'] = True 982 return self._lookup(name, *args, **kwargs) 983 984 def _lookup(self, name, *args, **kwargs): 985 instance = lookup_loader.get(name, loader=self._loader, templar=self) 986 987 if instance is None: 988 raise AnsibleError("lookup plugin (%s) not found" % name) 989 990 wantlist = kwargs.pop('wantlist', False) 991 allow_unsafe = kwargs.pop('allow_unsafe', C.DEFAULT_ALLOW_UNSAFE_LOOKUPS) 992 errors = kwargs.pop('errors', 'strict') 993 994 loop_terms = listify_lookup_plugin_terms(terms=args, templar=self, loader=self._loader, fail_on_undefined=True, convert_bare=False) 995 # safely catch run failures per #5059 996 try: 997 ran = instance.run(loop_terms, variables=self._available_variables, **kwargs) 998 except (AnsibleUndefinedVariable, UndefinedError) as e: 999 raise AnsibleUndefinedVariable(e) 1000 except Exception as e: 1001 if self._fail_on_lookup_errors: 1002 msg = u"An unhandled exception occurred while running the lookup plugin '%s'. Error was a %s, original message: %s" % \ 1003 (name, type(e), to_text(e)) 1004 if errors == 'warn': 1005 display.warning(msg) 1006 elif errors == 'ignore': 1007 display.display(msg, log_only=True) 1008 else: 1009 raise AnsibleError(to_native(msg)) 1010 return [] if wantlist else None 1011 1012 if ran and allow_unsafe is False: 1013 if self.cur_context: 1014 self.cur_context.unsafe = True 1015 1016 if wantlist: 1017 return wrap_var(ran) 1018 1019 try: 1020 if self.jinja2_native and isinstance(ran[0], NativeJinjaText): 1021 ran = wrap_var(NativeJinjaText(",".join(ran))) 1022 else: 1023 ran = wrap_var(",".join(ran)) 1024 except TypeError: 1025 # Lookup Plugins should always return lists. Throw an error if that's not 1026 # the case: 1027 if not isinstance(ran, Sequence): 1028 raise AnsibleError("The lookup plugin '%s' did not return a list." 1029 % name) 1030 1031 # The TypeError we can recover from is when the value *inside* of the list 1032 # is not a string 1033 if len(ran) == 1: 1034 ran = wrap_var(ran[0]) 1035 else: 1036 ran = wrap_var(ran) 1037 1038 return ran 1039 1040 def do_template(self, data, preserve_trailing_newlines=True, escape_backslashes=True, fail_on_undefined=None, overrides=None, disable_lookups=False): 1041 if self.jinja2_native and not isinstance(data, string_types): 1042 return data 1043 1044 # For preserving the number of input newlines in the output (used 1045 # later in this method) 1046 data_newlines = _count_newlines_from_end(data) 1047 1048 if fail_on_undefined is None: 1049 fail_on_undefined = self._fail_on_undefined_errors 1050 1051 has_template_overrides = data.startswith(JINJA2_OVERRIDE) 1052 1053 try: 1054 # NOTE Creating an overlay that lives only inside do_template means that overrides are not applied 1055 # when templating nested variables in AnsibleJ2Vars where Templar.environment is used, not the overlay. 1056 # This is historic behavior that is kept for backwards compatibility. 1057 if overrides: 1058 myenv = self.environment.overlay(overrides) 1059 elif has_template_overrides: 1060 myenv = self.environment.overlay() 1061 else: 1062 myenv = self.environment 1063 1064 # Get jinja env overrides from template 1065 if has_template_overrides: 1066 eol = data.find('\n') 1067 line = data[len(JINJA2_OVERRIDE):eol] 1068 data = data[eol + 1:] 1069 for pair in line.split(','): 1070 (key, val) = pair.split(':') 1071 key = key.strip() 1072 setattr(myenv, key, ast.literal_eval(val.strip())) 1073 1074 if escape_backslashes: 1075 # Allow users to specify backslashes in playbooks as "\\" instead of as "\\\\". 1076 data = _escape_backslashes(data, myenv) 1077 1078 try: 1079 t = myenv.from_string(data) 1080 except TemplateSyntaxError as e: 1081 raise AnsibleError("template error while templating string: %s. String: %s" % (to_native(e), to_native(data))) 1082 except Exception as e: 1083 if 'recursion' in to_native(e): 1084 raise AnsibleError("recursive loop detected in template string: %s" % to_native(data)) 1085 else: 1086 return data 1087 1088 if disable_lookups: 1089 t.globals['query'] = t.globals['q'] = t.globals['lookup'] = self._fail_lookup 1090 1091 jvars = AnsibleJ2Vars(self, t.globals) 1092 1093 self.cur_context = new_context = t.new_context(jvars, shared=True) 1094 rf = t.root_render_func(new_context) 1095 1096 try: 1097 if self.jinja2_native: 1098 res = ansible_native_concat(rf) 1099 else: 1100 res = j2_concat(rf) 1101 unsafe = getattr(new_context, 'unsafe', False) 1102 if unsafe: 1103 res = wrap_var(res) 1104 except TypeError as te: 1105 if 'AnsibleUndefined' in to_native(te): 1106 errmsg = "Unable to look up a name or access an attribute in template string (%s).\n" % to_native(data) 1107 errmsg += "Make sure your variable name does not contain invalid characters like '-': %s" % to_native(te) 1108 raise AnsibleUndefinedVariable(errmsg) 1109 else: 1110 display.debug("failing because of a type error, template data is: %s" % to_text(data)) 1111 raise AnsibleError("Unexpected templating type error occurred on (%s): %s" % (to_native(data), to_native(te))) 1112 1113 if self.jinja2_native and not isinstance(res, string_types): 1114 return res 1115 1116 if preserve_trailing_newlines: 1117 # The low level calls above do not preserve the newline 1118 # characters at the end of the input data, so we use the 1119 # calculate the difference in newlines and append them 1120 # to the resulting output for parity 1121 # 1122 # jinja2 added a keep_trailing_newline option in 2.7 when 1123 # creating an Environment. That would let us make this code 1124 # better (remove a single newline if 1125 # preserve_trailing_newlines is False). Once we can depend on 1126 # that version being present, modify our code to set that when 1127 # initializing self.environment and remove a single trailing 1128 # newline here if preserve_newlines is False. 1129 res_newlines = _count_newlines_from_end(res) 1130 if data_newlines > res_newlines: 1131 res += self.environment.newline_sequence * (data_newlines - res_newlines) 1132 if unsafe: 1133 res = wrap_var(res) 1134 return res 1135 except (UndefinedError, AnsibleUndefinedVariable) as e: 1136 if fail_on_undefined: 1137 raise AnsibleUndefinedVariable(e) 1138 else: 1139 display.debug("Ignoring undefined failure: %s" % to_text(e)) 1140 return data 1141 1142 # for backwards compatibility in case anyone is using old private method directly 1143 _do_template = do_template 1144