1# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
2#
3# This file is part of Ansible
4#
5# Ansible is free software: you can redistribute it and/or modify
6# it under the terms of the GNU General Public License as published by
7# the Free Software Foundation, either version 3 of the License, or
8# (at your option) any later version.
9#
10# Ansible is distributed in the hope that it will be useful,
11# but WITHOUT ANY WARRANTY; without even the implied warranty of
12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13# GNU General Public License for more details.
14#
15# You should have received a copy of the GNU General Public License
16# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
17
18# Make coding more python3-ish
19from __future__ import (absolute_import, division, print_function)
20__metaclass__ = type
21
22import ast
23import datetime
24import os
25import pkgutil
26import pwd
27import re
28import time
29
30from contextlib import contextmanager
31from distutils.version import LooseVersion
32from numbers import Number
33from traceback import format_exc
34
35try:
36    from hashlib import sha1
37except ImportError:
38    from sha import sha as sha1
39
40from jinja2.exceptions import TemplateSyntaxError, UndefinedError
41from jinja2.loaders import FileSystemLoader
42from jinja2.runtime import Context, StrictUndefined
43
44from ansible import constants as C
45from ansible.errors import AnsibleError, AnsibleFilterError, AnsiblePluginRemovedError, AnsibleUndefinedVariable, AnsibleAssertionError
46from ansible.module_utils.six import iteritems, string_types, text_type
47from ansible.module_utils.six.moves import range
48from ansible.module_utils._text import to_native, to_text, to_bytes
49from ansible.module_utils.common._collections_compat import Iterator, Sequence, Mapping, MappingView, MutableMapping
50from ansible.module_utils.common.collections import is_sequence
51from ansible.module_utils.compat.importlib import import_module
52from ansible.plugins.loader import filter_loader, lookup_loader, test_loader
53from ansible.template.safe_eval import safe_eval
54from ansible.template.template import AnsibleJ2Template
55from ansible.template.vars import AnsibleJ2Vars
56from ansible.utils.collection_loader import AnsibleCollectionRef
57from ansible.utils.display import Display
58from ansible.utils.collection_loader._collection_finder import _get_collection_metadata
59from ansible.utils.unsafe_proxy import wrap_var
60
61display = Display()
62
63
64__all__ = ['Templar', 'generate_ansible_template_vars']
65
66# A regex for checking to see if a variable we're trying to
67# expand is just a single variable name.
68
69# Primitive Types which we don't want Jinja to convert to strings.
70NON_TEMPLATED_TYPES = (bool, Number)
71
72JINJA2_OVERRIDE = '#jinja2:'
73
74from jinja2 import __version__ as j2_version
75
76USE_JINJA2_NATIVE = False
77if C.DEFAULT_JINJA2_NATIVE:
78    try:
79        from jinja2.nativetypes import NativeEnvironment as Environment
80        from ansible.template.native_helpers import ansible_native_concat as j2_concat
81        from ansible.template.native_helpers import NativeJinjaText
82        USE_JINJA2_NATIVE = True
83    except ImportError:
84        from jinja2 import Environment
85        from jinja2.utils import concat as j2_concat
86        display.warning(
87            'jinja2_native requires Jinja 2.10 and above. '
88            'Version detected: %s. Falling back to default.' % j2_version
89        )
90else:
91    from jinja2 import Environment
92    from jinja2.utils import concat as j2_concat
93
94
95JINJA2_BEGIN_TOKENS = frozenset(('variable_begin', 'block_begin', 'comment_begin', 'raw_begin'))
96JINJA2_END_TOKENS = frozenset(('variable_end', 'block_end', 'comment_end', 'raw_end'))
97
98
99RANGE_TYPE = type(range(0))
100
101
102def generate_ansible_template_vars(path, fullpath=None, dest_path=None):
103
104    if fullpath is None:
105        b_path = to_bytes(path)
106    else:
107        b_path = to_bytes(fullpath)
108
109    try:
110        template_uid = pwd.getpwuid(os.stat(b_path).st_uid).pw_name
111    except (KeyError, TypeError):
112        template_uid = os.stat(b_path).st_uid
113
114    temp_vars = {
115        'template_host': to_text(os.uname()[1]),
116        'template_path': path,
117        'template_mtime': datetime.datetime.fromtimestamp(os.path.getmtime(b_path)),
118        'template_uid': to_text(template_uid),
119        'template_run_date': datetime.datetime.now(),
120        'template_destpath': to_native(dest_path) if dest_path else None,
121    }
122
123    if fullpath is None:
124        temp_vars['template_fullpath'] = os.path.abspath(path)
125    else:
126        temp_vars['template_fullpath'] = fullpath
127
128    managed_default = C.DEFAULT_MANAGED_STR
129    managed_str = managed_default.format(
130        host=temp_vars['template_host'],
131        uid=temp_vars['template_uid'],
132        file=temp_vars['template_path'],
133    )
134    temp_vars['ansible_managed'] = to_text(time.strftime(to_native(managed_str), time.localtime(os.path.getmtime(b_path))))
135
136    return temp_vars
137
138
139def _escape_backslashes(data, jinja_env):
140    """Double backslashes within jinja2 expressions
141
142    A user may enter something like this in a playbook::
143
144      debug:
145        msg: "Test Case 1\\3; {{ test1_name | regex_replace('^(.*)_name$', '\\1')}}"
146
147    The string inside of the {{ gets interpreted multiple times First by yaml.
148    Then by python.  And finally by jinja2 as part of it's variable.  Because
149    it is processed by both python and jinja2, the backslash escaped
150    characters get unescaped twice.  This means that we'd normally have to use
151    four backslashes to escape that.  This is painful for playbook authors as
152    they have to remember different rules for inside vs outside of a jinja2
153    expression (The backslashes outside of the "{{ }}" only get processed by
154    yaml and python.  So they only need to be escaped once).  The following
155    code fixes this by automatically performing the extra quoting of
156    backslashes inside of a jinja2 expression.
157
158    """
159    if '\\' in data and '{{' in data:
160        new_data = []
161        d2 = jinja_env.preprocess(data)
162        in_var = False
163
164        for token in jinja_env.lex(d2):
165            if token[1] == 'variable_begin':
166                in_var = True
167                new_data.append(token[2])
168            elif token[1] == 'variable_end':
169                in_var = False
170                new_data.append(token[2])
171            elif in_var and token[1] == 'string':
172                # Double backslashes only if we're inside of a jinja2 variable
173                new_data.append(token[2].replace('\\', '\\\\'))
174            else:
175                new_data.append(token[2])
176
177        data = ''.join(new_data)
178
179    return data
180
181
182def is_template(data, jinja_env):
183    """This function attempts to quickly detect whether a value is a jinja2
184    template. To do so, we look for the first 2 matching jinja2 tokens for
185    start and end delimiters.
186    """
187    found = None
188    start = True
189    comment = False
190    d2 = jinja_env.preprocess(data)
191
192    # This wraps a lot of code, but this is due to lex returning a generator
193    # so we may get an exception at any part of the loop
194    try:
195        for token in jinja_env.lex(d2):
196            if token[1] in JINJA2_BEGIN_TOKENS:
197                if start and token[1] == 'comment_begin':
198                    # Comments can wrap other token types
199                    comment = True
200                start = False
201                # Example: variable_end -> variable
202                found = token[1].split('_')[0]
203            elif token[1] in JINJA2_END_TOKENS:
204                if token[1].split('_')[0] == found:
205                    return True
206                elif comment:
207                    continue
208                return False
209    except TemplateSyntaxError:
210        return False
211
212    return False
213
214
215def _count_newlines_from_end(in_str):
216    '''
217    Counts the number of newlines at the end of a string. This is used during
218    the jinja2 templating to ensure the count matches the input, since some newlines
219    may be thrown away during the templating.
220    '''
221
222    try:
223        i = len(in_str)
224        j = i - 1
225        while in_str[j] == '\n':
226            j -= 1
227        return i - 1 - j
228    except IndexError:
229        # Uncommon cases: zero length string and string containing only newlines
230        return i
231
232
233def recursive_check_defined(item):
234    from jinja2.runtime import Undefined
235
236    if isinstance(item, MutableMapping):
237        for key in item:
238            recursive_check_defined(item[key])
239    elif isinstance(item, list):
240        for i in item:
241            recursive_check_defined(i)
242    else:
243        if isinstance(item, Undefined):
244            raise AnsibleFilterError("{0} is undefined".format(item))
245
246
247def _is_rolled(value):
248    """Helper method to determine if something is an unrolled generator,
249    iterator, or similar object
250    """
251    return (
252        isinstance(value, Iterator) or
253        isinstance(value, MappingView) or
254        isinstance(value, RANGE_TYPE)
255    )
256
257
258def _unroll_iterator(func):
259    """Wrapper function, that intercepts the result of a filter
260    and auto unrolls a generator, so that users are not required to
261    explicitly use ``|list`` to unroll.
262    """
263    def wrapper(*args, **kwargs):
264        ret = func(*args, **kwargs)
265        if _is_rolled(ret):
266            return list(ret)
267        return ret
268
269    wrapper.__UNROLLED__ = True
270    return _update_wrapper(wrapper, func)
271
272
273def _update_wrapper(wrapper, func):
274    # This code is duplicated from ``functools.update_wrapper`` from Py3.7.
275    # ``functools.update_wrapper`` was failing when the func was ``functools.partial``
276    for attr in ('__module__', '__name__', '__qualname__', '__doc__', '__annotations__'):
277        try:
278            value = getattr(func, attr)
279        except AttributeError:
280            pass
281        else:
282            setattr(wrapper, attr, value)
283    for attr in ('__dict__',):
284        getattr(wrapper, attr).update(getattr(func, attr, {}))
285    wrapper.__wrapped__ = func
286    return wrapper
287
288
289def _wrap_native_text(func):
290    """Wrapper function, that intercepts the result of a filter
291    and wraps it into NativeJinjaText which is then used
292    in ``ansible_native_concat`` to indicate that it is a text
293    which should not be passed into ``literal_eval``.
294    """
295    def wrapper(*args, **kwargs):
296        ret = func(*args, **kwargs)
297        return NativeJinjaText(ret)
298
299    return _update_wrapper(wrapper, func)
300
301
302class AnsibleUndefined(StrictUndefined):
303    '''
304    A custom Undefined class, which returns further Undefined objects on access,
305    rather than throwing an exception.
306    '''
307    def __getattr__(self, name):
308        if name == '__UNSAFE__':
309            # AnsibleUndefined should never be assumed to be unsafe
310            # This prevents ``hasattr(val, '__UNSAFE__')`` from evaluating to ``True``
311            raise AttributeError(name)
312        # Return original Undefined object to preserve the first failure context
313        return self
314
315    def __getitem__(self, key):
316        # Return original Undefined object to preserve the first failure context
317        return self
318
319    def __repr__(self):
320        return 'AnsibleUndefined'
321
322    def __contains__(self, item):
323        # Return original Undefined object to preserve the first failure context
324        return self
325
326
327class AnsibleContext(Context):
328    '''
329    A custom context, which intercepts resolve() calls and sets a flag
330    internally if any variable lookup returns an AnsibleUnsafe value. This
331    flag is checked post-templating, and (when set) will result in the
332    final templated result being wrapped in AnsibleUnsafe.
333    '''
334    def __init__(self, *args, **kwargs):
335        super(AnsibleContext, self).__init__(*args, **kwargs)
336        self.unsafe = False
337
338    def _is_unsafe(self, val):
339        '''
340        Our helper function, which will also recursively check dict and
341        list entries due to the fact that they may be repr'd and contain
342        a key or value which contains jinja2 syntax and would otherwise
343        lose the AnsibleUnsafe value.
344        '''
345        if isinstance(val, dict):
346            for key in val.keys():
347                if self._is_unsafe(val[key]):
348                    return True
349        elif isinstance(val, list):
350            for item in val:
351                if self._is_unsafe(item):
352                    return True
353        elif getattr(val, '__UNSAFE__', False) is True:
354            return True
355        return False
356
357    def _update_unsafe(self, val):
358        if val is not None and not self.unsafe and self._is_unsafe(val):
359            self.unsafe = True
360
361    def resolve(self, key):
362        '''
363        The intercepted resolve(), which uses the helper above to set the
364        internal flag whenever an unsafe variable value is returned.
365        '''
366        val = super(AnsibleContext, self).resolve(key)
367        self._update_unsafe(val)
368        return val
369
370    def resolve_or_missing(self, key):
371        val = super(AnsibleContext, self).resolve_or_missing(key)
372        self._update_unsafe(val)
373        return val
374
375    def get_all(self):
376        """Return the complete context as a dict including the exported
377        variables. For optimizations reasons this might not return an
378        actual copy so be careful with using it.
379
380        This is to prevent from running ``AnsibleJ2Vars`` through dict():
381
382            ``dict(self.parent, **self.vars)``
383
384        In Ansible this means that ALL variables would be templated in the
385        process of re-creating the parent because ``AnsibleJ2Vars`` templates
386        each variable in its ``__getitem__`` method. Instead we re-create the
387        parent via ``AnsibleJ2Vars.add_locals`` that creates a new
388        ``AnsibleJ2Vars`` copy without templating each variable.
389
390        This will prevent unnecessarily templating unused variables in cases
391        like setting a local variable and passing it to {% include %}
392        in a template.
393
394        Also see ``AnsibleJ2Template``and
395        https://github.com/pallets/jinja/commit/d67f0fd4cc2a4af08f51f4466150d49da7798729
396        """
397        if LooseVersion(j2_version) >= LooseVersion('2.9'):
398            if not self.vars:
399                return self.parent
400            if not self.parent:
401                return self.vars
402
403        if isinstance(self.parent, AnsibleJ2Vars):
404            return self.parent.add_locals(self.vars)
405        else:
406            # can this happen in Ansible?
407            return dict(self.parent, **self.vars)
408
409
410class JinjaPluginIntercept(MutableMapping):
411    def __init__(self, delegatee, pluginloader, *args, **kwargs):
412        super(JinjaPluginIntercept, self).__init__(*args, **kwargs)
413        self._delegatee = delegatee
414        self._pluginloader = pluginloader
415
416        if self._pluginloader.class_name == 'FilterModule':
417            self._method_map_name = 'filters'
418            self._dirname = 'filter'
419        elif self._pluginloader.class_name == 'TestModule':
420            self._method_map_name = 'tests'
421            self._dirname = 'test'
422
423        self._collection_jinja_func_cache = {}
424
425    # FUTURE: we can cache FQ filter/test calls for the entire duration of a run, since a given collection's impl's
426    # aren't supposed to change during a run
427    def __getitem__(self, key):
428        try:
429            if not isinstance(key, string_types):
430                raise ValueError('key must be a string')
431
432            key = to_native(key)
433
434            if '.' not in key:  # might be a built-in or legacy, check the delegatee dict first, then try for a last-chance base redirect
435                func = self._delegatee.get(key)
436
437                if func:
438                    return func
439
440                # didn't find it in the pre-built Jinja env, assume it's a former builtin and follow the normal routing path
441                leaf_key = key
442                key = 'ansible.builtin.' + key
443            else:
444                leaf_key = key.split('.')[-1]
445
446            acr = AnsibleCollectionRef.try_parse_fqcr(key, self._dirname)
447
448            if not acr:
449                raise KeyError('invalid plugin name: {0}'.format(key))
450
451            ts = _get_collection_metadata(acr.collection)
452
453            # TODO: implement support for collection-backed redirect (currently only builtin)
454            # TODO: implement cycle detection (unified across collection redir as well)
455
456            routing_entry = ts.get('plugin_routing', {}).get(self._dirname, {}).get(leaf_key, {})
457
458            deprecation_entry = routing_entry.get('deprecation')
459            if deprecation_entry:
460                warning_text = deprecation_entry.get('warning_text')
461                removal_date = deprecation_entry.get('removal_date')
462                removal_version = deprecation_entry.get('removal_version')
463
464                if not warning_text:
465                    warning_text = '{0} "{1}" is deprecated'.format(self._dirname, key)
466
467                display.deprecated(warning_text, version=removal_version, date=removal_date, collection_name=acr.collection)
468
469            tombstone_entry = routing_entry.get('tombstone')
470
471            if tombstone_entry:
472                warning_text = tombstone_entry.get('warning_text')
473                removal_date = tombstone_entry.get('removal_date')
474                removal_version = tombstone_entry.get('removal_version')
475
476                if not warning_text:
477                    warning_text = '{0} "{1}" has been removed'.format(self._dirname, key)
478
479                exc_msg = display.get_deprecation_message(warning_text, version=removal_version, date=removal_date,
480                                                          collection_name=acr.collection, removed=True)
481
482                raise AnsiblePluginRemovedError(exc_msg)
483
484            redirect_fqcr = routing_entry.get('redirect', None)
485            if redirect_fqcr:
486                acr = AnsibleCollectionRef.from_fqcr(ref=redirect_fqcr, ref_type=self._dirname)
487                display.vvv('redirecting {0} {1} to {2}.{3}'.format(self._dirname, key, acr.collection, acr.resource))
488                key = redirect_fqcr
489            # TODO: handle recursive forwarding (not necessary for builtin, but definitely for further collection redirs)
490
491            func = self._collection_jinja_func_cache.get(key)
492
493            if func:
494                return func
495
496            try:
497                pkg = import_module(acr.n_python_package_name)
498            except ImportError:
499                raise KeyError()
500
501            parent_prefix = acr.collection
502
503            if acr.subdirs:
504                parent_prefix = '{0}.{1}'.format(parent_prefix, acr.subdirs)
505
506            # TODO: implement collection-level redirect
507
508            for dummy, module_name, ispkg in pkgutil.iter_modules(pkg.__path__, prefix=parent_prefix + '.'):
509                if ispkg:
510                    continue
511
512                try:
513                    plugin_impl = self._pluginloader.get(module_name)
514                except Exception as e:
515                    raise TemplateSyntaxError(to_native(e), 0)
516
517                method_map = getattr(plugin_impl, self._method_map_name)
518
519                for func_name, func in iteritems(method_map()):
520                    fq_name = '.'.join((parent_prefix, func_name))
521                    # FIXME: detect/warn on intra-collection function name collisions
522                    if USE_JINJA2_NATIVE and fq_name.startswith(('ansible.builtin.', 'ansible.legacy.')) and \
523                            func_name in C.STRING_TYPE_FILTERS:
524                        self._collection_jinja_func_cache[fq_name] = _wrap_native_text(func)
525                    else:
526                        self._collection_jinja_func_cache[fq_name] = _unroll_iterator(func)
527
528            function_impl = self._collection_jinja_func_cache[key]
529            return function_impl
530        except AnsiblePluginRemovedError as apre:
531            raise TemplateSyntaxError(to_native(apre), 0)
532        except KeyError:
533            raise
534        except Exception as ex:
535            display.warning('an unexpected error occurred during Jinja2 environment setup: {0}'.format(to_native(ex)))
536            display.vvv('exception during Jinja2 environment setup: {0}'.format(format_exc()))
537            raise TemplateSyntaxError(to_native(ex), 0)
538
539    def __setitem__(self, key, value):
540        return self._delegatee.__setitem__(key, value)
541
542    def __delitem__(self, key):
543        raise NotImplementedError()
544
545    def __iter__(self):
546        # not strictly accurate since we're not counting dynamically-loaded values
547        return iter(self._delegatee)
548
549    def __len__(self):
550        # not strictly accurate since we're not counting dynamically-loaded values
551        return len(self._delegatee)
552
553
554class AnsibleEnvironment(Environment):
555    '''
556    Our custom environment, which simply allows us to override the class-level
557    values for the Template and Context classes used by jinja2 internally.
558    '''
559    context_class = AnsibleContext
560    template_class = AnsibleJ2Template
561
562    def __init__(self, *args, **kwargs):
563        super(AnsibleEnvironment, self).__init__(*args, **kwargs)
564
565        self.filters = JinjaPluginIntercept(self.filters, filter_loader)
566        self.tests = JinjaPluginIntercept(self.tests, test_loader)
567
568
569class Templar:
570    '''
571    The main class for templating, with the main entry-point of template().
572    '''
573
574    def __init__(self, loader, shared_loader_obj=None, variables=None):
575        variables = {} if variables is None else variables
576
577        self._loader = loader
578        self._filters = None
579        self._tests = None
580        self._available_variables = variables
581        self._cached_result = {}
582
583        if loader:
584            self._basedir = loader.get_basedir()
585        else:
586            self._basedir = './'
587
588        if shared_loader_obj:
589            self._filter_loader = getattr(shared_loader_obj, 'filter_loader')
590            self._test_loader = getattr(shared_loader_obj, 'test_loader')
591            self._lookup_loader = getattr(shared_loader_obj, 'lookup_loader')
592        else:
593            self._filter_loader = filter_loader
594            self._test_loader = test_loader
595            self._lookup_loader = lookup_loader
596
597        # flags to determine whether certain failures during templating
598        # should result in fatal errors being raised
599        self._fail_on_lookup_errors = True
600        self._fail_on_filter_errors = True
601        self._fail_on_undefined_errors = C.DEFAULT_UNDEFINED_VAR_BEHAVIOR
602
603        self.environment = AnsibleEnvironment(
604            trim_blocks=True,
605            undefined=AnsibleUndefined,
606            extensions=self._get_extensions(),
607            finalize=self._finalize,
608            loader=FileSystemLoader(self._basedir),
609        )
610
611        # jinja2 global is inconsistent across versions, this normalizes them
612        self.environment.globals['dict'] = dict
613
614        # Custom globals
615        self.environment.globals['lookup'] = self._lookup
616        self.environment.globals['query'] = self.environment.globals['q'] = self._query_lookup
617        self.environment.globals['now'] = self._now_datetime
618        self.environment.globals['finalize'] = self._finalize
619
620        # the current rendering context under which the templar class is working
621        self.cur_context = None
622
623        self.SINGLE_VAR = re.compile(r"^%s\s*(\w*)\s*%s$" % (self.environment.variable_start_string, self.environment.variable_end_string))
624
625        self._clean_regex = re.compile(r'(?:%s|%s|%s|%s)' % (
626            self.environment.variable_start_string,
627            self.environment.block_start_string,
628            self.environment.block_end_string,
629            self.environment.variable_end_string
630        ))
631        self._no_type_regex = re.compile(r'.*?\|\s*(?:%s)(?:\([^\|]*\))?\s*\)?\s*(?:%s)' %
632                                         ('|'.join(C.STRING_TYPE_FILTERS), self.environment.variable_end_string))
633
634    def _get_filters(self):
635        '''
636        Returns filter plugins, after loading and caching them if need be
637        '''
638
639        if self._filters is not None:
640            return self._filters.copy()
641
642        self._filters = dict()
643
644        for fp in self._filter_loader.all():
645            self._filters.update(fp.filters())
646
647        if USE_JINJA2_NATIVE:
648            for string_filter in C.STRING_TYPE_FILTERS:
649                try:
650                    orig_filter = self._filters[string_filter]
651                except KeyError:
652                    try:
653                        orig_filter = self.environment.filters[string_filter]
654                    except KeyError:
655                        continue
656                self._filters[string_filter] = _wrap_native_text(orig_filter)
657
658        return self._filters.copy()
659
660    def _get_tests(self):
661        '''
662        Returns tests plugins, after loading and caching them if need be
663        '''
664
665        if self._tests is not None:
666            return self._tests.copy()
667
668        self._tests = dict()
669        for fp in self._test_loader.all():
670            self._tests.update(fp.tests())
671
672        return self._tests.copy()
673
674    def _get_extensions(self):
675        '''
676        Return jinja2 extensions to load.
677
678        If some extensions are set via jinja_extensions in ansible.cfg, we try
679        to load them with the jinja environment.
680        '''
681
682        jinja_exts = []
683        if C.DEFAULT_JINJA2_EXTENSIONS:
684            # make sure the configuration directive doesn't contain spaces
685            # and split extensions in an array
686            jinja_exts = C.DEFAULT_JINJA2_EXTENSIONS.replace(" ", "").split(',')
687
688        return jinja_exts
689
690    @property
691    def available_variables(self):
692        return self._available_variables
693
694    @available_variables.setter
695    def available_variables(self, variables):
696        '''
697        Sets the list of template variables this Templar instance will use
698        to template things, so we don't have to pass them around between
699        internal methods. We also clear the template cache here, as the variables
700        are being changed.
701        '''
702
703        if not isinstance(variables, Mapping):
704            raise AnsibleAssertionError("the type of 'variables' should be a Mapping but was a %s" % (type(variables)))
705        self._available_variables = variables
706        self._cached_result = {}
707
708    def set_available_variables(self, variables):
709        display.deprecated(
710            'set_available_variables is being deprecated. Use "@available_variables.setter" instead.',
711            version='2.13', collection_name='ansible.builtin'
712        )
713        self.available_variables = variables
714
715    @contextmanager
716    def set_temporary_context(self, **kwargs):
717        """Context manager used to set temporary templating context, without having to worry about resetting
718        original values afterward
719
720        Use a keyword that maps to the attr you are setting. Applies to ``self.environment`` by default, to
721        set context on another object, it must be in ``mapping``.
722        """
723        mapping = {
724            'available_variables': self,
725            'searchpath': self.environment.loader,
726        }
727        original = {}
728
729        for key, value in kwargs.items():
730            obj = mapping.get(key, self.environment)
731            try:
732                original[key] = getattr(obj, key)
733                if value is not None:
734                    setattr(obj, key, value)
735            except AttributeError:
736                # Ignore invalid attrs, lstrip_blocks was added in jinja2==2.7
737                pass
738
739        yield
740
741        for key in original:
742            obj = mapping.get(key, self.environment)
743            setattr(obj, key, original[key])
744
745    def template(self, variable, convert_bare=False, preserve_trailing_newlines=True, escape_backslashes=True, fail_on_undefined=None, overrides=None,
746                 convert_data=True, static_vars=None, cache=True, disable_lookups=False):
747        '''
748        Templates (possibly recursively) any given data as input. If convert_bare is
749        set to True, the given data will be wrapped as a jinja2 variable ('{{foo}}')
750        before being sent through the template engine.
751        '''
752        static_vars = [''] if static_vars is None else static_vars
753
754        # Don't template unsafe variables, just return them.
755        if hasattr(variable, '__UNSAFE__'):
756            return variable
757
758        if fail_on_undefined is None:
759            fail_on_undefined = self._fail_on_undefined_errors
760
761        try:
762            if convert_bare:
763                variable = self._convert_bare_variable(variable)
764
765            if isinstance(variable, string_types):
766                result = variable
767
768                if self.is_possibly_template(variable):
769                    # Check to see if the string we are trying to render is just referencing a single
770                    # var.  In this case we don't want to accidentally change the type of the variable
771                    # to a string by using the jinja template renderer. We just want to pass it.
772                    only_one = self.SINGLE_VAR.match(variable)
773                    if only_one:
774                        var_name = only_one.group(1)
775                        if var_name in self._available_variables:
776                            resolved_val = self._available_variables[var_name]
777                            if isinstance(resolved_val, NON_TEMPLATED_TYPES):
778                                return resolved_val
779                            elif resolved_val is None:
780                                return C.DEFAULT_NULL_REPRESENTATION
781
782                    # Using a cache in order to prevent template calls with already templated variables
783                    sha1_hash = None
784                    if cache:
785                        variable_hash = sha1(text_type(variable).encode('utf-8'))
786                        options_hash = sha1(
787                            (
788                                text_type(preserve_trailing_newlines) +
789                                text_type(escape_backslashes) +
790                                text_type(fail_on_undefined) +
791                                text_type(overrides)
792                            ).encode('utf-8')
793                        )
794                        sha1_hash = variable_hash.hexdigest() + options_hash.hexdigest()
795                    if cache and sha1_hash in self._cached_result:
796                        result = self._cached_result[sha1_hash]
797                    else:
798                        result = self.do_template(
799                            variable,
800                            preserve_trailing_newlines=preserve_trailing_newlines,
801                            escape_backslashes=escape_backslashes,
802                            fail_on_undefined=fail_on_undefined,
803                            overrides=overrides,
804                            disable_lookups=disable_lookups,
805                        )
806
807                        if not USE_JINJA2_NATIVE:
808                            unsafe = hasattr(result, '__UNSAFE__')
809                            if convert_data and not self._no_type_regex.match(variable):
810                                # if this looks like a dictionary or list, convert it to such using the safe_eval method
811                                if (result.startswith("{") and not result.startswith(self.environment.variable_start_string)) or \
812                                        result.startswith("[") or result in ("True", "False"):
813                                    eval_results = safe_eval(result, include_exceptions=True)
814                                    if eval_results[1] is None:
815                                        result = eval_results[0]
816                                        if unsafe:
817                                            result = wrap_var(result)
818                                    else:
819                                        # FIXME: if the safe_eval raised an error, should we do something with it?
820                                        pass
821
822                        # we only cache in the case where we have a single variable
823                        # name, to make sure we're not putting things which may otherwise
824                        # be dynamic in the cache (filters, lookups, etc.)
825                        if cache and only_one:
826                            self._cached_result[sha1_hash] = result
827
828                return result
829
830            elif is_sequence(variable):
831                return [self.template(
832                    v,
833                    preserve_trailing_newlines=preserve_trailing_newlines,
834                    fail_on_undefined=fail_on_undefined,
835                    overrides=overrides,
836                    disable_lookups=disable_lookups,
837                ) for v in variable]
838            elif isinstance(variable, Mapping):
839                d = {}
840                # we don't use iteritems() here to avoid problems if the underlying dict
841                # changes sizes due to the templating, which can happen with hostvars
842                for k in variable.keys():
843                    if k not in static_vars:
844                        d[k] = self.template(
845                            variable[k],
846                            preserve_trailing_newlines=preserve_trailing_newlines,
847                            fail_on_undefined=fail_on_undefined,
848                            overrides=overrides,
849                            disable_lookups=disable_lookups,
850                        )
851                    else:
852                        d[k] = variable[k]
853                return d
854            else:
855                return variable
856
857        except AnsibleFilterError:
858            if self._fail_on_filter_errors:
859                raise
860            else:
861                return variable
862
863    def is_template(self, data):
864        '''lets us know if data has a template'''
865        if isinstance(data, string_types):
866            return is_template(data, self.environment)
867        elif isinstance(data, (list, tuple)):
868            for v in data:
869                if self.is_template(v):
870                    return True
871        elif isinstance(data, dict):
872            for k in data:
873                if self.is_template(k) or self.is_template(data[k]):
874                    return True
875        return False
876
877    templatable = is_template
878
879    def is_possibly_template(self, data):
880        '''Determines if a string looks like a template, by seeing if it
881        contains a jinja2 start delimiter. Does not guarantee that the string
882        is actually a template.
883
884        This is different than ``is_template`` which is more strict.
885        This method may return ``True`` on a string that is not templatable.
886
887        Useful when guarding passing a string for templating, but when
888        you want to allow the templating engine to make the final
889        assessment which may result in ``TemplateSyntaxError``.
890        '''
891        env = self.environment
892        if isinstance(data, string_types):
893            for marker in (env.block_start_string, env.variable_start_string, env.comment_start_string):
894                if marker in data:
895                    return True
896        return False
897
898    def _convert_bare_variable(self, variable):
899        '''
900        Wraps a bare string, which may have an attribute portion (ie. foo.bar)
901        in jinja2 variable braces so that it is evaluated properly.
902        '''
903
904        if isinstance(variable, string_types):
905            contains_filters = "|" in variable
906            first_part = variable.split("|")[0].split(".")[0].split("[")[0]
907            if (contains_filters or first_part in self._available_variables) and self.environment.variable_start_string not in variable:
908                return "%s%s%s" % (self.environment.variable_start_string, variable, self.environment.variable_end_string)
909
910        # the variable didn't meet the conditions to be converted,
911        # so just return it as-is
912        return variable
913
914    def _finalize(self, thing):
915        '''
916        A custom finalize method for jinja2, which prevents None from being returned. This
917        avoids a string of ``"None"`` as ``None`` has no importance in YAML.
918
919        If using ANSIBLE_JINJA2_NATIVE we bypass this and return the actual value always
920        '''
921        if _is_rolled(thing):
922            # Auto unroll a generator, so that users are not required to
923            # explicitly use ``|list`` to unroll
924            # This only affects the scenario where the final result of templating
925            # is a generator, and not where a filter creates a generator in the middle
926            # of a template. See ``_unroll_iterator`` for the other case. This is probably
927            # unncessary
928            return list(thing)
929
930        if USE_JINJA2_NATIVE:
931            return thing
932
933        return thing if thing is not None else ''
934
935    def _fail_lookup(self, name, *args, **kwargs):
936        raise AnsibleError("The lookup `%s` was found, however lookups were disabled from templating" % name)
937
938    def _now_datetime(self, utc=False, fmt=None):
939        '''jinja2 global function to return current datetime, potentially formatted via strftime'''
940        if utc:
941            now = datetime.datetime.utcnow()
942        else:
943            now = datetime.datetime.now()
944
945        if fmt:
946            return now.strftime(fmt)
947
948        return now
949
950    def _query_lookup(self, name, *args, **kwargs):
951        ''' wrapper for lookup, force wantlist true'''
952        kwargs['wantlist'] = True
953        return self._lookup(name, *args, **kwargs)
954
955    def _lookup(self, name, *args, **kwargs):
956        instance = self._lookup_loader.get(name, loader=self._loader, templar=self)
957
958        if instance is not None:
959            wantlist = kwargs.pop('wantlist', False)
960            allow_unsafe = kwargs.pop('allow_unsafe', C.DEFAULT_ALLOW_UNSAFE_LOOKUPS)
961            errors = kwargs.pop('errors', 'strict')
962
963            from ansible.utils.listify import listify_lookup_plugin_terms
964            loop_terms = listify_lookup_plugin_terms(terms=args, templar=self, loader=self._loader, fail_on_undefined=True, convert_bare=False)
965            # safely catch run failures per #5059
966            try:
967                ran = instance.run(loop_terms, variables=self._available_variables, **kwargs)
968            except (AnsibleUndefinedVariable, UndefinedError) as e:
969                raise AnsibleUndefinedVariable(e)
970            except Exception as e:
971                if self._fail_on_lookup_errors:
972                    msg = u"An unhandled exception occurred while running the lookup plugin '%s'. Error was a %s, original message: %s" % \
973                          (name, type(e), to_text(e))
974                    if errors == 'warn':
975                        display.warning(msg)
976                    elif errors == 'ignore':
977                        display.display(msg, log_only=True)
978                    else:
979                        raise AnsibleError(to_native(msg))
980                ran = [] if wantlist else None
981
982            if ran and not allow_unsafe:
983                if wantlist:
984                    ran = wrap_var(ran)
985                else:
986                    try:
987                        ran = wrap_var(",".join(ran))
988                    except TypeError:
989                        # Lookup Plugins should always return lists.  Throw an error if that's not
990                        # the case:
991                        if not isinstance(ran, Sequence):
992                            raise AnsibleError("The lookup plugin '%s' did not return a list."
993                                               % name)
994
995                        # The TypeError we can recover from is when the value *inside* of the list
996                        # is not a string
997                        if len(ran) == 1:
998                            ran = wrap_var(ran[0])
999                        else:
1000                            ran = wrap_var(ran)
1001
1002                if self.cur_context:
1003                    self.cur_context.unsafe = True
1004            return ran
1005        else:
1006            raise AnsibleError("lookup plugin (%s) not found" % name)
1007
1008    def do_template(self, data, preserve_trailing_newlines=True, escape_backslashes=True, fail_on_undefined=None, overrides=None, disable_lookups=False):
1009        if USE_JINJA2_NATIVE and not isinstance(data, string_types):
1010            return data
1011
1012        # For preserving the number of input newlines in the output (used
1013        # later in this method)
1014        data_newlines = _count_newlines_from_end(data)
1015
1016        if fail_on_undefined is None:
1017            fail_on_undefined = self._fail_on_undefined_errors
1018
1019        try:
1020            # allows template header overrides to change jinja2 options.
1021            if overrides is None:
1022                myenv = self.environment.overlay()
1023            else:
1024                myenv = self.environment.overlay(overrides)
1025
1026            # Get jinja env overrides from template
1027            if hasattr(data, 'startswith') and data.startswith(JINJA2_OVERRIDE):
1028                eol = data.find('\n')
1029                line = data[len(JINJA2_OVERRIDE):eol]
1030                data = data[eol + 1:]
1031                for pair in line.split(','):
1032                    (key, val) = pair.split(':')
1033                    key = key.strip()
1034                    setattr(myenv, key, ast.literal_eval(val.strip()))
1035
1036            # Adds Ansible custom filters and tests
1037            myenv.filters.update(self._get_filters())
1038            for k in myenv.filters:
1039                if not getattr(myenv.filters[k], '__UNROLLED__', False):
1040                    myenv.filters[k] = _unroll_iterator(myenv.filters[k])
1041            myenv.tests.update(self._get_tests())
1042
1043            if escape_backslashes:
1044                # Allow users to specify backslashes in playbooks as "\\" instead of as "\\\\".
1045                data = _escape_backslashes(data, myenv)
1046
1047            try:
1048                t = myenv.from_string(data)
1049            except TemplateSyntaxError as e:
1050                raise AnsibleError("template error while templating string: %s. String: %s" % (to_native(e), to_native(data)))
1051            except Exception as e:
1052                if 'recursion' in to_native(e):
1053                    raise AnsibleError("recursive loop detected in template string: %s" % to_native(data))
1054                else:
1055                    return data
1056
1057            if disable_lookups:
1058                t.globals['query'] = t.globals['q'] = t.globals['lookup'] = self._fail_lookup
1059
1060            jvars = AnsibleJ2Vars(self, t.globals)
1061
1062            self.cur_context = new_context = t.new_context(jvars, shared=True)
1063            rf = t.root_render_func(new_context)
1064
1065            try:
1066                res = j2_concat(rf)
1067                unsafe = getattr(new_context, 'unsafe', False)
1068                if unsafe:
1069                    res = wrap_var(res)
1070            except TypeError as te:
1071                if 'AnsibleUndefined' in to_native(te):
1072                    errmsg = "Unable to look up a name or access an attribute in template string (%s).\n" % to_native(data)
1073                    errmsg += "Make sure your variable name does not contain invalid characters like '-': %s" % to_native(te)
1074                    raise AnsibleUndefinedVariable(errmsg)
1075                else:
1076                    display.debug("failing because of a type error, template data is: %s" % to_text(data))
1077                    raise AnsibleError("Unexpected templating type error occurred on (%s): %s" % (to_native(data), to_native(te)))
1078
1079            if USE_JINJA2_NATIVE and not isinstance(res, string_types):
1080                return res
1081
1082            if preserve_trailing_newlines:
1083                # The low level calls above do not preserve the newline
1084                # characters at the end of the input data, so we use the
1085                # calculate the difference in newlines and append them
1086                # to the resulting output for parity
1087                #
1088                # jinja2 added a keep_trailing_newline option in 2.7 when
1089                # creating an Environment.  That would let us make this code
1090                # better (remove a single newline if
1091                # preserve_trailing_newlines is False).  Once we can depend on
1092                # that version being present, modify our code to set that when
1093                # initializing self.environment and remove a single trailing
1094                # newline here if preserve_newlines is False.
1095                res_newlines = _count_newlines_from_end(res)
1096                if data_newlines > res_newlines:
1097                    res += self.environment.newline_sequence * (data_newlines - res_newlines)
1098                    if unsafe:
1099                        res = wrap_var(res)
1100            return res
1101        except (UndefinedError, AnsibleUndefinedVariable) as e:
1102            if fail_on_undefined:
1103                raise AnsibleUndefinedVariable(e)
1104            else:
1105                display.debug("Ignoring undefined failure: %s" % to_text(e))
1106                return data
1107
1108    # for backwards compatibility in case anyone is using old private method directly
1109    _do_template = do_template
1110