1# mako/template.py
2# Copyright 2006-2019 the Mako authors and contributors <see AUTHORS file>
3#
4# This module is part of Mako and is released under
5# the MIT License: http://www.opensource.org/licenses/mit-license.php
6
7"""Provides the Template class, a facade for parsing, generating and executing
8template strings, as well as template runtime operations."""
9
10import os
11import re
12import shutil
13import stat
14import sys
15import tempfile
16import types
17import weakref
18
19from mako import cache
20from mako import codegen
21from mako import compat
22from mako import exceptions
23from mako import runtime
24from mako import util
25from mako.lexer import Lexer
26
27
28class Template(object):
29
30    r"""Represents a compiled template.
31
32    :class:`.Template` includes a reference to the original
33    template source (via the :attr:`.source` attribute)
34    as well as the source code of the
35    generated Python module (i.e. the :attr:`.code` attribute),
36    as well as a reference to an actual Python module.
37
38    :class:`.Template` is constructed using either a literal string
39    representing the template text, or a filename representing a filesystem
40    path to a source file.
41
42    :param text: textual template source.  This argument is mutually
43     exclusive versus the ``filename`` parameter.
44
45    :param filename: filename of the source template.  This argument is
46     mutually exclusive versus the ``text`` parameter.
47
48    :param buffer_filters: string list of filters to be applied
49     to the output of ``%def``\ s which are buffered, cached, or otherwise
50     filtered, after all filters
51     defined with the ``%def`` itself have been applied. Allows the
52     creation of default expression filters that let the output
53     of return-valued ``%def``\ s "opt out" of that filtering via
54     passing special attributes or objects.
55
56    :param bytestring_passthrough: When ``True``, and ``output_encoding`` is
57     set to ``None``, and :meth:`.Template.render` is used to render,
58     the `StringIO` or `cStringIO` buffer will be used instead of the
59     default "fast" buffer.   This allows raw bytestrings in the
60     output stream, such as in expressions, to pass straight
61     through to the buffer.  This flag is forced
62     to ``True`` if ``disable_unicode`` is also configured.
63
64     .. versionadded:: 0.4
65        Added to provide the same behavior as that of the previous series.
66
67    :param cache_args: Dictionary of cache configuration arguments that
68     will be passed to the :class:`.CacheImpl`.   See :ref:`caching_toplevel`.
69
70    :param cache_dir:
71
72     .. deprecated:: 0.6
73        Use the ``'dir'`` argument in the ``cache_args`` dictionary.
74        See :ref:`caching_toplevel`.
75
76    :param cache_enabled: Boolean flag which enables caching of this
77     template.  See :ref:`caching_toplevel`.
78
79    :param cache_impl: String name of a :class:`.CacheImpl` caching
80     implementation to use.   Defaults to ``'beaker'``.
81
82    :param cache_type:
83
84     .. deprecated:: 0.6
85        Use the ``'type'`` argument in the ``cache_args`` dictionary.
86        See :ref:`caching_toplevel`.
87
88    :param cache_url:
89
90     .. deprecated:: 0.6
91        Use the ``'url'`` argument in the ``cache_args`` dictionary.
92        See :ref:`caching_toplevel`.
93
94    :param default_filters: List of string filter names that will
95     be applied to all expressions.  See :ref:`filtering_default_filters`.
96
97    :param disable_unicode: Disables all awareness of Python Unicode
98     objects.  See :ref:`unicode_disabled`.
99
100    :param enable_loop: When ``True``, enable the ``loop`` context variable.
101     This can be set to ``False`` to support templates that may
102     be making usage of the name "``loop``".   Individual templates can
103     re-enable the "loop" context by placing the directive
104     ``enable_loop="True"`` inside the ``<%page>`` tag -- see
105     :ref:`migrating_loop`.
106
107    :param encoding_errors: Error parameter passed to ``encode()`` when
108     string encoding is performed. See :ref:`usage_unicode`.
109
110    :param error_handler: Python callable which is called whenever
111     compile or runtime exceptions occur. The callable is passed
112     the current context as well as the exception. If the
113     callable returns ``True``, the exception is considered to
114     be handled, else it is re-raised after the function
115     completes. Is used to provide custom error-rendering
116     functions.
117
118     .. seealso::
119
120        :paramref:`.Template.include_error_handler` - include-specific
121        error handler function
122
123    :param format_exceptions: if ``True``, exceptions which occur during
124     the render phase of this template will be caught and
125     formatted into an HTML error page, which then becomes the
126     rendered result of the :meth:`.render` call. Otherwise,
127     runtime exceptions are propagated outwards.
128
129    :param imports: String list of Python statements, typically individual
130     "import" lines, which will be placed into the module level
131     preamble of all generated Python modules. See the example
132     in :ref:`filtering_default_filters`.
133
134    :param future_imports: String list of names to import from `__future__`.
135     These will be concatenated into a comma-separated string and inserted
136     into the beginning of the template, e.g. ``futures_imports=['FOO',
137     'BAR']`` results in ``from __future__ import FOO, BAR``.  If you're
138     interested in using features like the new division operator, you must
139     use future_imports to convey that to the renderer, as otherwise the
140     import will not appear as the first executed statement in the generated
141     code and will therefore not have the desired effect.
142
143    :param include_error_handler: An error handler that runs when this template
144     is included within another one via the ``<%include>`` tag, and raises an
145     error.  Compare to the :paramref:`.Template.error_handler` option.
146
147     .. versionadded:: 1.0.6
148
149     .. seealso::
150
151        :paramref:`.Template.error_handler` - top-level error handler function
152
153    :param input_encoding: Encoding of the template's source code.  Can
154     be used in lieu of the coding comment. See
155     :ref:`usage_unicode` as well as :ref:`unicode_toplevel` for
156     details on source encoding.
157
158    :param lookup: a :class:`.TemplateLookup` instance that will be used
159     for all file lookups via the ``<%namespace>``,
160     ``<%include>``, and ``<%inherit>`` tags. See
161     :ref:`usage_templatelookup`.
162
163    :param module_directory: Filesystem location where generated
164     Python module files will be placed.
165
166    :param module_filename: Overrides the filename of the generated
167     Python module file. For advanced usage only.
168
169    :param module_writer: A callable which overrides how the Python
170     module is written entirely.  The callable is passed the
171     encoded source content of the module and the destination
172     path to be written to.   The default behavior of module writing
173     uses a tempfile in conjunction with a file move in order
174     to make the operation atomic.   So a user-defined module
175     writing function that mimics the default behavior would be:
176
177     .. sourcecode:: python
178
179         import tempfile
180         import os
181         import shutil
182
183         def module_writer(source, outputpath):
184             (dest, name) = \\
185                 tempfile.mkstemp(
186                     dir=os.path.dirname(outputpath)
187                 )
188
189             os.write(dest, source)
190             os.close(dest)
191             shutil.move(name, outputpath)
192
193         from mako.template import Template
194         mytemplate = Template(
195                         filename="index.html",
196                         module_directory="/path/to/modules",
197                         module_writer=module_writer
198                     )
199
200     The function is provided for unusual configurations where
201     certain platform-specific permissions or other special
202     steps are needed.
203
204    :param output_encoding: The encoding to use when :meth:`.render`
205     is called.
206     See :ref:`usage_unicode` as well as :ref:`unicode_toplevel`.
207
208    :param preprocessor: Python callable which will be passed
209     the full template source before it is parsed. The return
210     result of the callable will be used as the template source
211     code.
212
213    :param lexer_cls: A :class:`.Lexer` class used to parse
214     the template.   The :class:`.Lexer` class is used by
215     default.
216
217     .. versionadded:: 0.7.4
218
219    :param strict_undefined: Replaces the automatic usage of
220     ``UNDEFINED`` for any undeclared variables not located in
221     the :class:`.Context` with an immediate raise of
222     ``NameError``. The advantage is immediate reporting of
223     missing variables which include the name.
224
225     .. versionadded:: 0.3.6
226
227    :param uri: string URI or other identifier for this template.
228     If not provided, the ``uri`` is generated from the filesystem
229     path, or from the in-memory identity of a non-file-based
230     template. The primary usage of the ``uri`` is to provide a key
231     within :class:`.TemplateLookup`, as well as to generate the
232     file path of the generated Python module file, if
233     ``module_directory`` is specified.
234
235    """
236
237    lexer_cls = Lexer
238
239    def __init__(
240        self,
241        text=None,
242        filename=None,
243        uri=None,
244        format_exceptions=False,
245        error_handler=None,
246        lookup=None,
247        output_encoding=None,
248        encoding_errors="strict",
249        module_directory=None,
250        cache_args=None,
251        cache_impl="beaker",
252        cache_enabled=True,
253        cache_type=None,
254        cache_dir=None,
255        cache_url=None,
256        module_filename=None,
257        input_encoding=None,
258        disable_unicode=False,
259        module_writer=None,
260        bytestring_passthrough=False,
261        default_filters=None,
262        buffer_filters=(),
263        strict_undefined=False,
264        imports=None,
265        future_imports=None,
266        enable_loop=True,
267        preprocessor=None,
268        lexer_cls=None,
269        include_error_handler=None,
270    ):
271        if uri:
272            self.module_id = re.sub(r"\W", "_", uri)
273            self.uri = uri
274        elif filename:
275            self.module_id = re.sub(r"\W", "_", filename)
276            drive, path = os.path.splitdrive(filename)
277            path = os.path.normpath(path).replace(os.path.sep, "/")
278            self.uri = path
279        else:
280            self.module_id = "memory:" + hex(id(self))
281            self.uri = self.module_id
282
283        u_norm = self.uri
284        if u_norm.startswith("/"):
285            u_norm = u_norm[1:]
286        u_norm = os.path.normpath(u_norm)
287        if u_norm.startswith(".."):
288            raise exceptions.TemplateLookupException(
289                'Template uri "%s" is invalid - '
290                "it cannot be relative outside "
291                "of the root path." % self.uri
292            )
293
294        self.input_encoding = input_encoding
295        self.output_encoding = output_encoding
296        self.encoding_errors = encoding_errors
297        self.disable_unicode = disable_unicode
298        self.bytestring_passthrough = bytestring_passthrough or disable_unicode
299        self.enable_loop = enable_loop
300        self.strict_undefined = strict_undefined
301        self.module_writer = module_writer
302
303        if compat.py3k and disable_unicode:
304            raise exceptions.UnsupportedError(
305                "Mako for Python 3 does not " "support disabling Unicode"
306            )
307        elif output_encoding and disable_unicode:
308            raise exceptions.UnsupportedError(
309                "output_encoding must be set to "
310                "None when disable_unicode is used."
311            )
312        if default_filters is None:
313            if compat.py3k or self.disable_unicode:
314                self.default_filters = ["str"]
315            else:
316                self.default_filters = ["unicode"]
317        else:
318            self.default_filters = default_filters
319        self.buffer_filters = buffer_filters
320
321        self.imports = imports
322        self.future_imports = future_imports
323        self.preprocessor = preprocessor
324
325        if lexer_cls is not None:
326            self.lexer_cls = lexer_cls
327
328        # if plain text, compile code in memory only
329        if text is not None:
330            (code, module) = _compile_text(self, text, filename)
331            self._code = code
332            self._source = text
333            ModuleInfo(module, None, self, filename, code, text, uri)
334        elif filename is not None:
335            # if template filename and a module directory, load
336            # a filesystem-based module file, generating if needed
337            if module_filename is not None:
338                path = module_filename
339            elif module_directory is not None:
340                path = os.path.abspath(
341                    os.path.join(
342                        os.path.normpath(module_directory), u_norm + ".py"
343                    )
344                )
345            else:
346                path = None
347            module = self._compile_from_file(path, filename)
348        else:
349            raise exceptions.RuntimeException(
350                "Template requires text or filename"
351            )
352
353        self.module = module
354        self.filename = filename
355        self.callable_ = self.module.render_body
356        self.format_exceptions = format_exceptions
357        self.error_handler = error_handler
358        self.include_error_handler = include_error_handler
359        self.lookup = lookup
360
361        self.module_directory = module_directory
362
363        self._setup_cache_args(
364            cache_impl,
365            cache_enabled,
366            cache_args,
367            cache_type,
368            cache_dir,
369            cache_url,
370        )
371
372    @util.memoized_property
373    def reserved_names(self):
374        if self.enable_loop:
375            return codegen.RESERVED_NAMES
376        else:
377            return codegen.RESERVED_NAMES.difference(["loop"])
378
379    def _setup_cache_args(
380        self,
381        cache_impl,
382        cache_enabled,
383        cache_args,
384        cache_type,
385        cache_dir,
386        cache_url,
387    ):
388        self.cache_impl = cache_impl
389        self.cache_enabled = cache_enabled
390        if cache_args:
391            self.cache_args = cache_args
392        else:
393            self.cache_args = {}
394
395        # transfer deprecated cache_* args
396        if cache_type:
397            self.cache_args["type"] = cache_type
398        if cache_dir:
399            self.cache_args["dir"] = cache_dir
400        if cache_url:
401            self.cache_args["url"] = cache_url
402
403    def _compile_from_file(self, path, filename):
404        if path is not None:
405            util.verify_directory(os.path.dirname(path))
406            filemtime = os.stat(filename)[stat.ST_MTIME]
407            if (
408                not os.path.exists(path)
409                or os.stat(path)[stat.ST_MTIME] < filemtime
410            ):
411                data = util.read_file(filename)
412                _compile_module_file(
413                    self, data, filename, path, self.module_writer
414                )
415            module = compat.load_module(self.module_id, path)
416            del sys.modules[self.module_id]
417            if module._magic_number != codegen.MAGIC_NUMBER:
418                data = util.read_file(filename)
419                _compile_module_file(
420                    self, data, filename, path, self.module_writer
421                )
422                module = compat.load_module(self.module_id, path)
423                del sys.modules[self.module_id]
424            ModuleInfo(module, path, self, filename, None, None, None)
425        else:
426            # template filename and no module directory, compile code
427            # in memory
428            data = util.read_file(filename)
429            code, module = _compile_text(self, data, filename)
430            self._source = None
431            self._code = code
432            ModuleInfo(module, None, self, filename, code, None, None)
433        return module
434
435    @property
436    def source(self):
437        """Return the template source code for this :class:`.Template`."""
438
439        return _get_module_info_from_callable(self.callable_).source
440
441    @property
442    def code(self):
443        """Return the module source code for this :class:`.Template`."""
444
445        return _get_module_info_from_callable(self.callable_).code
446
447    @util.memoized_property
448    def cache(self):
449        return cache.Cache(self)
450
451    @property
452    def cache_dir(self):
453        return self.cache_args["dir"]
454
455    @property
456    def cache_url(self):
457        return self.cache_args["url"]
458
459    @property
460    def cache_type(self):
461        return self.cache_args["type"]
462
463    def render(self, *args, **data):
464        """Render the output of this template as a string.
465
466        If the template specifies an output encoding, the string
467        will be encoded accordingly, else the output is raw (raw
468        output uses `cStringIO` and can't handle multibyte
469        characters). A :class:`.Context` object is created corresponding
470        to the given data. Arguments that are explicitly declared
471        by this template's internal rendering method are also
472        pulled from the given ``*args``, ``**data`` members.
473
474        """
475        return runtime._render(self, self.callable_, args, data)
476
477    def render_unicode(self, *args, **data):
478        """Render the output of this template as a unicode object."""
479
480        return runtime._render(
481            self, self.callable_, args, data, as_unicode=True
482        )
483
484    def render_context(self, context, *args, **kwargs):
485        """Render this :class:`.Template` with the given context.
486
487        The data is written to the context's buffer.
488
489        """
490        if getattr(context, "_with_template", None) is None:
491            context._set_with_template(self)
492        runtime._render_context(self, self.callable_, context, *args, **kwargs)
493
494    def has_def(self, name):
495        return hasattr(self.module, "render_%s" % name)
496
497    def get_def(self, name):
498        """Return a def of this template as a :class:`.DefTemplate`."""
499
500        return DefTemplate(self, getattr(self.module, "render_%s" % name))
501
502    def list_defs(self):
503        """return a list of defs in the template.
504
505        .. versionadded:: 1.0.4
506
507        """
508        return [i[7:] for i in dir(self.module) if i[:7] == "render_"]
509
510    def _get_def_callable(self, name):
511        return getattr(self.module, "render_%s" % name)
512
513    @property
514    def last_modified(self):
515        return self.module._modified_time
516
517
518class ModuleTemplate(Template):
519
520    """A Template which is constructed given an existing Python module.
521
522        e.g.::
523
524        t = Template("this is a template")
525        f = file("mymodule.py", "w")
526        f.write(t.code)
527        f.close()
528
529        import mymodule
530
531        t = ModuleTemplate(mymodule)
532        print t.render()
533
534    """
535
536    def __init__(
537        self,
538        module,
539        module_filename=None,
540        template=None,
541        template_filename=None,
542        module_source=None,
543        template_source=None,
544        output_encoding=None,
545        encoding_errors="strict",
546        disable_unicode=False,
547        bytestring_passthrough=False,
548        format_exceptions=False,
549        error_handler=None,
550        lookup=None,
551        cache_args=None,
552        cache_impl="beaker",
553        cache_enabled=True,
554        cache_type=None,
555        cache_dir=None,
556        cache_url=None,
557        include_error_handler=None,
558    ):
559        self.module_id = re.sub(r"\W", "_", module._template_uri)
560        self.uri = module._template_uri
561        self.input_encoding = module._source_encoding
562        self.output_encoding = output_encoding
563        self.encoding_errors = encoding_errors
564        self.disable_unicode = disable_unicode
565        self.bytestring_passthrough = bytestring_passthrough or disable_unicode
566        self.enable_loop = module._enable_loop
567
568        if compat.py3k and disable_unicode:
569            raise exceptions.UnsupportedError(
570                "Mako for Python 3 does not " "support disabling Unicode"
571            )
572        elif output_encoding and disable_unicode:
573            raise exceptions.UnsupportedError(
574                "output_encoding must be set to "
575                "None when disable_unicode is used."
576            )
577
578        self.module = module
579        self.filename = template_filename
580        ModuleInfo(
581            module,
582            module_filename,
583            self,
584            template_filename,
585            module_source,
586            template_source,
587            module._template_uri,
588        )
589
590        self.callable_ = self.module.render_body
591        self.format_exceptions = format_exceptions
592        self.error_handler = error_handler
593        self.include_error_handler = include_error_handler
594        self.lookup = lookup
595        self._setup_cache_args(
596            cache_impl,
597            cache_enabled,
598            cache_args,
599            cache_type,
600            cache_dir,
601            cache_url,
602        )
603
604
605class DefTemplate(Template):
606
607    """A :class:`.Template` which represents a callable def in a parent
608    template."""
609
610    def __init__(self, parent, callable_):
611        self.parent = parent
612        self.callable_ = callable_
613        self.output_encoding = parent.output_encoding
614        self.module = parent.module
615        self.encoding_errors = parent.encoding_errors
616        self.format_exceptions = parent.format_exceptions
617        self.error_handler = parent.error_handler
618        self.include_error_handler = parent.include_error_handler
619        self.enable_loop = parent.enable_loop
620        self.lookup = parent.lookup
621        self.bytestring_passthrough = parent.bytestring_passthrough
622
623    def get_def(self, name):
624        return self.parent.get_def(name)
625
626
627class ModuleInfo(object):
628
629    """Stores information about a module currently loaded into
630    memory, provides reverse lookups of template source, module
631    source code based on a module's identifier.
632
633     """
634
635    _modules = weakref.WeakValueDictionary()
636
637    def __init__(
638        self,
639        module,
640        module_filename,
641        template,
642        template_filename,
643        module_source,
644        template_source,
645        template_uri,
646    ):
647        self.module = module
648        self.module_filename = module_filename
649        self.template_filename = template_filename
650        self.module_source = module_source
651        self.template_source = template_source
652        self.template_uri = template_uri
653        self._modules[module.__name__] = template._mmarker = self
654        if module_filename:
655            self._modules[module_filename] = self
656
657    @classmethod
658    def get_module_source_metadata(cls, module_source, full_line_map=False):
659        source_map = re.search(
660            r"__M_BEGIN_METADATA(.+?)__M_END_METADATA", module_source, re.S
661        ).group(1)
662        source_map = compat.json.loads(source_map)
663        source_map["line_map"] = dict(
664            (int(k), int(v)) for k, v in source_map["line_map"].items()
665        )
666        if full_line_map:
667            f_line_map = source_map["full_line_map"] = []
668            line_map = source_map["line_map"]
669
670            curr_templ_line = 1
671            for mod_line in range(1, max(line_map)):
672                if mod_line in line_map:
673                    curr_templ_line = line_map[mod_line]
674                f_line_map.append(curr_templ_line)
675        return source_map
676
677    @property
678    def code(self):
679        if self.module_source is not None:
680            return self.module_source
681        else:
682            return util.read_python_file(self.module_filename)
683
684    @property
685    def source(self):
686        if self.template_source is not None:
687            if self.module._source_encoding and not isinstance(
688                self.template_source, compat.text_type
689            ):
690                return self.template_source.decode(
691                    self.module._source_encoding
692                )
693            else:
694                return self.template_source
695        else:
696            data = util.read_file(self.template_filename)
697            if self.module._source_encoding:
698                return data.decode(self.module._source_encoding)
699            else:
700                return data
701
702
703def _compile(template, text, filename, generate_magic_comment):
704    lexer = template.lexer_cls(
705        text,
706        filename,
707        disable_unicode=template.disable_unicode,
708        input_encoding=template.input_encoding,
709        preprocessor=template.preprocessor,
710    )
711    node = lexer.parse()
712    source = codegen.compile(
713        node,
714        template.uri,
715        filename,
716        default_filters=template.default_filters,
717        buffer_filters=template.buffer_filters,
718        imports=template.imports,
719        future_imports=template.future_imports,
720        source_encoding=lexer.encoding,
721        generate_magic_comment=generate_magic_comment,
722        disable_unicode=template.disable_unicode,
723        strict_undefined=template.strict_undefined,
724        enable_loop=template.enable_loop,
725        reserved_names=template.reserved_names,
726    )
727    return source, lexer
728
729
730def _compile_text(template, text, filename):
731    identifier = template.module_id
732    source, lexer = _compile(
733        template,
734        text,
735        filename,
736        generate_magic_comment=template.disable_unicode,
737    )
738
739    cid = identifier
740    if not compat.py3k and isinstance(cid, compat.text_type):
741        cid = cid.encode()
742    module = types.ModuleType(cid)
743    code = compile(source, cid, "exec")
744
745    # this exec() works for 2.4->3.3.
746    exec(code, module.__dict__, module.__dict__)
747    return (source, module)
748
749
750def _compile_module_file(template, text, filename, outputpath, module_writer):
751    source, lexer = _compile(
752        template, text, filename, generate_magic_comment=True
753    )
754
755    if isinstance(source, compat.text_type):
756        source = source.encode(lexer.encoding or "ascii")
757
758    if module_writer:
759        module_writer(source, outputpath)
760    else:
761        # make tempfiles in the same location as the ultimate
762        # location.   this ensures they're on the same filesystem,
763        # avoiding synchronization issues.
764        (dest, name) = tempfile.mkstemp(dir=os.path.dirname(outputpath))
765
766        os.write(dest, source)
767        os.close(dest)
768        shutil.move(name, outputpath)
769
770
771def _get_module_info_from_callable(callable_):
772    if compat.py3k:
773        return _get_module_info(callable_.__globals__["__name__"])
774    else:
775        return _get_module_info(callable_.func_globals["__name__"])
776
777
778def _get_module_info(filename):
779    return ModuleInfo._modules[filename]
780