1# -*- coding: utf-8 -*-
2"""The optional bytecode cache system. This is useful if you have very
3complex template situations and the compilation of all those templates
4slows down your application too much.
5
6Situations where this is useful are often forking web applications that
7are initialized on the first request.
8"""
9import errno
10import fnmatch
11import os
12import stat
13import sys
14import tempfile
15from hashlib import sha1
16from os import listdir
17from os import path
18
19from ._compat import BytesIO
20from ._compat import marshal_dump
21from ._compat import marshal_load
22from ._compat import pickle
23from ._compat import text_type
24from .utils import open_if_exists
25
26bc_version = 4
27# Magic bytes to identify Jinja bytecode cache files. Contains the
28# Python major and minor version to avoid loading incompatible bytecode
29# if a project upgrades its Python version.
30bc_magic = (
31    b"j2"
32    + pickle.dumps(bc_version, 2)
33    + pickle.dumps((sys.version_info[0] << 24) | sys.version_info[1], 2)
34)
35
36
37class Bucket(object):
38    """Buckets are used to store the bytecode for one template.  It's created
39    and initialized by the bytecode cache and passed to the loading functions.
40
41    The buckets get an internal checksum from the cache assigned and use this
42    to automatically reject outdated cache material.  Individual bytecode
43    cache subclasses don't have to care about cache invalidation.
44    """
45
46    def __init__(self, environment, key, checksum):
47        self.environment = environment
48        self.key = key
49        self.checksum = checksum
50        self.reset()
51
52    def reset(self):
53        """Resets the bucket (unloads the bytecode)."""
54        self.code = None
55
56    def load_bytecode(self, f):
57        """Loads bytecode from a file or file like object."""
58        # make sure the magic header is correct
59        magic = f.read(len(bc_magic))
60        if magic != bc_magic:
61            self.reset()
62            return
63        # the source code of the file changed, we need to reload
64        checksum = pickle.load(f)
65        if self.checksum != checksum:
66            self.reset()
67            return
68        # if marshal_load fails then we need to reload
69        try:
70            self.code = marshal_load(f)
71        except (EOFError, ValueError, TypeError):
72            self.reset()
73            return
74
75    def write_bytecode(self, f):
76        """Dump the bytecode into the file or file like object passed."""
77        if self.code is None:
78            raise TypeError("can't write empty bucket")
79        f.write(bc_magic)
80        pickle.dump(self.checksum, f, 2)
81        marshal_dump(self.code, f)
82
83    def bytecode_from_string(self, string):
84        """Load bytecode from a string."""
85        self.load_bytecode(BytesIO(string))
86
87    def bytecode_to_string(self):
88        """Return the bytecode as string."""
89        out = BytesIO()
90        self.write_bytecode(out)
91        return out.getvalue()
92
93
94class BytecodeCache(object):
95    """To implement your own bytecode cache you have to subclass this class
96    and override :meth:`load_bytecode` and :meth:`dump_bytecode`.  Both of
97    these methods are passed a :class:`~jinja2.bccache.Bucket`.
98
99    A very basic bytecode cache that saves the bytecode on the file system::
100
101        from os import path
102
103        class MyCache(BytecodeCache):
104
105            def __init__(self, directory):
106                self.directory = directory
107
108            def load_bytecode(self, bucket):
109                filename = path.join(self.directory, bucket.key)
110                if path.exists(filename):
111                    with open(filename, 'rb') as f:
112                        bucket.load_bytecode(f)
113
114            def dump_bytecode(self, bucket):
115                filename = path.join(self.directory, bucket.key)
116                with open(filename, 'wb') as f:
117                    bucket.write_bytecode(f)
118
119    A more advanced version of a filesystem based bytecode cache is part of
120    Jinja.
121    """
122
123    def load_bytecode(self, bucket):
124        """Subclasses have to override this method to load bytecode into a
125        bucket.  If they are not able to find code in the cache for the
126        bucket, it must not do anything.
127        """
128        raise NotImplementedError()
129
130    def dump_bytecode(self, bucket):
131        """Subclasses have to override this method to write the bytecode
132        from a bucket back to the cache.  If it unable to do so it must not
133        fail silently but raise an exception.
134        """
135        raise NotImplementedError()
136
137    def clear(self):
138        """Clears the cache.  This method is not used by Jinja but should be
139        implemented to allow applications to clear the bytecode cache used
140        by a particular environment.
141        """
142
143    def get_cache_key(self, name, filename=None):
144        """Returns the unique hash key for this template name."""
145        hash = sha1(name.encode("utf-8"))
146        if filename is not None:
147            filename = "|" + filename
148            if isinstance(filename, text_type):
149                filename = filename.encode("utf-8")
150            hash.update(filename)
151        return hash.hexdigest()
152
153    def get_source_checksum(self, source):
154        """Returns a checksum for the source."""
155        return sha1(source.encode("utf-8")).hexdigest()
156
157    def get_bucket(self, environment, name, filename, source):
158        """Return a cache bucket for the given template.  All arguments are
159        mandatory but filename may be `None`.
160        """
161        key = self.get_cache_key(name, filename)
162        checksum = self.get_source_checksum(source)
163        bucket = Bucket(environment, key, checksum)
164        self.load_bytecode(bucket)
165        return bucket
166
167    def set_bucket(self, bucket):
168        """Put the bucket into the cache."""
169        self.dump_bytecode(bucket)
170
171
172class FileSystemBytecodeCache(BytecodeCache):
173    """A bytecode cache that stores bytecode on the filesystem.  It accepts
174    two arguments: The directory where the cache items are stored and a
175    pattern string that is used to build the filename.
176
177    If no directory is specified a default cache directory is selected.  On
178    Windows the user's temp directory is used, on UNIX systems a directory
179    is created for the user in the system temp directory.
180
181    The pattern can be used to have multiple separate caches operate on the
182    same directory.  The default pattern is ``'__jinja2_%s.cache'``.  ``%s``
183    is replaced with the cache key.
184
185    >>> bcc = FileSystemBytecodeCache('/tmp/jinja_cache', '%s.cache')
186
187    This bytecode cache supports clearing of the cache using the clear method.
188    """
189
190    def __init__(self, directory=None, pattern="__jinja2_%s.cache"):
191        if directory is None:
192            directory = self._get_default_cache_dir()
193        self.directory = directory
194        self.pattern = pattern
195
196    def _get_default_cache_dir(self):
197        def _unsafe_dir():
198            raise RuntimeError(
199                "Cannot determine safe temp directory.  You "
200                "need to explicitly provide one."
201            )
202
203        tmpdir = tempfile.gettempdir()
204
205        # On windows the temporary directory is used specific unless
206        # explicitly forced otherwise.  We can just use that.
207        if os.name == "nt":
208            return tmpdir
209        if not hasattr(os, "getuid"):
210            _unsafe_dir()
211
212        dirname = "_jinja2-cache-%d" % os.getuid()
213        actual_dir = os.path.join(tmpdir, dirname)
214
215        try:
216            os.mkdir(actual_dir, stat.S_IRWXU)
217        except OSError as e:
218            if e.errno != errno.EEXIST:
219                raise
220        try:
221            os.chmod(actual_dir, stat.S_IRWXU)
222            actual_dir_stat = os.lstat(actual_dir)
223            if (
224                actual_dir_stat.st_uid != os.getuid()
225                or not stat.S_ISDIR(actual_dir_stat.st_mode)
226                or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU
227            ):
228                _unsafe_dir()
229        except OSError as e:
230            if e.errno != errno.EEXIST:
231                raise
232
233        actual_dir_stat = os.lstat(actual_dir)
234        if (
235            actual_dir_stat.st_uid != os.getuid()
236            or not stat.S_ISDIR(actual_dir_stat.st_mode)
237            or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU
238        ):
239            _unsafe_dir()
240
241        return actual_dir
242
243    def _get_cache_filename(self, bucket):
244        return path.join(self.directory, self.pattern % bucket.key)
245
246    def load_bytecode(self, bucket):
247        f = open_if_exists(self._get_cache_filename(bucket), "rb")
248        if f is not None:
249            try:
250                bucket.load_bytecode(f)
251            finally:
252                f.close()
253
254    def dump_bytecode(self, bucket):
255        f = open(self._get_cache_filename(bucket), "wb")
256        try:
257            bucket.write_bytecode(f)
258        finally:
259            f.close()
260
261    def clear(self):
262        # imported lazily here because google app-engine doesn't support
263        # write access on the file system and the function does not exist
264        # normally.
265        from os import remove
266
267        files = fnmatch.filter(listdir(self.directory), self.pattern % "*")
268        for filename in files:
269            try:
270                remove(path.join(self.directory, filename))
271            except OSError:
272                pass
273
274
275class MemcachedBytecodeCache(BytecodeCache):
276    """This class implements a bytecode cache that uses a memcache cache for
277    storing the information.  It does not enforce a specific memcache library
278    (tummy's memcache or cmemcache) but will accept any class that provides
279    the minimal interface required.
280
281    Libraries compatible with this class:
282
283    -   `cachelib <https://github.com/pallets/cachelib>`_
284    -   `python-memcached <https://pypi.org/project/python-memcached/>`_
285
286    (Unfortunately the django cache interface is not compatible because it
287    does not support storing binary data, only unicode.  You can however pass
288    the underlying cache client to the bytecode cache which is available
289    as `django.core.cache.cache._client`.)
290
291    The minimal interface for the client passed to the constructor is this:
292
293    .. class:: MinimalClientInterface
294
295        .. method:: set(key, value[, timeout])
296
297            Stores the bytecode in the cache.  `value` is a string and
298            `timeout` the timeout of the key.  If timeout is not provided
299            a default timeout or no timeout should be assumed, if it's
300            provided it's an integer with the number of seconds the cache
301            item should exist.
302
303        .. method:: get(key)
304
305            Returns the value for the cache key.  If the item does not
306            exist in the cache the return value must be `None`.
307
308    The other arguments to the constructor are the prefix for all keys that
309    is added before the actual cache key and the timeout for the bytecode in
310    the cache system.  We recommend a high (or no) timeout.
311
312    This bytecode cache does not support clearing of used items in the cache.
313    The clear method is a no-operation function.
314
315    .. versionadded:: 2.7
316       Added support for ignoring memcache errors through the
317       `ignore_memcache_errors` parameter.
318    """
319
320    def __init__(
321        self,
322        client,
323        prefix="jinja2/bytecode/",
324        timeout=None,
325        ignore_memcache_errors=True,
326    ):
327        self.client = client
328        self.prefix = prefix
329        self.timeout = timeout
330        self.ignore_memcache_errors = ignore_memcache_errors
331
332    def load_bytecode(self, bucket):
333        try:
334            code = self.client.get(self.prefix + bucket.key)
335        except Exception:
336            if not self.ignore_memcache_errors:
337                raise
338            code = None
339        if code is not None:
340            bucket.bytecode_from_string(code)
341
342    def dump_bytecode(self, bucket):
343        args = (self.prefix + bucket.key, bucket.bytecode_to_string())
344        if self.timeout is not None:
345            args += (self.timeout,)
346        try:
347            self.client.set(*args)
348        except Exception:
349            if not self.ignore_memcache_errors:
350                raise
351