1# -*- coding: utf-8 -*-
2"""
3    jinja2.bccache
4    ~~~~~~~~~~~~~~
5
6    This module implements the bytecode cache system Jinja is optionally
7    using.  This is useful if you have very complex template situations and
8    the compiliation of all those templates slow down your application too
9    much.
10
11    Situations where this is useful are often forking web applications that
12    are initialized on the first request.
13
14    :copyright: (c) 2017 by the Jinja Team.
15    :license: BSD.
16"""
17from os import path, listdir
18import os
19import sys
20import stat
21import errno
22import marshal
23import tempfile
24import fnmatch
25from hashlib import sha1
26from jinja2.utils import open_if_exists
27from jinja2._compat import BytesIO, pickle, PY2, text_type
28
29
30# marshal works better on 3.x, one hack less required
31if not PY2:
32    marshal_dump = marshal.dump
33    marshal_load = marshal.load
34else:
35
36    def marshal_dump(code, f):
37        if isinstance(f, file):
38            marshal.dump(code, f)
39        else:
40            f.write(marshal.dumps(code))
41
42    def marshal_load(f):
43        if isinstance(f, file):
44            return marshal.load(f)
45        return marshal.loads(f.read())
46
47
48bc_version = 3
49
50# magic version used to only change with new jinja versions.  With 2.6
51# we change this to also take Python version changes into account.  The
52# reason for this is that Python tends to segfault if fed earlier bytecode
53# versions because someone thought it would be a good idea to reuse opcodes
54# or make Python incompatible with earlier versions.
55bc_magic = 'j2'.encode('ascii') + \
56    pickle.dumps(bc_version, 2) + \
57    pickle.dumps((sys.version_info[0] << 24) | sys.version_info[1])
58
59
60class Bucket(object):
61    """Buckets are used to store the bytecode for one template.  It's created
62    and initialized by the bytecode cache and passed to the loading functions.
63
64    The buckets get an internal checksum from the cache assigned and use this
65    to automatically reject outdated cache material.  Individual bytecode
66    cache subclasses don't have to care about cache invalidation.
67    """
68
69    def __init__(self, environment, key, checksum):
70        self.environment = environment
71        self.key = key
72        self.checksum = checksum
73        self.reset()
74
75    def reset(self):
76        """Resets the bucket (unloads the bytecode)."""
77        self.code = None
78
79    def load_bytecode(self, f):
80        """Loads bytecode from a file or file like object."""
81        # make sure the magic header is correct
82        magic = f.read(len(bc_magic))
83        if magic != bc_magic:
84            self.reset()
85            return
86        # the source code of the file changed, we need to reload
87        checksum = pickle.load(f)
88        if self.checksum != checksum:
89            self.reset()
90            return
91        # if marshal_load fails then we need to reload
92        try:
93            self.code = marshal_load(f)
94        except (EOFError, ValueError, TypeError):
95            self.reset()
96            return
97
98    def write_bytecode(self, f):
99        """Dump the bytecode into the file or file like object passed."""
100        if self.code is None:
101            raise TypeError('can\'t write empty bucket')
102        f.write(bc_magic)
103        pickle.dump(self.checksum, f, 2)
104        marshal_dump(self.code, f)
105
106    def bytecode_from_string(self, string):
107        """Load bytecode from a string."""
108        self.load_bytecode(BytesIO(string))
109
110    def bytecode_to_string(self):
111        """Return the bytecode as string."""
112        out = BytesIO()
113        self.write_bytecode(out)
114        return out.getvalue()
115
116
117class BytecodeCache(object):
118    """To implement your own bytecode cache you have to subclass this class
119    and override :meth:`load_bytecode` and :meth:`dump_bytecode`.  Both of
120    these methods are passed a :class:`~jinja2.bccache.Bucket`.
121
122    A very basic bytecode cache that saves the bytecode on the file system::
123
124        from os import path
125
126        class MyCache(BytecodeCache):
127
128            def __init__(self, directory):
129                self.directory = directory
130
131            def load_bytecode(self, bucket):
132                filename = path.join(self.directory, bucket.key)
133                if path.exists(filename):
134                    with open(filename, 'rb') as f:
135                        bucket.load_bytecode(f)
136
137            def dump_bytecode(self, bucket):
138                filename = path.join(self.directory, bucket.key)
139                with open(filename, 'wb') as f:
140                    bucket.write_bytecode(f)
141
142    A more advanced version of a filesystem based bytecode cache is part of
143    Jinja2.
144    """
145
146    def load_bytecode(self, bucket):
147        """Subclasses have to override this method to load bytecode into a
148        bucket.  If they are not able to find code in the cache for the
149        bucket, it must not do anything.
150        """
151        raise NotImplementedError()
152
153    def dump_bytecode(self, bucket):
154        """Subclasses have to override this method to write the bytecode
155        from a bucket back to the cache.  If it unable to do so it must not
156        fail silently but raise an exception.
157        """
158        raise NotImplementedError()
159
160    def clear(self):
161        """Clears the cache.  This method is not used by Jinja2 but should be
162        implemented to allow applications to clear the bytecode cache used
163        by a particular environment.
164        """
165
166    def get_cache_key(self, name, filename=None):
167        """Returns the unique hash key for this template name."""
168        hash = sha1(name.encode('utf-8'))
169        if filename is not None:
170            filename = '|' + filename
171            if isinstance(filename, text_type):
172                filename = filename.encode('utf-8')
173            hash.update(filename)
174        return hash.hexdigest()
175
176    def get_source_checksum(self, source):
177        """Returns a checksum for the source."""
178        return sha1(source.encode('utf-8')).hexdigest()
179
180    def get_bucket(self, environment, name, filename, source):
181        """Return a cache bucket for the given template.  All arguments are
182        mandatory but filename may be `None`.
183        """
184        key = self.get_cache_key(name, filename)
185        checksum = self.get_source_checksum(source)
186        bucket = Bucket(environment, key, checksum)
187        self.load_bytecode(bucket)
188        return bucket
189
190    def set_bucket(self, bucket):
191        """Put the bucket into the cache."""
192        self.dump_bytecode(bucket)
193
194
195class FileSystemBytecodeCache(BytecodeCache):
196    """A bytecode cache that stores bytecode on the filesystem.  It accepts
197    two arguments: The directory where the cache items are stored and a
198    pattern string that is used to build the filename.
199
200    If no directory is specified a default cache directory is selected.  On
201    Windows the user's temp directory is used, on UNIX systems a directory
202    is created for the user in the system temp directory.
203
204    The pattern can be used to have multiple separate caches operate on the
205    same directory.  The default pattern is ``'__jinja2_%s.cache'``.  ``%s``
206    is replaced with the cache key.
207
208    >>> bcc = FileSystemBytecodeCache('/tmp/jinja_cache', '%s.cache')
209
210    This bytecode cache supports clearing of the cache using the clear method.
211    """
212
213    def __init__(self, directory=None, pattern='__jinja2_%s.cache'):
214        if directory is None:
215            directory = self._get_default_cache_dir()
216        self.directory = directory
217        self.pattern = pattern
218
219    def _get_default_cache_dir(self):
220        def _unsafe_dir():
221            raise RuntimeError('Cannot determine safe temp directory.  You '
222                               'need to explicitly provide one.')
223
224        tmpdir = tempfile.gettempdir()
225
226        # On windows the temporary directory is used specific unless
227        # explicitly forced otherwise.  We can just use that.
228        if os.name == 'nt':
229            return tmpdir
230        if not hasattr(os, 'getuid'):
231            _unsafe_dir()
232
233        dirname = '_jinja2-cache-%d' % os.getuid()
234        actual_dir = os.path.join(tmpdir, dirname)
235
236        try:
237            os.mkdir(actual_dir, stat.S_IRWXU)
238        except OSError as e:
239            if e.errno != errno.EEXIST:
240                raise
241        try:
242            os.chmod(actual_dir, stat.S_IRWXU)
243            actual_dir_stat = os.lstat(actual_dir)
244            if actual_dir_stat.st_uid != os.getuid() \
245               or not stat.S_ISDIR(actual_dir_stat.st_mode) \
246               or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU:
247                _unsafe_dir()
248        except OSError as e:
249            if e.errno != errno.EEXIST:
250                raise
251
252        actual_dir_stat = os.lstat(actual_dir)
253        if actual_dir_stat.st_uid != os.getuid() \
254           or not stat.S_ISDIR(actual_dir_stat.st_mode) \
255           or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU:
256            _unsafe_dir()
257
258        return actual_dir
259
260    def _get_cache_filename(self, bucket):
261        return path.join(self.directory, self.pattern % bucket.key)
262
263    def load_bytecode(self, bucket):
264        f = open_if_exists(self._get_cache_filename(bucket), 'rb')
265        if f is not None:
266            try:
267                bucket.load_bytecode(f)
268            finally:
269                f.close()
270
271    def dump_bytecode(self, bucket):
272        f = open(self._get_cache_filename(bucket), 'wb')
273        try:
274            bucket.write_bytecode(f)
275        finally:
276            f.close()
277
278    def clear(self):
279        # imported lazily here because google app-engine doesn't support
280        # write access on the file system and the function does not exist
281        # normally.
282        from os import remove
283        files = fnmatch.filter(listdir(self.directory), self.pattern % '*')
284        for filename in files:
285            try:
286                remove(path.join(self.directory, filename))
287            except OSError:
288                pass
289
290
291class MemcachedBytecodeCache(BytecodeCache):
292    """This class implements a bytecode cache that uses a memcache cache for
293    storing the information.  It does not enforce a specific memcache library
294    (tummy's memcache or cmemcache) but will accept any class that provides
295    the minimal interface required.
296
297    Libraries compatible with this class:
298
299    -   `werkzeug <http://werkzeug.pocoo.org/>`_.contrib.cache
300    -   `python-memcached <https://www.tummy.com/Community/software/python-memcached/>`_
301    -   `cmemcache <http://gijsbert.org/cmemcache/>`_
302
303    (Unfortunately the django cache interface is not compatible because it
304    does not support storing binary data, only unicode.  You can however pass
305    the underlying cache client to the bytecode cache which is available
306    as `django.core.cache.cache._client`.)
307
308    The minimal interface for the client passed to the constructor is this:
309
310    .. class:: MinimalClientInterface
311
312        .. method:: set(key, value[, timeout])
313
314            Stores the bytecode in the cache.  `value` is a string and
315            `timeout` the timeout of the key.  If timeout is not provided
316            a default timeout or no timeout should be assumed, if it's
317            provided it's an integer with the number of seconds the cache
318            item should exist.
319
320        .. method:: get(key)
321
322            Returns the value for the cache key.  If the item does not
323            exist in the cache the return value must be `None`.
324
325    The other arguments to the constructor are the prefix for all keys that
326    is added before the actual cache key and the timeout for the bytecode in
327    the cache system.  We recommend a high (or no) timeout.
328
329    This bytecode cache does not support clearing of used items in the cache.
330    The clear method is a no-operation function.
331
332    .. versionadded:: 2.7
333       Added support for ignoring memcache errors through the
334       `ignore_memcache_errors` parameter.
335    """
336
337    def __init__(self, client, prefix='jinja2/bytecode/', timeout=None,
338                 ignore_memcache_errors=True):
339        self.client = client
340        self.prefix = prefix
341        self.timeout = timeout
342        self.ignore_memcache_errors = ignore_memcache_errors
343
344    def load_bytecode(self, bucket):
345        try:
346            code = self.client.get(self.prefix + bucket.key)
347        except Exception:
348            if not self.ignore_memcache_errors:
349                raise
350            code = None
351        if code is not None:
352            bucket.bytecode_from_string(code)
353
354    def dump_bytecode(self, bucket):
355        args = (self.prefix + bucket.key, bucket.bytecode_to_string())
356        if self.timeout is not None:
357            args += (self.timeout,)
358        try:
359            self.client.set(*args)
360        except Exception:
361            if not self.ignore_memcache_errors:
362                raise
363