1# -*- coding: utf-8 -*- 2""" 3 jinja2.bccache 4 ~~~~~~~~~~~~~~ 5 6 This module implements the bytecode cache system Jinja is optionally 7 using. This is useful if you have very complex template situations and 8 the compiliation of all those templates slow down your application too 9 much. 10 11 Situations where this is useful are often forking web applications that 12 are initialized on the first request. 13 14 :copyright: (c) 2010 by the Jinja Team. 15 :license: BSD. 16""" 17from os import path, listdir 18import marshal 19import tempfile 20import cPickle as pickle 21import fnmatch 22from cStringIO import StringIO 23try: 24 from hashlib import sha1 25except ImportError: 26 from sha import new as sha1 27from jinja2.utils import open_if_exists 28 29 30bc_version = 1 31bc_magic = 'j2'.encode('ascii') + pickle.dumps(bc_version, 2) 32 33 34class Bucket(object): 35 """Buckets are used to store the bytecode for one template. It's created 36 and initialized by the bytecode cache and passed to the loading functions. 37 38 The buckets get an internal checksum from the cache assigned and use this 39 to automatically reject outdated cache material. Individual bytecode 40 cache subclasses don't have to care about cache invalidation. 41 """ 42 43 def __init__(self, environment, key, checksum): 44 self.environment = environment 45 self.key = key 46 self.checksum = checksum 47 self.reset() 48 49 def reset(self): 50 """Resets the bucket (unloads the bytecode).""" 51 self.code = None 52 53 def load_bytecode(self, f): 54 """Loads bytecode from a file or file like object.""" 55 # make sure the magic header is correct 56 magic = f.read(len(bc_magic)) 57 if magic != bc_magic: 58 self.reset() 59 return 60 # the source code of the file changed, we need to reload 61 checksum = pickle.load(f) 62 if self.checksum != checksum: 63 self.reset() 64 return 65 # now load the code. Because marshal is not able to load 66 # from arbitrary streams we have to work around that 67 if isinstance(f, file): 68 self.code = marshal.load(f) 69 else: 70 self.code = marshal.loads(f.read()) 71 72 def write_bytecode(self, f): 73 """Dump the bytecode into the file or file like object passed.""" 74 if self.code is None: 75 raise TypeError('can\'t write empty bucket') 76 f.write(bc_magic) 77 pickle.dump(self.checksum, f, 2) 78 if isinstance(f, file): 79 marshal.dump(self.code, f) 80 else: 81 f.write(marshal.dumps(self.code)) 82 83 def bytecode_from_string(self, string): 84 """Load bytecode from a string.""" 85 self.load_bytecode(StringIO(string)) 86 87 def bytecode_to_string(self): 88 """Return the bytecode as string.""" 89 out = StringIO() 90 self.write_bytecode(out) 91 return out.getvalue() 92 93 94class BytecodeCache(object): 95 """To implement your own bytecode cache you have to subclass this class 96 and override :meth:`load_bytecode` and :meth:`dump_bytecode`. Both of 97 these methods are passed a :class:`~jinja2.bccache.Bucket`. 98 99 A very basic bytecode cache that saves the bytecode on the file system:: 100 101 from os import path 102 103 class MyCache(BytecodeCache): 104 105 def __init__(self, directory): 106 self.directory = directory 107 108 def load_bytecode(self, bucket): 109 filename = path.join(self.directory, bucket.key) 110 if path.exists(filename): 111 with open(filename, 'rb') as f: 112 bucket.load_bytecode(f) 113 114 def dump_bytecode(self, bucket): 115 filename = path.join(self.directory, bucket.key) 116 with open(filename, 'wb') as f: 117 bucket.write_bytecode(f) 118 119 A more advanced version of a filesystem based bytecode cache is part of 120 Jinja2. 121 """ 122 123 def load_bytecode(self, bucket): 124 """Subclasses have to override this method to load bytecode into a 125 bucket. If they are not able to find code in the cache for the 126 bucket, it must not do anything. 127 """ 128 raise NotImplementedError() 129 130 def dump_bytecode(self, bucket): 131 """Subclasses have to override this method to write the bytecode 132 from a bucket back to the cache. If it unable to do so it must not 133 fail silently but raise an exception. 134 """ 135 raise NotImplementedError() 136 137 def clear(self): 138 """Clears the cache. This method is not used by Jinja2 but should be 139 implemented to allow applications to clear the bytecode cache used 140 by a particular environment. 141 """ 142 143 def get_cache_key(self, name, filename=None): 144 """Returns the unique hash key for this template name.""" 145 hash = sha1(name.encode('utf-8')) 146 if filename is not None: 147 if isinstance(filename, unicode): 148 filename = filename.encode('utf-8') 149 hash.update('|' + filename) 150 return hash.hexdigest() 151 152 def get_source_checksum(self, source): 153 """Returns a checksum for the source.""" 154 return sha1(source.encode('utf-8')).hexdigest() 155 156 def get_bucket(self, environment, name, filename, source): 157 """Return a cache bucket for the given template. All arguments are 158 mandatory but filename may be `None`. 159 """ 160 key = self.get_cache_key(name, filename) 161 checksum = self.get_source_checksum(source) 162 bucket = Bucket(environment, key, checksum) 163 self.load_bytecode(bucket) 164 return bucket 165 166 def set_bucket(self, bucket): 167 """Put the bucket into the cache.""" 168 self.dump_bytecode(bucket) 169 170 171class FileSystemBytecodeCache(BytecodeCache): 172 """A bytecode cache that stores bytecode on the filesystem. It accepts 173 two arguments: The directory where the cache items are stored and a 174 pattern string that is used to build the filename. 175 176 If no directory is specified the system temporary items folder is used. 177 178 The pattern can be used to have multiple separate caches operate on the 179 same directory. The default pattern is ``'__jinja2_%s.cache'``. ``%s`` 180 is replaced with the cache key. 181 182 >>> bcc = FileSystemBytecodeCache('/tmp/jinja_cache', '%s.cache') 183 184 This bytecode cache supports clearing of the cache using the clear method. 185 """ 186 187 def __init__(self, directory=None, pattern='__jinja2_%s.cache'): 188 if directory is None: 189 directory = tempfile.gettempdir() 190 self.directory = directory 191 self.pattern = pattern 192 193 def _get_cache_filename(self, bucket): 194 return path.join(self.directory, self.pattern % bucket.key) 195 196 def load_bytecode(self, bucket): 197 f = open_if_exists(self._get_cache_filename(bucket), 'rb') 198 if f is not None: 199 try: 200 bucket.load_bytecode(f) 201 finally: 202 f.close() 203 204 def dump_bytecode(self, bucket): 205 f = open(self._get_cache_filename(bucket), 'wb') 206 try: 207 bucket.write_bytecode(f) 208 finally: 209 f.close() 210 211 def clear(self): 212 # imported lazily here because google app-engine doesn't support 213 # write access on the file system and the function does not exist 214 # normally. 215 from os import remove 216 files = fnmatch.filter(listdir(self.directory), self.pattern % '*') 217 for filename in files: 218 try: 219 remove(path.join(self.directory, filename)) 220 except OSError: 221 pass 222 223 224class MemcachedBytecodeCache(BytecodeCache): 225 """This class implements a bytecode cache that uses a memcache cache for 226 storing the information. It does not enforce a specific memcache library 227 (tummy's memcache or cmemcache) but will accept any class that provides 228 the minimal interface required. 229 230 Libraries compatible with this class: 231 232 - `werkzeug <http://werkzeug.pocoo.org/>`_.contrib.cache 233 - `python-memcached <http://www.tummy.com/Community/software/python-memcached/>`_ 234 - `cmemcache <http://gijsbert.org/cmemcache/>`_ 235 236 (Unfortunately the django cache interface is not compatible because it 237 does not support storing binary data, only unicode. You can however pass 238 the underlying cache client to the bytecode cache which is available 239 as `django.core.cache.cache._client`.) 240 241 The minimal interface for the client passed to the constructor is this: 242 243 .. class:: MinimalClientInterface 244 245 .. method:: set(key, value[, timeout]) 246 247 Stores the bytecode in the cache. `value` is a string and 248 `timeout` the timeout of the key. If timeout is not provided 249 a default timeout or no timeout should be assumed, if it's 250 provided it's an integer with the number of seconds the cache 251 item should exist. 252 253 .. method:: get(key) 254 255 Returns the value for the cache key. If the item does not 256 exist in the cache the return value must be `None`. 257 258 The other arguments to the constructor are the prefix for all keys that 259 is added before the actual cache key and the timeout for the bytecode in 260 the cache system. We recommend a high (or no) timeout. 261 262 This bytecode cache does not support clearing of used items in the cache. 263 The clear method is a no-operation function. 264 """ 265 266 def __init__(self, client, prefix='jinja2/bytecode/', timeout=None): 267 self.client = client 268 self.prefix = prefix 269 self.timeout = timeout 270 271 def load_bytecode(self, bucket): 272 code = self.client.get(self.prefix + bucket.key) 273 if code is not None: 274 bucket.bytecode_from_string(code) 275 276 def dump_bytecode(self, bucket): 277 args = (self.prefix + bucket.key, bucket.bytecode_to_string()) 278 if self.timeout is not None: 279 args += (self.timeout,) 280 self.client.set(*args) 281