1# -*- coding: utf-8 -*- 2import json 3import os 4import re 5import warnings 6from collections import deque 7from random import choice 8from random import randrange 9from string import ascii_letters as _letters 10from string import digits as _digits 11from threading import Lock 12 13from markupsafe import escape 14from markupsafe import Markup 15 16from ._compat import abc 17from ._compat import string_types 18from ._compat import text_type 19from ._compat import url_quote 20 21# special singleton representing missing values for the runtime 22missing = type("MissingType", (), {"__repr__": lambda x: "missing"})() 23 24# internal code 25internal_code = set() 26 27concat = u"".join 28 29_slash_escape = "\\/" not in json.dumps("/") 30 31 32def contextfunction(f): 33 """This decorator can be used to mark a function or method context callable. 34 A context callable is passed the active :class:`Context` as first argument when 35 called from the template. This is useful if a function wants to get access 36 to the context or functions provided on the context object. For example 37 a function that returns a sorted list of template variables the current 38 template exports could look like this:: 39 40 @contextfunction 41 def get_exported_names(context): 42 return sorted(context.exported_vars) 43 """ 44 f.contextfunction = True 45 return f 46 47 48def evalcontextfunction(f): 49 """This decorator can be used to mark a function or method as an eval 50 context callable. This is similar to the :func:`contextfunction` 51 but instead of passing the context, an evaluation context object is 52 passed. For more information about the eval context, see 53 :ref:`eval-context`. 54 55 .. versionadded:: 2.4 56 """ 57 f.evalcontextfunction = True 58 return f 59 60 61def environmentfunction(f): 62 """This decorator can be used to mark a function or method as environment 63 callable. This decorator works exactly like the :func:`contextfunction` 64 decorator just that the first argument is the active :class:`Environment` 65 and not context. 66 """ 67 f.environmentfunction = True 68 return f 69 70 71def internalcode(f): 72 """Marks the function as internally used""" 73 internal_code.add(f.__code__) 74 return f 75 76 77def is_undefined(obj): 78 """Check if the object passed is undefined. This does nothing more than 79 performing an instance check against :class:`Undefined` but looks nicer. 80 This can be used for custom filters or tests that want to react to 81 undefined variables. For example a custom default filter can look like 82 this:: 83 84 def default(var, default=''): 85 if is_undefined(var): 86 return default 87 return var 88 """ 89 from .runtime import Undefined 90 91 return isinstance(obj, Undefined) 92 93 94def consume(iterable): 95 """Consumes an iterable without doing anything with it.""" 96 for _ in iterable: 97 pass 98 99 100def clear_caches(): 101 """Jinja keeps internal caches for environments and lexers. These are 102 used so that Jinja doesn't have to recreate environments and lexers all 103 the time. Normally you don't have to care about that but if you are 104 measuring memory consumption you may want to clean the caches. 105 """ 106 from .environment import _spontaneous_environments 107 from .lexer import _lexer_cache 108 109 _spontaneous_environments.clear() 110 _lexer_cache.clear() 111 112 113def import_string(import_name, silent=False): 114 """Imports an object based on a string. This is useful if you want to 115 use import paths as endpoints or something similar. An import path can 116 be specified either in dotted notation (``xml.sax.saxutils.escape``) 117 or with a colon as object delimiter (``xml.sax.saxutils:escape``). 118 119 If the `silent` is True the return value will be `None` if the import 120 fails. 121 122 :return: imported object 123 """ 124 try: 125 if ":" in import_name: 126 module, obj = import_name.split(":", 1) 127 elif "." in import_name: 128 module, _, obj = import_name.rpartition(".") 129 else: 130 return __import__(import_name) 131 return getattr(__import__(module, None, None, [obj]), obj) 132 except (ImportError, AttributeError): 133 if not silent: 134 raise 135 136 137def open_if_exists(filename, mode="rb"): 138 """Returns a file descriptor for the filename if that file exists, 139 otherwise ``None``. 140 """ 141 if not os.path.isfile(filename): 142 return None 143 144 return open(filename, mode) 145 146 147def object_type_repr(obj): 148 """Returns the name of the object's type. For some recognized 149 singletons the name of the object is returned instead. (For 150 example for `None` and `Ellipsis`). 151 """ 152 if obj is None: 153 return "None" 154 elif obj is Ellipsis: 155 return "Ellipsis" 156 157 cls = type(obj) 158 159 # __builtin__ in 2.x, builtins in 3.x 160 if cls.__module__ in ("__builtin__", "builtins"): 161 name = cls.__name__ 162 else: 163 name = cls.__module__ + "." + cls.__name__ 164 165 return "%s object" % name 166 167 168def pformat(obj, verbose=False): 169 """Prettyprint an object. Either use the `pretty` library or the 170 builtin `pprint`. 171 """ 172 try: 173 from pretty import pretty 174 175 return pretty(obj, verbose=verbose) 176 except ImportError: 177 from pprint import pformat 178 179 return pformat(obj) 180 181 182def urlize(text, trim_url_limit=None, rel=None, target=None): 183 """Converts any URLs in text into clickable links. Works on http://, 184 https:// and www. links. Links can have trailing punctuation (periods, 185 commas, close-parens) and leading punctuation (opening parens) and 186 it'll still do the right thing. 187 188 If trim_url_limit is not None, the URLs in link text will be limited 189 to trim_url_limit characters. 190 191 If nofollow is True, the URLs in link text will get a rel="nofollow" 192 attribute. 193 194 If target is not None, a target attribute will be added to the link. 195 """ 196 trim_url = ( 197 lambda x, limit=trim_url_limit: limit is not None 198 and (x[:limit] + (len(x) >= limit and "..." or "")) 199 or x 200 ) 201 words = re.split(r"(\s+)", text_type(escape(text))) 202 rel_attr = rel and ' rel="%s"' % text_type(escape(rel)) or "" 203 target_attr = target and ' target="%s"' % escape(target) or "" 204 205 for i, word in enumerate(words): 206 head, middle, tail = "", word, "" 207 match = re.match(r"^([(<]|<)+", middle) 208 209 if match: 210 head = match.group() 211 middle = middle[match.end() :] 212 213 # Unlike lead, which is anchored to the start of the string, 214 # need to check that the string ends with any of the characters 215 # before trying to match all of them, to avoid backtracking. 216 if middle.endswith((")", ">", ".", ",", "\n", ">")): 217 match = re.search(r"([)>.,\n]|>)+$", middle) 218 219 if match: 220 tail = match.group() 221 middle = middle[: match.start()] 222 223 if middle.startswith("www.") or ( 224 "@" not in middle 225 and not middle.startswith("http://") 226 and not middle.startswith("https://") 227 and len(middle) > 0 228 and middle[0] in _letters + _digits 229 and ( 230 middle.endswith(".org") 231 or middle.endswith(".net") 232 or middle.endswith(".com") 233 ) 234 ): 235 middle = '<a href="http://%s"%s%s>%s</a>' % ( 236 middle, 237 rel_attr, 238 target_attr, 239 trim_url(middle), 240 ) 241 242 if middle.startswith("http://") or middle.startswith("https://"): 243 middle = '<a href="%s"%s%s>%s</a>' % ( 244 middle, 245 rel_attr, 246 target_attr, 247 trim_url(middle), 248 ) 249 250 if ( 251 "@" in middle 252 and not middle.startswith("www.") 253 and ":" not in middle 254 and re.match(r"^\S+@\w[\w.-]*\.\w+$", middle) 255 ): 256 middle = '<a href="mailto:%s">%s</a>' % (middle, middle) 257 258 words[i] = head + middle + tail 259 260 return u"".join(words) 261 262 263def generate_lorem_ipsum(n=5, html=True, min=20, max=100): 264 """Generate some lorem ipsum for the template.""" 265 from .constants import LOREM_IPSUM_WORDS 266 267 words = LOREM_IPSUM_WORDS.split() 268 result = [] 269 270 for _ in range(n): 271 next_capitalized = True 272 last_comma = last_fullstop = 0 273 word = None 274 last = None 275 p = [] 276 277 # each paragraph contains out of 20 to 100 words. 278 for idx, _ in enumerate(range(randrange(min, max))): 279 while True: 280 word = choice(words) 281 if word != last: 282 last = word 283 break 284 if next_capitalized: 285 word = word.capitalize() 286 next_capitalized = False 287 # add commas 288 if idx - randrange(3, 8) > last_comma: 289 last_comma = idx 290 last_fullstop += 2 291 word += "," 292 # add end of sentences 293 if idx - randrange(10, 20) > last_fullstop: 294 last_comma = last_fullstop = idx 295 word += "." 296 next_capitalized = True 297 p.append(word) 298 299 # ensure that the paragraph ends with a dot. 300 p = u" ".join(p) 301 if p.endswith(","): 302 p = p[:-1] + "." 303 elif not p.endswith("."): 304 p += "." 305 result.append(p) 306 307 if not html: 308 return u"\n\n".join(result) 309 return Markup(u"\n".join(u"<p>%s</p>" % escape(x) for x in result)) 310 311 312def unicode_urlencode(obj, charset="utf-8", for_qs=False): 313 """Quote a string for use in a URL using the given charset. 314 315 This function is misnamed, it is a wrapper around 316 :func:`urllib.parse.quote`. 317 318 :param obj: String or bytes to quote. Other types are converted to 319 string then encoded to bytes using the given charset. 320 :param charset: Encode text to bytes using this charset. 321 :param for_qs: Quote "/" and use "+" for spaces. 322 """ 323 if not isinstance(obj, string_types): 324 obj = text_type(obj) 325 326 if isinstance(obj, text_type): 327 obj = obj.encode(charset) 328 329 safe = b"" if for_qs else b"/" 330 rv = url_quote(obj, safe) 331 332 if not isinstance(rv, text_type): 333 rv = rv.decode("utf-8") 334 335 if for_qs: 336 rv = rv.replace("%20", "+") 337 338 return rv 339 340 341class LRUCache(object): 342 """A simple LRU Cache implementation.""" 343 344 # this is fast for small capacities (something below 1000) but doesn't 345 # scale. But as long as it's only used as storage for templates this 346 # won't do any harm. 347 348 def __init__(self, capacity): 349 self.capacity = capacity 350 self._mapping = {} 351 self._queue = deque() 352 self._postinit() 353 354 def _postinit(self): 355 # alias all queue methods for faster lookup 356 self._popleft = self._queue.popleft 357 self._pop = self._queue.pop 358 self._remove = self._queue.remove 359 self._wlock = Lock() 360 self._append = self._queue.append 361 362 def __getstate__(self): 363 return { 364 "capacity": self.capacity, 365 "_mapping": self._mapping, 366 "_queue": self._queue, 367 } 368 369 def __setstate__(self, d): 370 self.__dict__.update(d) 371 self._postinit() 372 373 def __getnewargs__(self): 374 return (self.capacity,) 375 376 def copy(self): 377 """Return a shallow copy of the instance.""" 378 rv = self.__class__(self.capacity) 379 rv._mapping.update(self._mapping) 380 rv._queue.extend(self._queue) 381 return rv 382 383 def get(self, key, default=None): 384 """Return an item from the cache dict or `default`""" 385 try: 386 return self[key] 387 except KeyError: 388 return default 389 390 def setdefault(self, key, default=None): 391 """Set `default` if the key is not in the cache otherwise 392 leave unchanged. Return the value of this key. 393 """ 394 try: 395 return self[key] 396 except KeyError: 397 self[key] = default 398 return default 399 400 def clear(self): 401 """Clear the cache.""" 402 self._wlock.acquire() 403 try: 404 self._mapping.clear() 405 self._queue.clear() 406 finally: 407 self._wlock.release() 408 409 def __contains__(self, key): 410 """Check if a key exists in this cache.""" 411 return key in self._mapping 412 413 def __len__(self): 414 """Return the current size of the cache.""" 415 return len(self._mapping) 416 417 def __repr__(self): 418 return "<%s %r>" % (self.__class__.__name__, self._mapping) 419 420 def __getitem__(self, key): 421 """Get an item from the cache. Moves the item up so that it has the 422 highest priority then. 423 424 Raise a `KeyError` if it does not exist. 425 """ 426 self._wlock.acquire() 427 try: 428 rv = self._mapping[key] 429 if self._queue[-1] != key: 430 try: 431 self._remove(key) 432 except ValueError: 433 # if something removed the key from the container 434 # when we read, ignore the ValueError that we would 435 # get otherwise. 436 pass 437 self._append(key) 438 return rv 439 finally: 440 self._wlock.release() 441 442 def __setitem__(self, key, value): 443 """Sets the value for an item. Moves the item up so that it 444 has the highest priority then. 445 """ 446 self._wlock.acquire() 447 try: 448 if key in self._mapping: 449 self._remove(key) 450 elif len(self._mapping) == self.capacity: 451 del self._mapping[self._popleft()] 452 self._append(key) 453 self._mapping[key] = value 454 finally: 455 self._wlock.release() 456 457 def __delitem__(self, key): 458 """Remove an item from the cache dict. 459 Raise a `KeyError` if it does not exist. 460 """ 461 self._wlock.acquire() 462 try: 463 del self._mapping[key] 464 try: 465 self._remove(key) 466 except ValueError: 467 pass 468 finally: 469 self._wlock.release() 470 471 def items(self): 472 """Return a list of items.""" 473 result = [(key, self._mapping[key]) for key in list(self._queue)] 474 result.reverse() 475 return result 476 477 def iteritems(self): 478 """Iterate over all items.""" 479 warnings.warn( 480 "'iteritems()' will be removed in version 3.0. Use" 481 " 'iter(cache.items())' instead.", 482 DeprecationWarning, 483 stacklevel=2, 484 ) 485 return iter(self.items()) 486 487 def values(self): 488 """Return a list of all values.""" 489 return [x[1] for x in self.items()] 490 491 def itervalue(self): 492 """Iterate over all values.""" 493 warnings.warn( 494 "'itervalue()' will be removed in version 3.0. Use" 495 " 'iter(cache.values())' instead.", 496 DeprecationWarning, 497 stacklevel=2, 498 ) 499 return iter(self.values()) 500 501 def itervalues(self): 502 """Iterate over all values.""" 503 warnings.warn( 504 "'itervalues()' will be removed in version 3.0. Use" 505 " 'iter(cache.values())' instead.", 506 DeprecationWarning, 507 stacklevel=2, 508 ) 509 return iter(self.values()) 510 511 def keys(self): 512 """Return a list of all keys ordered by most recent usage.""" 513 return list(self) 514 515 def iterkeys(self): 516 """Iterate over all keys in the cache dict, ordered by 517 the most recent usage. 518 """ 519 warnings.warn( 520 "'iterkeys()' will be removed in version 3.0. Use" 521 " 'iter(cache.keys())' instead.", 522 DeprecationWarning, 523 stacklevel=2, 524 ) 525 return iter(self) 526 527 def __iter__(self): 528 return reversed(tuple(self._queue)) 529 530 def __reversed__(self): 531 """Iterate over the keys in the cache dict, oldest items 532 coming first. 533 """ 534 return iter(tuple(self._queue)) 535 536 __copy__ = copy 537 538 539abc.MutableMapping.register(LRUCache) 540 541 542def select_autoescape( 543 enabled_extensions=("html", "htm", "xml"), 544 disabled_extensions=(), 545 default_for_string=True, 546 default=False, 547): 548 """Intelligently sets the initial value of autoescaping based on the 549 filename of the template. This is the recommended way to configure 550 autoescaping if you do not want to write a custom function yourself. 551 552 If you want to enable it for all templates created from strings or 553 for all templates with `.html` and `.xml` extensions:: 554 555 from jinja2 import Environment, select_autoescape 556 env = Environment(autoescape=select_autoescape( 557 enabled_extensions=('html', 'xml'), 558 default_for_string=True, 559 )) 560 561 Example configuration to turn it on at all times except if the template 562 ends with `.txt`:: 563 564 from jinja2 import Environment, select_autoescape 565 env = Environment(autoescape=select_autoescape( 566 disabled_extensions=('txt',), 567 default_for_string=True, 568 default=True, 569 )) 570 571 The `enabled_extensions` is an iterable of all the extensions that 572 autoescaping should be enabled for. Likewise `disabled_extensions` is 573 a list of all templates it should be disabled for. If a template is 574 loaded from a string then the default from `default_for_string` is used. 575 If nothing matches then the initial value of autoescaping is set to the 576 value of `default`. 577 578 For security reasons this function operates case insensitive. 579 580 .. versionadded:: 2.9 581 """ 582 enabled_patterns = tuple("." + x.lstrip(".").lower() for x in enabled_extensions) 583 disabled_patterns = tuple("." + x.lstrip(".").lower() for x in disabled_extensions) 584 585 def autoescape(template_name): 586 if template_name is None: 587 return default_for_string 588 template_name = template_name.lower() 589 if template_name.endswith(enabled_patterns): 590 return True 591 if template_name.endswith(disabled_patterns): 592 return False 593 return default 594 595 return autoescape 596 597 598def htmlsafe_json_dumps(obj, dumper=None, **kwargs): 599 """Works exactly like :func:`dumps` but is safe for use in ``<script>`` 600 tags. It accepts the same arguments and returns a JSON string. Note that 601 this is available in templates through the ``|tojson`` filter which will 602 also mark the result as safe. Due to how this function escapes certain 603 characters this is safe even if used outside of ``<script>`` tags. 604 605 The following characters are escaped in strings: 606 607 - ``<`` 608 - ``>`` 609 - ``&`` 610 - ``'`` 611 612 This makes it safe to embed such strings in any place in HTML with the 613 notable exception of double quoted attributes. In that case single 614 quote your attributes or HTML escape it in addition. 615 """ 616 if dumper is None: 617 dumper = json.dumps 618 rv = ( 619 dumper(obj, **kwargs) 620 .replace(u"<", u"\\u003c") 621 .replace(u">", u"\\u003e") 622 .replace(u"&", u"\\u0026") 623 .replace(u"'", u"\\u0027") 624 ) 625 return Markup(rv) 626 627 628class Cycler(object): 629 """Cycle through values by yield them one at a time, then restarting 630 once the end is reached. Available as ``cycler`` in templates. 631 632 Similar to ``loop.cycle``, but can be used outside loops or across 633 multiple loops. For example, render a list of folders and files in a 634 list, alternating giving them "odd" and "even" classes. 635 636 .. code-block:: html+jinja 637 638 {% set row_class = cycler("odd", "even") %} 639 <ul class="browser"> 640 {% for folder in folders %} 641 <li class="folder {{ row_class.next() }}">{{ folder }} 642 {% endfor %} 643 {% for file in files %} 644 <li class="file {{ row_class.next() }}">{{ file }} 645 {% endfor %} 646 </ul> 647 648 :param items: Each positional argument will be yielded in the order 649 given for each cycle. 650 651 .. versionadded:: 2.1 652 """ 653 654 def __init__(self, *items): 655 if not items: 656 raise RuntimeError("at least one item has to be provided") 657 self.items = items 658 self.pos = 0 659 660 def reset(self): 661 """Resets the current item to the first item.""" 662 self.pos = 0 663 664 @property 665 def current(self): 666 """Return the current item. Equivalent to the item that will be 667 returned next time :meth:`next` is called. 668 """ 669 return self.items[self.pos] 670 671 def next(self): 672 """Return the current item, then advance :attr:`current` to the 673 next item. 674 """ 675 rv = self.current 676 self.pos = (self.pos + 1) % len(self.items) 677 return rv 678 679 __next__ = next 680 681 682class Joiner(object): 683 """A joining helper for templates.""" 684 685 def __init__(self, sep=u", "): 686 self.sep = sep 687 self.used = False 688 689 def __call__(self): 690 if not self.used: 691 self.used = True 692 return u"" 693 return self.sep 694 695 696class Namespace(object): 697 """A namespace object that can hold arbitrary attributes. It may be 698 initialized from a dictionary or with keyword arguments.""" 699 700 def __init__(*args, **kwargs): # noqa: B902 701 self, args = args[0], args[1:] 702 self.__attrs = dict(*args, **kwargs) 703 704 def __getattribute__(self, name): 705 # __class__ is needed for the awaitable check in async mode 706 if name in {"_Namespace__attrs", "__class__"}: 707 return object.__getattribute__(self, name) 708 try: 709 return self.__attrs[name] 710 except KeyError: 711 raise AttributeError(name) 712 713 def __setitem__(self, name, value): 714 self.__attrs[name] = value 715 716 def __repr__(self): 717 return "<Namespace %r>" % self.__attrs 718 719 720# does this python version support async for in and async generators? 721try: 722 exec("async def _():\n async for _ in ():\n yield _") 723 have_async_gen = True 724except SyntaxError: 725 have_async_gen = False 726 727 728def soft_unicode(s): 729 from markupsafe import soft_unicode 730 731 warnings.warn( 732 "'jinja2.utils.soft_unicode' will be removed in version 3.0." 733 " Use 'markupsafe.soft_unicode' instead.", 734 DeprecationWarning, 735 stacklevel=2, 736 ) 737 return soft_unicode(s) 738