1# -*- coding: utf-8 -*- 2# 3# Copyright (C) 2013-2017 Vinay Sajip. 4# Licensed to the Python Software Foundation under a contributor agreement. 5# See LICENSE.txt and CONTRIBUTORS.txt. 6# 7from __future__ import absolute_import 8 9import os 10import re 11import sys 12 13try: 14 import ssl 15except ImportError: # pragma: no cover 16 ssl = None 17 18if sys.version_info[0] < 3: # pragma: no cover 19 from StringIO import StringIO 20 string_types = basestring, 21 text_type = unicode 22 from types import FileType as file_type 23 import __builtin__ as builtins 24 import ConfigParser as configparser 25 from ._backport import shutil 26 from urlparse import urlparse, urlunparse, urljoin, urlsplit, urlunsplit 27 from urllib import (urlretrieve, quote as _quote, unquote, url2pathname, 28 pathname2url, ContentTooShortError, splittype) 29 30 def quote(s): 31 if isinstance(s, unicode): 32 s = s.encode('utf-8') 33 return _quote(s) 34 35 import urllib2 36 from urllib2 import (Request, urlopen, URLError, HTTPError, 37 HTTPBasicAuthHandler, HTTPPasswordMgr, 38 HTTPHandler, HTTPRedirectHandler, 39 build_opener) 40 if ssl: 41 from urllib2 import HTTPSHandler 42 import httplib 43 import xmlrpclib 44 import Queue as queue 45 from HTMLParser import HTMLParser 46 import htmlentitydefs 47 raw_input = raw_input 48 from itertools import ifilter as filter 49 from itertools import ifilterfalse as filterfalse 50 51 _userprog = None 52 def splituser(host): 53 """splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'.""" 54 global _userprog 55 if _userprog is None: 56 import re 57 _userprog = re.compile('^(.*)@(.*)$') 58 59 match = _userprog.match(host) 60 if match: return match.group(1, 2) 61 return None, host 62 63else: # pragma: no cover 64 from io import StringIO 65 string_types = str, 66 text_type = str 67 from io import TextIOWrapper as file_type 68 import builtins 69 import configparser 70 import shutil 71 from urllib.parse import (urlparse, urlunparse, urljoin, splituser, quote, 72 unquote, urlsplit, urlunsplit, splittype) 73 from urllib.request import (urlopen, urlretrieve, Request, url2pathname, 74 pathname2url, 75 HTTPBasicAuthHandler, HTTPPasswordMgr, 76 HTTPHandler, HTTPRedirectHandler, 77 build_opener) 78 if ssl: 79 from urllib.request import HTTPSHandler 80 from urllib.error import HTTPError, URLError, ContentTooShortError 81 import http.client as httplib 82 import urllib.request as urllib2 83 import xmlrpc.client as xmlrpclib 84 import queue 85 from html.parser import HTMLParser 86 import html.entities as htmlentitydefs 87 raw_input = input 88 from itertools import filterfalse 89 filter = filter 90 91try: 92 from ssl import match_hostname, CertificateError 93except ImportError: # pragma: no cover 94 class CertificateError(ValueError): 95 pass 96 97 98 def _dnsname_match(dn, hostname, max_wildcards=1): 99 """Matching according to RFC 6125, section 6.4.3 100 101 http://tools.ietf.org/html/rfc6125#section-6.4.3 102 """ 103 pats = [] 104 if not dn: 105 return False 106 107 parts = dn.split('.') 108 leftmost, remainder = parts[0], parts[1:] 109 110 wildcards = leftmost.count('*') 111 if wildcards > max_wildcards: 112 # Issue #17980: avoid denials of service by refusing more 113 # than one wildcard per fragment. A survey of established 114 # policy among SSL implementations showed it to be a 115 # reasonable choice. 116 raise CertificateError( 117 "too many wildcards in certificate DNS name: " + repr(dn)) 118 119 # speed up common case w/o wildcards 120 if not wildcards: 121 return dn.lower() == hostname.lower() 122 123 # RFC 6125, section 6.4.3, subitem 1. 124 # The client SHOULD NOT attempt to match a presented identifier in which 125 # the wildcard character comprises a label other than the left-most label. 126 if leftmost == '*': 127 # When '*' is a fragment by itself, it matches a non-empty dotless 128 # fragment. 129 pats.append('[^.]+') 130 elif leftmost.startswith('xn--') or hostname.startswith('xn--'): 131 # RFC 6125, section 6.4.3, subitem 3. 132 # The client SHOULD NOT attempt to match a presented identifier 133 # where the wildcard character is embedded within an A-label or 134 # U-label of an internationalized domain name. 135 pats.append(re.escape(leftmost)) 136 else: 137 # Otherwise, '*' matches any dotless string, e.g. www* 138 pats.append(re.escape(leftmost).replace(r'\*', '[^.]*')) 139 140 # add the remaining fragments, ignore any wildcards 141 for frag in remainder: 142 pats.append(re.escape(frag)) 143 144 pat = re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE) 145 return pat.match(hostname) 146 147 148 def match_hostname(cert, hostname): 149 """Verify that *cert* (in decoded format as returned by 150 SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 and RFC 6125 151 rules are followed, but IP addresses are not accepted for *hostname*. 152 153 CertificateError is raised on failure. On success, the function 154 returns nothing. 155 """ 156 if not cert: 157 raise ValueError("empty or no certificate, match_hostname needs a " 158 "SSL socket or SSL context with either " 159 "CERT_OPTIONAL or CERT_REQUIRED") 160 dnsnames = [] 161 san = cert.get('subjectAltName', ()) 162 for key, value in san: 163 if key == 'DNS': 164 if _dnsname_match(value, hostname): 165 return 166 dnsnames.append(value) 167 if not dnsnames: 168 # The subject is only checked when there is no dNSName entry 169 # in subjectAltName 170 for sub in cert.get('subject', ()): 171 for key, value in sub: 172 # XXX according to RFC 2818, the most specific Common Name 173 # must be used. 174 if key == 'commonName': 175 if _dnsname_match(value, hostname): 176 return 177 dnsnames.append(value) 178 if len(dnsnames) > 1: 179 raise CertificateError("hostname %r " 180 "doesn't match either of %s" 181 % (hostname, ', '.join(map(repr, dnsnames)))) 182 elif len(dnsnames) == 1: 183 raise CertificateError("hostname %r " 184 "doesn't match %r" 185 % (hostname, dnsnames[0])) 186 else: 187 raise CertificateError("no appropriate commonName or " 188 "subjectAltName fields were found") 189 190 191try: 192 from types import SimpleNamespace as Container 193except ImportError: # pragma: no cover 194 class Container(object): 195 """ 196 A generic container for when multiple values need to be returned 197 """ 198 def __init__(self, **kwargs): 199 self.__dict__.update(kwargs) 200 201 202try: 203 from shutil import which 204except ImportError: # pragma: no cover 205 # Implementation from Python 3.3 206 def which(cmd, mode=os.F_OK | os.X_OK, path=None): 207 """Given a command, mode, and a PATH string, return the path which 208 conforms to the given mode on the PATH, or None if there is no such 209 file. 210 211 `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result 212 of os.environ.get("PATH"), or can be overridden with a custom search 213 path. 214 215 """ 216 # Check that a given file can be accessed with the correct mode. 217 # Additionally check that `file` is not a directory, as on Windows 218 # directories pass the os.access check. 219 def _access_check(fn, mode): 220 return (os.path.exists(fn) and os.access(fn, mode) 221 and not os.path.isdir(fn)) 222 223 # If we're given a path with a directory part, look it up directly rather 224 # than referring to PATH directories. This includes checking relative to the 225 # current directory, e.g. ./script 226 if os.path.dirname(cmd): 227 if _access_check(cmd, mode): 228 return cmd 229 return None 230 231 if path is None: 232 path = os.environ.get("PATH", os.defpath) 233 if not path: 234 return None 235 path = path.split(os.pathsep) 236 237 if sys.platform == "win32": 238 # The current directory takes precedence on Windows. 239 if not os.curdir in path: 240 path.insert(0, os.curdir) 241 242 # PATHEXT is necessary to check on Windows. 243 pathext = os.environ.get("PATHEXT", "").split(os.pathsep) 244 # See if the given file matches any of the expected path extensions. 245 # This will allow us to short circuit when given "python.exe". 246 # If it does match, only test that one, otherwise we have to try 247 # others. 248 if any(cmd.lower().endswith(ext.lower()) for ext in pathext): 249 files = [cmd] 250 else: 251 files = [cmd + ext for ext in pathext] 252 else: 253 # On other platforms you don't have things like PATHEXT to tell you 254 # what file suffixes are executable, so just pass on cmd as-is. 255 files = [cmd] 256 257 seen = set() 258 for dir in path: 259 normdir = os.path.normcase(dir) 260 if not normdir in seen: 261 seen.add(normdir) 262 for thefile in files: 263 name = os.path.join(dir, thefile) 264 if _access_check(name, mode): 265 return name 266 return None 267 268 269# ZipFile is a context manager in 2.7, but not in 2.6 270 271from zipfile import ZipFile as BaseZipFile 272 273if hasattr(BaseZipFile, '__enter__'): # pragma: no cover 274 ZipFile = BaseZipFile 275else: # pragma: no cover 276 from zipfile import ZipExtFile as BaseZipExtFile 277 278 class ZipExtFile(BaseZipExtFile): 279 def __init__(self, base): 280 self.__dict__.update(base.__dict__) 281 282 def __enter__(self): 283 return self 284 285 def __exit__(self, *exc_info): 286 self.close() 287 # return None, so if an exception occurred, it will propagate 288 289 class ZipFile(BaseZipFile): 290 def __enter__(self): 291 return self 292 293 def __exit__(self, *exc_info): 294 self.close() 295 # return None, so if an exception occurred, it will propagate 296 297 def open(self, *args, **kwargs): 298 base = BaseZipFile.open(self, *args, **kwargs) 299 return ZipExtFile(base) 300 301try: 302 from platform import python_implementation 303except ImportError: # pragma: no cover 304 def python_implementation(): 305 """Return a string identifying the Python implementation.""" 306 if 'PyPy' in sys.version: 307 return 'PyPy' 308 if os.name == 'java': 309 return 'Jython' 310 if sys.version.startswith('IronPython'): 311 return 'IronPython' 312 return 'CPython' 313 314try: 315 import sysconfig 316except ImportError: # pragma: no cover 317 from ._backport import sysconfig 318 319try: 320 callable = callable 321except NameError: # pragma: no cover 322 from collections.abc import Callable 323 324 def callable(obj): 325 return isinstance(obj, Callable) 326 327 328try: 329 fsencode = os.fsencode 330 fsdecode = os.fsdecode 331except AttributeError: # pragma: no cover 332 # Issue #99: on some systems (e.g. containerised), 333 # sys.getfilesystemencoding() returns None, and we need a real value, 334 # so fall back to utf-8. From the CPython 2.7 docs relating to Unix and 335 # sys.getfilesystemencoding(): the return value is "the user’s preference 336 # according to the result of nl_langinfo(CODESET), or None if the 337 # nl_langinfo(CODESET) failed." 338 _fsencoding = sys.getfilesystemencoding() or 'utf-8' 339 if _fsencoding == 'mbcs': 340 _fserrors = 'strict' 341 else: 342 _fserrors = 'surrogateescape' 343 344 def fsencode(filename): 345 if isinstance(filename, bytes): 346 return filename 347 elif isinstance(filename, text_type): 348 return filename.encode(_fsencoding, _fserrors) 349 else: 350 raise TypeError("expect bytes or str, not %s" % 351 type(filename).__name__) 352 353 def fsdecode(filename): 354 if isinstance(filename, text_type): 355 return filename 356 elif isinstance(filename, bytes): 357 return filename.decode(_fsencoding, _fserrors) 358 else: 359 raise TypeError("expect bytes or str, not %s" % 360 type(filename).__name__) 361 362try: 363 from tokenize import detect_encoding 364except ImportError: # pragma: no cover 365 from codecs import BOM_UTF8, lookup 366 import re 367 368 cookie_re = re.compile(r"coding[:=]\s*([-\w.]+)") 369 370 def _get_normal_name(orig_enc): 371 """Imitates get_normal_name in tokenizer.c.""" 372 # Only care about the first 12 characters. 373 enc = orig_enc[:12].lower().replace("_", "-") 374 if enc == "utf-8" or enc.startswith("utf-8-"): 375 return "utf-8" 376 if enc in ("latin-1", "iso-8859-1", "iso-latin-1") or \ 377 enc.startswith(("latin-1-", "iso-8859-1-", "iso-latin-1-")): 378 return "iso-8859-1" 379 return orig_enc 380 381 def detect_encoding(readline): 382 """ 383 The detect_encoding() function is used to detect the encoding that should 384 be used to decode a Python source file. It requires one argument, readline, 385 in the same way as the tokenize() generator. 386 387 It will call readline a maximum of twice, and return the encoding used 388 (as a string) and a list of any lines (left as bytes) it has read in. 389 390 It detects the encoding from the presence of a utf-8 bom or an encoding 391 cookie as specified in pep-0263. If both a bom and a cookie are present, 392 but disagree, a SyntaxError will be raised. If the encoding cookie is an 393 invalid charset, raise a SyntaxError. Note that if a utf-8 bom is found, 394 'utf-8-sig' is returned. 395 396 If no encoding is specified, then the default of 'utf-8' will be returned. 397 """ 398 try: 399 filename = readline.__self__.name 400 except AttributeError: 401 filename = None 402 bom_found = False 403 encoding = None 404 default = 'utf-8' 405 def read_or_stop(): 406 try: 407 return readline() 408 except StopIteration: 409 return b'' 410 411 def find_cookie(line): 412 try: 413 # Decode as UTF-8. Either the line is an encoding declaration, 414 # in which case it should be pure ASCII, or it must be UTF-8 415 # per default encoding. 416 line_string = line.decode('utf-8') 417 except UnicodeDecodeError: 418 msg = "invalid or missing encoding declaration" 419 if filename is not None: 420 msg = '{} for {!r}'.format(msg, filename) 421 raise SyntaxError(msg) 422 423 matches = cookie_re.findall(line_string) 424 if not matches: 425 return None 426 encoding = _get_normal_name(matches[0]) 427 try: 428 codec = lookup(encoding) 429 except LookupError: 430 # This behaviour mimics the Python interpreter 431 if filename is None: 432 msg = "unknown encoding: " + encoding 433 else: 434 msg = "unknown encoding for {!r}: {}".format(filename, 435 encoding) 436 raise SyntaxError(msg) 437 438 if bom_found: 439 if codec.name != 'utf-8': 440 # This behaviour mimics the Python interpreter 441 if filename is None: 442 msg = 'encoding problem: utf-8' 443 else: 444 msg = 'encoding problem for {!r}: utf-8'.format(filename) 445 raise SyntaxError(msg) 446 encoding += '-sig' 447 return encoding 448 449 first = read_or_stop() 450 if first.startswith(BOM_UTF8): 451 bom_found = True 452 first = first[3:] 453 default = 'utf-8-sig' 454 if not first: 455 return default, [] 456 457 encoding = find_cookie(first) 458 if encoding: 459 return encoding, [first] 460 461 second = read_or_stop() 462 if not second: 463 return default, [first] 464 465 encoding = find_cookie(second) 466 if encoding: 467 return encoding, [first, second] 468 469 return default, [first, second] 470 471# For converting & <-> & etc. 472try: 473 from html import escape 474except ImportError: 475 from cgi import escape 476if sys.version_info[:2] < (3, 4): 477 unescape = HTMLParser().unescape 478else: 479 from html import unescape 480 481try: 482 from collections import ChainMap 483except ImportError: # pragma: no cover 484 from collections import MutableMapping 485 486 try: 487 from reprlib import recursive_repr as _recursive_repr 488 except ImportError: 489 def _recursive_repr(fillvalue='...'): 490 ''' 491 Decorator to make a repr function return fillvalue for a recursive 492 call 493 ''' 494 495 def decorating_function(user_function): 496 repr_running = set() 497 498 def wrapper(self): 499 key = id(self), get_ident() 500 if key in repr_running: 501 return fillvalue 502 repr_running.add(key) 503 try: 504 result = user_function(self) 505 finally: 506 repr_running.discard(key) 507 return result 508 509 # Can't use functools.wraps() here because of bootstrap issues 510 wrapper.__module__ = getattr(user_function, '__module__') 511 wrapper.__doc__ = getattr(user_function, '__doc__') 512 wrapper.__name__ = getattr(user_function, '__name__') 513 wrapper.__annotations__ = getattr(user_function, '__annotations__', {}) 514 return wrapper 515 516 return decorating_function 517 518 class ChainMap(MutableMapping): 519 ''' A ChainMap groups multiple dicts (or other mappings) together 520 to create a single, updateable view. 521 522 The underlying mappings are stored in a list. That list is public and can 523 accessed or updated using the *maps* attribute. There is no other state. 524 525 Lookups search the underlying mappings successively until a key is found. 526 In contrast, writes, updates, and deletions only operate on the first 527 mapping. 528 529 ''' 530 531 def __init__(self, *maps): 532 '''Initialize a ChainMap by setting *maps* to the given mappings. 533 If no mappings are provided, a single empty dictionary is used. 534 535 ''' 536 self.maps = list(maps) or [{}] # always at least one map 537 538 def __missing__(self, key): 539 raise KeyError(key) 540 541 def __getitem__(self, key): 542 for mapping in self.maps: 543 try: 544 return mapping[key] # can't use 'key in mapping' with defaultdict 545 except KeyError: 546 pass 547 return self.__missing__(key) # support subclasses that define __missing__ 548 549 def get(self, key, default=None): 550 return self[key] if key in self else default 551 552 def __len__(self): 553 return len(set().union(*self.maps)) # reuses stored hash values if possible 554 555 def __iter__(self): 556 return iter(set().union(*self.maps)) 557 558 def __contains__(self, key): 559 return any(key in m for m in self.maps) 560 561 def __bool__(self): 562 return any(self.maps) 563 564 @_recursive_repr() 565 def __repr__(self): 566 return '{0.__class__.__name__}({1})'.format( 567 self, ', '.join(map(repr, self.maps))) 568 569 @classmethod 570 def fromkeys(cls, iterable, *args): 571 'Create a ChainMap with a single dict created from the iterable.' 572 return cls(dict.fromkeys(iterable, *args)) 573 574 def copy(self): 575 'New ChainMap or subclass with a new copy of maps[0] and refs to maps[1:]' 576 return self.__class__(self.maps[0].copy(), *self.maps[1:]) 577 578 __copy__ = copy 579 580 def new_child(self): # like Django's Context.push() 581 'New ChainMap with a new dict followed by all previous maps.' 582 return self.__class__({}, *self.maps) 583 584 @property 585 def parents(self): # like Django's Context.pop() 586 'New ChainMap from maps[1:].' 587 return self.__class__(*self.maps[1:]) 588 589 def __setitem__(self, key, value): 590 self.maps[0][key] = value 591 592 def __delitem__(self, key): 593 try: 594 del self.maps[0][key] 595 except KeyError: 596 raise KeyError('Key not found in the first mapping: {!r}'.format(key)) 597 598 def popitem(self): 599 'Remove and return an item pair from maps[0]. Raise KeyError is maps[0] is empty.' 600 try: 601 return self.maps[0].popitem() 602 except KeyError: 603 raise KeyError('No keys found in the first mapping.') 604 605 def pop(self, key, *args): 606 'Remove *key* from maps[0] and return its value. Raise KeyError if *key* not in maps[0].' 607 try: 608 return self.maps[0].pop(key, *args) 609 except KeyError: 610 raise KeyError('Key not found in the first mapping: {!r}'.format(key)) 611 612 def clear(self): 613 'Clear maps[0], leaving maps[1:] intact.' 614 self.maps[0].clear() 615 616try: 617 from importlib.util import cache_from_source # Python >= 3.4 618except ImportError: # pragma: no cover 619 try: 620 from imp import cache_from_source 621 except ImportError: # pragma: no cover 622 def cache_from_source(path, debug_override=None): 623 assert path.endswith('.py') 624 if debug_override is None: 625 debug_override = __debug__ 626 if debug_override: 627 suffix = 'c' 628 else: 629 suffix = 'o' 630 return path + suffix 631 632try: 633 from collections import OrderedDict 634except ImportError: # pragma: no cover 635## {{{ http://code.activestate.com/recipes/576693/ (r9) 636# Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy. 637# Passes Python2.7's test suite and incorporates all the latest updates. 638 try: 639 from thread import get_ident as _get_ident 640 except ImportError: 641 from dummy_thread import get_ident as _get_ident 642 643 try: 644 from _abcoll import KeysView, ValuesView, ItemsView 645 except ImportError: 646 pass 647 648 649 class OrderedDict(dict): 650 'Dictionary that remembers insertion order' 651 # An inherited dict maps keys to values. 652 # The inherited dict provides __getitem__, __len__, __contains__, and get. 653 # The remaining methods are order-aware. 654 # Big-O running times for all methods are the same as for regular dictionaries. 655 656 # The internal self.__map dictionary maps keys to links in a doubly linked list. 657 # The circular doubly linked list starts and ends with a sentinel element. 658 # The sentinel element never gets deleted (this simplifies the algorithm). 659 # Each link is stored as a list of length three: [PREV, NEXT, KEY]. 660 661 def __init__(self, *args, **kwds): 662 '''Initialize an ordered dictionary. Signature is the same as for 663 regular dictionaries, but keyword arguments are not recommended 664 because their insertion order is arbitrary. 665 666 ''' 667 if len(args) > 1: 668 raise TypeError('expected at most 1 arguments, got %d' % len(args)) 669 try: 670 self.__root 671 except AttributeError: 672 self.__root = root = [] # sentinel node 673 root[:] = [root, root, None] 674 self.__map = {} 675 self.__update(*args, **kwds) 676 677 def __setitem__(self, key, value, dict_setitem=dict.__setitem__): 678 'od.__setitem__(i, y) <==> od[i]=y' 679 # Setting a new item creates a new link which goes at the end of the linked 680 # list, and the inherited dictionary is updated with the new key/value pair. 681 if key not in self: 682 root = self.__root 683 last = root[0] 684 last[1] = root[0] = self.__map[key] = [last, root, key] 685 dict_setitem(self, key, value) 686 687 def __delitem__(self, key, dict_delitem=dict.__delitem__): 688 'od.__delitem__(y) <==> del od[y]' 689 # Deleting an existing item uses self.__map to find the link which is 690 # then removed by updating the links in the predecessor and successor nodes. 691 dict_delitem(self, key) 692 link_prev, link_next, key = self.__map.pop(key) 693 link_prev[1] = link_next 694 link_next[0] = link_prev 695 696 def __iter__(self): 697 'od.__iter__() <==> iter(od)' 698 root = self.__root 699 curr = root[1] 700 while curr is not root: 701 yield curr[2] 702 curr = curr[1] 703 704 def __reversed__(self): 705 'od.__reversed__() <==> reversed(od)' 706 root = self.__root 707 curr = root[0] 708 while curr is not root: 709 yield curr[2] 710 curr = curr[0] 711 712 def clear(self): 713 'od.clear() -> None. Remove all items from od.' 714 try: 715 for node in self.__map.itervalues(): 716 del node[:] 717 root = self.__root 718 root[:] = [root, root, None] 719 self.__map.clear() 720 except AttributeError: 721 pass 722 dict.clear(self) 723 724 def popitem(self, last=True): 725 '''od.popitem() -> (k, v), return and remove a (key, value) pair. 726 Pairs are returned in LIFO order if last is true or FIFO order if false. 727 728 ''' 729 if not self: 730 raise KeyError('dictionary is empty') 731 root = self.__root 732 if last: 733 link = root[0] 734 link_prev = link[0] 735 link_prev[1] = root 736 root[0] = link_prev 737 else: 738 link = root[1] 739 link_next = link[1] 740 root[1] = link_next 741 link_next[0] = root 742 key = link[2] 743 del self.__map[key] 744 value = dict.pop(self, key) 745 return key, value 746 747 # -- the following methods do not depend on the internal structure -- 748 749 def keys(self): 750 'od.keys() -> list of keys in od' 751 return list(self) 752 753 def values(self): 754 'od.values() -> list of values in od' 755 return [self[key] for key in self] 756 757 def items(self): 758 'od.items() -> list of (key, value) pairs in od' 759 return [(key, self[key]) for key in self] 760 761 def iterkeys(self): 762 'od.iterkeys() -> an iterator over the keys in od' 763 return iter(self) 764 765 def itervalues(self): 766 'od.itervalues -> an iterator over the values in od' 767 for k in self: 768 yield self[k] 769 770 def iteritems(self): 771 'od.iteritems -> an iterator over the (key, value) items in od' 772 for k in self: 773 yield (k, self[k]) 774 775 def update(*args, **kwds): 776 '''od.update(E, **F) -> None. Update od from dict/iterable E and F. 777 778 If E is a dict instance, does: for k in E: od[k] = E[k] 779 If E has a .keys() method, does: for k in E.keys(): od[k] = E[k] 780 Or if E is an iterable of items, does: for k, v in E: od[k] = v 781 In either case, this is followed by: for k, v in F.items(): od[k] = v 782 783 ''' 784 if len(args) > 2: 785 raise TypeError('update() takes at most 2 positional ' 786 'arguments (%d given)' % (len(args),)) 787 elif not args: 788 raise TypeError('update() takes at least 1 argument (0 given)') 789 self = args[0] 790 # Make progressively weaker assumptions about "other" 791 other = () 792 if len(args) == 2: 793 other = args[1] 794 if isinstance(other, dict): 795 for key in other: 796 self[key] = other[key] 797 elif hasattr(other, 'keys'): 798 for key in other.keys(): 799 self[key] = other[key] 800 else: 801 for key, value in other: 802 self[key] = value 803 for key, value in kwds.items(): 804 self[key] = value 805 806 __update = update # let subclasses override update without breaking __init__ 807 808 __marker = object() 809 810 def pop(self, key, default=__marker): 811 '''od.pop(k[,d]) -> v, remove specified key and return the corresponding value. 812 If key is not found, d is returned if given, otherwise KeyError is raised. 813 814 ''' 815 if key in self: 816 result = self[key] 817 del self[key] 818 return result 819 if default is self.__marker: 820 raise KeyError(key) 821 return default 822 823 def setdefault(self, key, default=None): 824 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od' 825 if key in self: 826 return self[key] 827 self[key] = default 828 return default 829 830 def __repr__(self, _repr_running=None): 831 'od.__repr__() <==> repr(od)' 832 if not _repr_running: _repr_running = {} 833 call_key = id(self), _get_ident() 834 if call_key in _repr_running: 835 return '...' 836 _repr_running[call_key] = 1 837 try: 838 if not self: 839 return '%s()' % (self.__class__.__name__,) 840 return '%s(%r)' % (self.__class__.__name__, self.items()) 841 finally: 842 del _repr_running[call_key] 843 844 def __reduce__(self): 845 'Return state information for pickling' 846 items = [[k, self[k]] for k in self] 847 inst_dict = vars(self).copy() 848 for k in vars(OrderedDict()): 849 inst_dict.pop(k, None) 850 if inst_dict: 851 return (self.__class__, (items,), inst_dict) 852 return self.__class__, (items,) 853 854 def copy(self): 855 'od.copy() -> a shallow copy of od' 856 return self.__class__(self) 857 858 @classmethod 859 def fromkeys(cls, iterable, value=None): 860 '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S 861 and values equal to v (which defaults to None). 862 863 ''' 864 d = cls() 865 for key in iterable: 866 d[key] = value 867 return d 868 869 def __eq__(self, other): 870 '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive 871 while comparison to a regular mapping is order-insensitive. 872 873 ''' 874 if isinstance(other, OrderedDict): 875 return len(self)==len(other) and self.items() == other.items() 876 return dict.__eq__(self, other) 877 878 def __ne__(self, other): 879 return not self == other 880 881 # -- the following methods are only used in Python 2.7 -- 882 883 def viewkeys(self): 884 "od.viewkeys() -> a set-like object providing a view on od's keys" 885 return KeysView(self) 886 887 def viewvalues(self): 888 "od.viewvalues() -> an object providing a view on od's values" 889 return ValuesView(self) 890 891 def viewitems(self): 892 "od.viewitems() -> a set-like object providing a view on od's items" 893 return ItemsView(self) 894 895try: 896 from logging.config import BaseConfigurator, valid_ident 897except ImportError: # pragma: no cover 898 IDENTIFIER = re.compile('^[a-z_][a-z0-9_]*$', re.I) 899 900 901 def valid_ident(s): 902 m = IDENTIFIER.match(s) 903 if not m: 904 raise ValueError('Not a valid Python identifier: %r' % s) 905 return True 906 907 908 # The ConvertingXXX classes are wrappers around standard Python containers, 909 # and they serve to convert any suitable values in the container. The 910 # conversion converts base dicts, lists and tuples to their wrapped 911 # equivalents, whereas strings which match a conversion format are converted 912 # appropriately. 913 # 914 # Each wrapper should have a configurator attribute holding the actual 915 # configurator to use for conversion. 916 917 class ConvertingDict(dict): 918 """A converting dictionary wrapper.""" 919 920 def __getitem__(self, key): 921 value = dict.__getitem__(self, key) 922 result = self.configurator.convert(value) 923 #If the converted value is different, save for next time 924 if value is not result: 925 self[key] = result 926 if type(result) in (ConvertingDict, ConvertingList, 927 ConvertingTuple): 928 result.parent = self 929 result.key = key 930 return result 931 932 def get(self, key, default=None): 933 value = dict.get(self, key, default) 934 result = self.configurator.convert(value) 935 #If the converted value is different, save for next time 936 if value is not result: 937 self[key] = result 938 if type(result) in (ConvertingDict, ConvertingList, 939 ConvertingTuple): 940 result.parent = self 941 result.key = key 942 return result 943 944 def pop(self, key, default=None): 945 value = dict.pop(self, key, default) 946 result = self.configurator.convert(value) 947 if value is not result: 948 if type(result) in (ConvertingDict, ConvertingList, 949 ConvertingTuple): 950 result.parent = self 951 result.key = key 952 return result 953 954 class ConvertingList(list): 955 """A converting list wrapper.""" 956 def __getitem__(self, key): 957 value = list.__getitem__(self, key) 958 result = self.configurator.convert(value) 959 #If the converted value is different, save for next time 960 if value is not result: 961 self[key] = result 962 if type(result) in (ConvertingDict, ConvertingList, 963 ConvertingTuple): 964 result.parent = self 965 result.key = key 966 return result 967 968 def pop(self, idx=-1): 969 value = list.pop(self, idx) 970 result = self.configurator.convert(value) 971 if value is not result: 972 if type(result) in (ConvertingDict, ConvertingList, 973 ConvertingTuple): 974 result.parent = self 975 return result 976 977 class ConvertingTuple(tuple): 978 """A converting tuple wrapper.""" 979 def __getitem__(self, key): 980 value = tuple.__getitem__(self, key) 981 result = self.configurator.convert(value) 982 if value is not result: 983 if type(result) in (ConvertingDict, ConvertingList, 984 ConvertingTuple): 985 result.parent = self 986 result.key = key 987 return result 988 989 class BaseConfigurator(object): 990 """ 991 The configurator base class which defines some useful defaults. 992 """ 993 994 CONVERT_PATTERN = re.compile(r'^(?P<prefix>[a-z]+)://(?P<suffix>.*)$') 995 996 WORD_PATTERN = re.compile(r'^\s*(\w+)\s*') 997 DOT_PATTERN = re.compile(r'^\.\s*(\w+)\s*') 998 INDEX_PATTERN = re.compile(r'^\[\s*(\w+)\s*\]\s*') 999 DIGIT_PATTERN = re.compile(r'^\d+$') 1000 1001 value_converters = { 1002 'ext' : 'ext_convert', 1003 'cfg' : 'cfg_convert', 1004 } 1005 1006 # We might want to use a different one, e.g. importlib 1007 importer = staticmethod(__import__) 1008 1009 def __init__(self, config): 1010 self.config = ConvertingDict(config) 1011 self.config.configurator = self 1012 1013 def resolve(self, s): 1014 """ 1015 Resolve strings to objects using standard import and attribute 1016 syntax. 1017 """ 1018 name = s.split('.') 1019 used = name.pop(0) 1020 try: 1021 found = self.importer(used) 1022 for frag in name: 1023 used += '.' + frag 1024 try: 1025 found = getattr(found, frag) 1026 except AttributeError: 1027 self.importer(used) 1028 found = getattr(found, frag) 1029 return found 1030 except ImportError: 1031 e, tb = sys.exc_info()[1:] 1032 v = ValueError('Cannot resolve %r: %s' % (s, e)) 1033 v.__cause__, v.__traceback__ = e, tb 1034 raise v 1035 1036 def ext_convert(self, value): 1037 """Default converter for the ext:// protocol.""" 1038 return self.resolve(value) 1039 1040 def cfg_convert(self, value): 1041 """Default converter for the cfg:// protocol.""" 1042 rest = value 1043 m = self.WORD_PATTERN.match(rest) 1044 if m is None: 1045 raise ValueError("Unable to convert %r" % value) 1046 else: 1047 rest = rest[m.end():] 1048 d = self.config[m.groups()[0]] 1049 #print d, rest 1050 while rest: 1051 m = self.DOT_PATTERN.match(rest) 1052 if m: 1053 d = d[m.groups()[0]] 1054 else: 1055 m = self.INDEX_PATTERN.match(rest) 1056 if m: 1057 idx = m.groups()[0] 1058 if not self.DIGIT_PATTERN.match(idx): 1059 d = d[idx] 1060 else: 1061 try: 1062 n = int(idx) # try as number first (most likely) 1063 d = d[n] 1064 except TypeError: 1065 d = d[idx] 1066 if m: 1067 rest = rest[m.end():] 1068 else: 1069 raise ValueError('Unable to convert ' 1070 '%r at %r' % (value, rest)) 1071 #rest should be empty 1072 return d 1073 1074 def convert(self, value): 1075 """ 1076 Convert values to an appropriate type. dicts, lists and tuples are 1077 replaced by their converting alternatives. Strings are checked to 1078 see if they have a conversion format and are converted if they do. 1079 """ 1080 if not isinstance(value, ConvertingDict) and isinstance(value, dict): 1081 value = ConvertingDict(value) 1082 value.configurator = self 1083 elif not isinstance(value, ConvertingList) and isinstance(value, list): 1084 value = ConvertingList(value) 1085 value.configurator = self 1086 elif not isinstance(value, ConvertingTuple) and\ 1087 isinstance(value, tuple): 1088 value = ConvertingTuple(value) 1089 value.configurator = self 1090 elif isinstance(value, string_types): 1091 m = self.CONVERT_PATTERN.match(value) 1092 if m: 1093 d = m.groupdict() 1094 prefix = d['prefix'] 1095 converter = self.value_converters.get(prefix, None) 1096 if converter: 1097 suffix = d['suffix'] 1098 converter = getattr(self, converter) 1099 value = converter(suffix) 1100 return value 1101 1102 def configure_custom(self, config): 1103 """Configure an object with a user-supplied factory.""" 1104 c = config.pop('()') 1105 if not callable(c): 1106 c = self.resolve(c) 1107 props = config.pop('.', None) 1108 # Check for valid identifiers 1109 kwargs = dict([(k, config[k]) for k in config if valid_ident(k)]) 1110 result = c(**kwargs) 1111 if props: 1112 for name, value in props.items(): 1113 setattr(result, name, value) 1114 return result 1115 1116 def as_tuple(self, value): 1117 """Utility function which converts lists to tuples.""" 1118 if isinstance(value, list): 1119 value = tuple(value) 1120 return value 1121