1# -*- coding: utf-8 -*- 2 3""" 4requests.cookies 5~~~~~~~~~~~~~~~~ 6 7Compatibility code to be able to use `cookielib.CookieJar` with requests. 8 9requests.utils imports from here, so be careful with imports. 10""" 11 12import copy 13import time 14import calendar 15import collections 16 17from ._internal_utils import to_native_string 18from .compat import cookielib, urlparse, urlunparse, Morsel 19 20try: 21 import threading 22except ImportError: 23 import dummy_threading as threading 24 25 26class MockRequest(object): 27 """Wraps a `requests.Request` to mimic a `urllib2.Request`. 28 29 The code in `cookielib.CookieJar` expects this interface in order to correctly 30 manage cookie policies, i.e., determine whether a cookie can be set, given the 31 domains of the request and the cookie. 32 33 The original request object is read-only. The client is responsible for collecting 34 the new headers via `get_new_headers()` and interpreting them appropriately. You 35 probably want `get_cookie_header`, defined below. 36 """ 37 38 def __init__(self, request): 39 self._r = request 40 self._new_headers = {} 41 self.type = urlparse(self._r.url).scheme 42 43 def get_type(self): 44 return self.type 45 46 def get_host(self): 47 return urlparse(self._r.url).netloc 48 49 def get_origin_req_host(self): 50 return self.get_host() 51 52 def get_full_url(self): 53 # Only return the response's URL if the user hadn't set the Host 54 # header 55 if not self._r.headers.get('Host'): 56 return self._r.url 57 # If they did set it, retrieve it and reconstruct the expected domain 58 host = to_native_string(self._r.headers['Host'], encoding='utf-8') 59 parsed = urlparse(self._r.url) 60 # Reconstruct the URL as we expect it 61 return urlunparse([ 62 parsed.scheme, host, parsed.path, parsed.params, parsed.query, 63 parsed.fragment 64 ]) 65 66 def is_unverifiable(self): 67 return True 68 69 def has_header(self, name): 70 return name in self._r.headers or name in self._new_headers 71 72 def get_header(self, name, default=None): 73 return self._r.headers.get(name, self._new_headers.get(name, default)) 74 75 def add_header(self, key, val): 76 """cookielib has no legitimate use for this method; add it back if you find one.""" 77 raise NotImplementedError("Cookie headers should be added with add_unredirected_header()") 78 79 def add_unredirected_header(self, name, value): 80 self._new_headers[name] = value 81 82 def get_new_headers(self): 83 return self._new_headers 84 85 @property 86 def unverifiable(self): 87 return self.is_unverifiable() 88 89 @property 90 def origin_req_host(self): 91 return self.get_origin_req_host() 92 93 @property 94 def host(self): 95 return self.get_host() 96 97 98class MockResponse(object): 99 """Wraps a `httplib.HTTPMessage` to mimic a `urllib.addinfourl`. 100 101 ...what? Basically, expose the parsed HTTP headers from the server response 102 the way `cookielib` expects to see them. 103 """ 104 105 def __init__(self, headers): 106 """Make a MockResponse for `cookielib` to read. 107 108 :param headers: a httplib.HTTPMessage or analogous carrying the headers 109 """ 110 self._headers = headers 111 112 def info(self): 113 return self._headers 114 115 def getheaders(self, name): 116 self._headers.getheaders(name) 117 118 119def extract_cookies_to_jar(jar, request, response): 120 """Extract the cookies from the response into a CookieJar. 121 122 :param jar: cookielib.CookieJar (not necessarily a RequestsCookieJar) 123 :param request: our own requests.Request object 124 :param response: urllib3.HTTPResponse object 125 """ 126 if not (hasattr(response, '_original_response') and 127 response._original_response): 128 return 129 # the _original_response field is the wrapped httplib.HTTPResponse object, 130 req = MockRequest(request) 131 # pull out the HTTPMessage with the headers and put it in the mock: 132 res = MockResponse(response._original_response.msg) 133 jar.extract_cookies(res, req) 134 135 136def get_cookie_header(jar, request): 137 """ 138 Produce an appropriate Cookie header string to be sent with `request`, or None. 139 140 :rtype: str 141 """ 142 r = MockRequest(request) 143 jar.add_cookie_header(r) 144 return r.get_new_headers().get('Cookie') 145 146 147def remove_cookie_by_name(cookiejar, name, domain=None, path=None): 148 """Unsets a cookie by name, by default over all domains and paths. 149 150 Wraps CookieJar.clear(), is O(n). 151 """ 152 clearables = [] 153 for cookie in cookiejar: 154 if cookie.name != name: 155 continue 156 if domain is not None and domain != cookie.domain: 157 continue 158 if path is not None and path != cookie.path: 159 continue 160 clearables.append((cookie.domain, cookie.path, cookie.name)) 161 162 for domain, path, name in clearables: 163 cookiejar.clear(domain, path, name) 164 165 166class CookieConflictError(RuntimeError): 167 """There are two cookies that meet the criteria specified in the cookie jar. 168 Use .get and .set and include domain and path args in order to be more specific. 169 """ 170 171 172class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): 173 """Compatibility class; is a cookielib.CookieJar, but exposes a dict 174 interface. 175 176 This is the CookieJar we create by default for requests and sessions that 177 don't specify one, since some clients may expect response.cookies and 178 session.cookies to support dict operations. 179 180 Requests does not use the dict interface internally; it's just for 181 compatibility with external client code. All requests code should work 182 out of the box with externally provided instances of ``CookieJar``, e.g. 183 ``LWPCookieJar`` and ``FileCookieJar``. 184 185 Unlike a regular CookieJar, this class is pickleable. 186 187 .. warning:: dictionary operations that are normally O(1) may be O(n). 188 """ 189 190 def get(self, name, default=None, domain=None, path=None): 191 """Dict-like get() that also supports optional domain and path args in 192 order to resolve naming collisions from using one cookie jar over 193 multiple domains. 194 195 .. warning:: operation is O(n), not O(1). 196 """ 197 try: 198 return self._find_no_duplicates(name, domain, path) 199 except KeyError: 200 return default 201 202 def set(self, name, value, **kwargs): 203 """Dict-like set() that also supports optional domain and path args in 204 order to resolve naming collisions from using one cookie jar over 205 multiple domains. 206 """ 207 # support client code that unsets cookies by assignment of a None value: 208 if value is None: 209 remove_cookie_by_name(self, name, domain=kwargs.get('domain'), path=kwargs.get('path')) 210 return 211 212 if isinstance(value, Morsel): 213 c = morsel_to_cookie(value) 214 else: 215 c = create_cookie(name, value, **kwargs) 216 self.set_cookie(c) 217 return c 218 219 def iterkeys(self): 220 """Dict-like iterkeys() that returns an iterator of names of cookies 221 from the jar. 222 223 .. seealso:: itervalues() and iteritems(). 224 """ 225 for cookie in iter(self): 226 yield cookie.name 227 228 def keys(self): 229 """Dict-like keys() that returns a list of names of cookies from the 230 jar. 231 232 .. seealso:: values() and items(). 233 """ 234 return list(self.iterkeys()) 235 236 def itervalues(self): 237 """Dict-like itervalues() that returns an iterator of values of cookies 238 from the jar. 239 240 .. seealso:: iterkeys() and iteritems(). 241 """ 242 for cookie in iter(self): 243 yield cookie.value 244 245 def values(self): 246 """Dict-like values() that returns a list of values of cookies from the 247 jar. 248 249 .. seealso:: keys() and items(). 250 """ 251 return list(self.itervalues()) 252 253 def iteritems(self): 254 """Dict-like iteritems() that returns an iterator of name-value tuples 255 from the jar. 256 257 .. seealso:: iterkeys() and itervalues(). 258 """ 259 for cookie in iter(self): 260 yield cookie.name, cookie.value 261 262 def items(self): 263 """Dict-like items() that returns a list of name-value tuples from the 264 jar. Allows client-code to call ``dict(RequestsCookieJar)`` and get a 265 vanilla python dict of key value pairs. 266 267 .. seealso:: keys() and values(). 268 """ 269 return list(self.iteritems()) 270 271 def list_domains(self): 272 """Utility method to list all the domains in the jar.""" 273 domains = [] 274 for cookie in iter(self): 275 if cookie.domain not in domains: 276 domains.append(cookie.domain) 277 return domains 278 279 def list_paths(self): 280 """Utility method to list all the paths in the jar.""" 281 paths = [] 282 for cookie in iter(self): 283 if cookie.path not in paths: 284 paths.append(cookie.path) 285 return paths 286 287 def multiple_domains(self): 288 """Returns True if there are multiple domains in the jar. 289 Returns False otherwise. 290 291 :rtype: bool 292 """ 293 domains = [] 294 for cookie in iter(self): 295 if cookie.domain is not None and cookie.domain in domains: 296 return True 297 domains.append(cookie.domain) 298 return False # there is only one domain in jar 299 300 def get_dict(self, domain=None, path=None): 301 """Takes as an argument an optional domain and path and returns a plain 302 old Python dict of name-value pairs of cookies that meet the 303 requirements. 304 305 :rtype: dict 306 """ 307 dictionary = {} 308 for cookie in iter(self): 309 if ( 310 (domain is None or cookie.domain == domain) and 311 (path is None or cookie.path == path) 312 ): 313 dictionary[cookie.name] = cookie.value 314 return dictionary 315 316 def __contains__(self, name): 317 try: 318 return super(RequestsCookieJar, self).__contains__(name) 319 except CookieConflictError: 320 return True 321 322 def __getitem__(self, name): 323 """Dict-like __getitem__() for compatibility with client code. Throws 324 exception if there are more than one cookie with name. In that case, 325 use the more explicit get() method instead. 326 327 .. warning:: operation is O(n), not O(1). 328 """ 329 return self._find_no_duplicates(name) 330 331 def __setitem__(self, name, value): 332 """Dict-like __setitem__ for compatibility with client code. Throws 333 exception if there is already a cookie of that name in the jar. In that 334 case, use the more explicit set() method instead. 335 """ 336 self.set(name, value) 337 338 def __delitem__(self, name): 339 """Deletes a cookie given a name. Wraps ``cookielib.CookieJar``'s 340 ``remove_cookie_by_name()``. 341 """ 342 remove_cookie_by_name(self, name) 343 344 def set_cookie(self, cookie, *args, **kwargs): 345 if hasattr(cookie.value, 'startswith') and cookie.value.startswith('"') and cookie.value.endswith('"'): 346 cookie.value = cookie.value.replace('\\"', '') 347 return super(RequestsCookieJar, self).set_cookie(cookie, *args, **kwargs) 348 349 def update(self, other): 350 """Updates this jar with cookies from another CookieJar or dict-like""" 351 if isinstance(other, cookielib.CookieJar): 352 for cookie in other: 353 self.set_cookie(copy.copy(cookie)) 354 else: 355 super(RequestsCookieJar, self).update(other) 356 357 def _find(self, name, domain=None, path=None): 358 """Requests uses this method internally to get cookie values. 359 360 If there are conflicting cookies, _find arbitrarily chooses one. 361 See _find_no_duplicates if you want an exception thrown if there are 362 conflicting cookies. 363 364 :param name: a string containing name of cookie 365 :param domain: (optional) string containing domain of cookie 366 :param path: (optional) string containing path of cookie 367 :return: cookie.value 368 """ 369 for cookie in iter(self): 370 if cookie.name == name: 371 if domain is None or cookie.domain == domain: 372 if path is None or cookie.path == path: 373 return cookie.value 374 375 raise KeyError('name=%r, domain=%r, path=%r' % (name, domain, path)) 376 377 def _find_no_duplicates(self, name, domain=None, path=None): 378 """Both ``__get_item__`` and ``get`` call this function: it's never 379 used elsewhere in Requests. 380 381 :param name: a string containing name of cookie 382 :param domain: (optional) string containing domain of cookie 383 :param path: (optional) string containing path of cookie 384 :raises KeyError: if cookie is not found 385 :raises CookieConflictError: if there are multiple cookies 386 that match name and optionally domain and path 387 :return: cookie.value 388 """ 389 toReturn = None 390 for cookie in iter(self): 391 if cookie.name == name: 392 if domain is None or cookie.domain == domain: 393 if path is None or cookie.path == path: 394 if toReturn is not None: # if there are multiple cookies that meet passed in criteria 395 raise CookieConflictError('There are multiple cookies with name, %r' % (name)) 396 toReturn = cookie.value # we will eventually return this as long as no cookie conflict 397 398 if toReturn: 399 return toReturn 400 raise KeyError('name=%r, domain=%r, path=%r' % (name, domain, path)) 401 402 def __getstate__(self): 403 """Unlike a normal CookieJar, this class is pickleable.""" 404 state = self.__dict__.copy() 405 # remove the unpickleable RLock object 406 state.pop('_cookies_lock') 407 return state 408 409 def __setstate__(self, state): 410 """Unlike a normal CookieJar, this class is pickleable.""" 411 self.__dict__.update(state) 412 if '_cookies_lock' not in self.__dict__: 413 self._cookies_lock = threading.RLock() 414 415 def copy(self): 416 """Return a copy of this RequestsCookieJar.""" 417 new_cj = RequestsCookieJar() 418 new_cj.update(self) 419 return new_cj 420 421 422def _copy_cookie_jar(jar): 423 if jar is None: 424 return None 425 426 if hasattr(jar, 'copy'): 427 # We're dealing with an instance of RequestsCookieJar 428 return jar.copy() 429 # We're dealing with a generic CookieJar instance 430 new_jar = copy.copy(jar) 431 new_jar.clear() 432 for cookie in jar: 433 new_jar.set_cookie(copy.copy(cookie)) 434 return new_jar 435 436 437def create_cookie(name, value, **kwargs): 438 """Make a cookie from underspecified parameters. 439 440 By default, the pair of `name` and `value` will be set for the domain '' 441 and sent on every request (this is sometimes called a "supercookie"). 442 """ 443 result = dict( 444 version=0, 445 name=name, 446 value=value, 447 port=None, 448 domain='', 449 path='/', 450 secure=False, 451 expires=None, 452 discard=True, 453 comment=None, 454 comment_url=None, 455 rest={'HttpOnly': None}, 456 rfc2109=False,) 457 458 badargs = set(kwargs) - set(result) 459 if badargs: 460 err = 'create_cookie() got unexpected keyword arguments: %s' 461 raise TypeError(err % list(badargs)) 462 463 result.update(kwargs) 464 result['port_specified'] = bool(result['port']) 465 result['domain_specified'] = bool(result['domain']) 466 result['domain_initial_dot'] = result['domain'].startswith('.') 467 result['path_specified'] = bool(result['path']) 468 469 return cookielib.Cookie(**result) 470 471 472def morsel_to_cookie(morsel): 473 """Convert a Morsel object into a Cookie containing the one k/v pair.""" 474 475 expires = None 476 if morsel['max-age']: 477 try: 478 expires = int(time.time() + int(morsel['max-age'])) 479 except ValueError: 480 raise TypeError('max-age: %s must be integer' % morsel['max-age']) 481 elif morsel['expires']: 482 time_template = '%a, %d-%b-%Y %H:%M:%S GMT' 483 expires = calendar.timegm( 484 time.strptime(morsel['expires'], time_template) 485 ) 486 return create_cookie( 487 comment=morsel['comment'], 488 comment_url=bool(morsel['comment']), 489 discard=False, 490 domain=morsel['domain'], 491 expires=expires, 492 name=morsel.key, 493 path=morsel['path'], 494 port=None, 495 rest={'HttpOnly': morsel['httponly']}, 496 rfc2109=False, 497 secure=bool(morsel['secure']), 498 value=morsel.value, 499 version=morsel['version'] or 0, 500 ) 501 502 503def cookiejar_from_dict(cookie_dict, cookiejar=None, overwrite=True): 504 """Returns a CookieJar from a key/value dictionary. 505 506 :param cookie_dict: Dict of key/values to insert into CookieJar. 507 :param cookiejar: (optional) A cookiejar to add the cookies to. 508 :param overwrite: (optional) If False, will not replace cookies 509 already in the jar with new ones. 510 """ 511 if cookiejar is None: 512 cookiejar = RequestsCookieJar() 513 514 if cookie_dict is not None: 515 names_from_jar = [cookie.name for cookie in cookiejar] 516 for name in cookie_dict: 517 if overwrite or (name not in names_from_jar): 518 cookiejar.set_cookie(create_cookie(name, cookie_dict[name])) 519 520 return cookiejar 521 522 523def merge_cookies(cookiejar, cookies): 524 """Add cookies to cookiejar and returns a merged CookieJar. 525 526 :param cookiejar: CookieJar object to add the cookies to. 527 :param cookies: Dictionary or CookieJar object to be added. 528 """ 529 if not isinstance(cookiejar, cookielib.CookieJar): 530 raise ValueError('You can only merge into CookieJar') 531 532 if isinstance(cookies, dict): 533 cookiejar = cookiejar_from_dict( 534 cookies, cookiejar=cookiejar, overwrite=False) 535 elif isinstance(cookies, cookielib.CookieJar): 536 try: 537 cookiejar.update(cookies) 538 except AttributeError: 539 for cookie_in_jar in cookies: 540 cookiejar.set_cookie(cookie_in_jar) 541 542 return cookiejar 543