1"""Helper functions for scrapy.http objects (Request, Response)""" 2 3from typing import Union 4from urllib.parse import urlparse, ParseResult 5from weakref import WeakKeyDictionary 6 7from scrapy.http import Request, Response 8 9 10_urlparse_cache: "WeakKeyDictionary[Union[Request, Response], ParseResult]" = WeakKeyDictionary() 11 12 13def urlparse_cached(request_or_response: Union[Request, Response]) -> ParseResult: 14 """Return urlparse.urlparse caching the result, where the argument can be a 15 Request or Response object 16 """ 17 if request_or_response not in _urlparse_cache: 18 _urlparse_cache[request_or_response] = urlparse(request_or_response.url) 19 return _urlparse_cache[request_or_response] 20