1""" 2Downloader Middleware manager 3 4See documentation in docs/topics/downloader-middleware.rst 5""" 6from twisted.internet import defer 7 8from scrapy.exceptions import _InvalidOutput 9from scrapy.http import Request, Response 10from scrapy.middleware import MiddlewareManager 11from scrapy.utils.defer import mustbe_deferred, deferred_from_coro 12from scrapy.utils.conf import build_component_list 13 14 15class DownloaderMiddlewareManager(MiddlewareManager): 16 17 component_name = 'downloader middleware' 18 19 @classmethod 20 def _get_mwlist_from_settings(cls, settings): 21 return build_component_list( 22 settings.getwithbase('DOWNLOADER_MIDDLEWARES')) 23 24 def _add_middleware(self, mw): 25 if hasattr(mw, 'process_request'): 26 self.methods['process_request'].append(mw.process_request) 27 if hasattr(mw, 'process_response'): 28 self.methods['process_response'].appendleft(mw.process_response) 29 if hasattr(mw, 'process_exception'): 30 self.methods['process_exception'].appendleft(mw.process_exception) 31 32 def download(self, download_func, request, spider): 33 @defer.inlineCallbacks 34 def process_request(request): 35 for method in self.methods['process_request']: 36 response = yield deferred_from_coro(method(request=request, spider=spider)) 37 if response is not None and not isinstance(response, (Response, Request)): 38 raise _InvalidOutput( 39 f"Middleware {method.__qualname__} must return None, Response or " 40 f"Request, got {response.__class__.__name__}" 41 ) 42 if response: 43 return response 44 return (yield download_func(request=request, spider=spider)) 45 46 @defer.inlineCallbacks 47 def process_response(response): 48 if response is None: 49 raise TypeError("Received None in process_response") 50 elif isinstance(response, Request): 51 return response 52 53 for method in self.methods['process_response']: 54 response = yield deferred_from_coro(method(request=request, response=response, spider=spider)) 55 if not isinstance(response, (Response, Request)): 56 raise _InvalidOutput( 57 f"Middleware {method.__qualname__} must return Response or Request, " 58 f"got {type(response)}" 59 ) 60 if isinstance(response, Request): 61 return response 62 return response 63 64 @defer.inlineCallbacks 65 def process_exception(failure): 66 exception = failure.value 67 for method in self.methods['process_exception']: 68 response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider)) 69 if response is not None and not isinstance(response, (Response, Request)): 70 raise _InvalidOutput( 71 f"Middleware {method.__qualname__} must return None, Response or " 72 f"Request, got {type(response)}" 73 ) 74 if response: 75 return response 76 return failure 77 78 deferred = mustbe_deferred(process_request, request) 79 deferred.addErrback(process_exception) 80 deferred.addCallback(process_response) 81 return deferred 82