1"""
2Downloader Middleware manager
3
4See documentation in docs/topics/downloader-middleware.rst
5"""
6from twisted.internet import defer
7
8from scrapy.exceptions import _InvalidOutput
9from scrapy.http import Request, Response
10from scrapy.middleware import MiddlewareManager
11from scrapy.utils.defer import mustbe_deferred, deferred_from_coro
12from scrapy.utils.conf import build_component_list
13
14
15class DownloaderMiddlewareManager(MiddlewareManager):
16
17    component_name = 'downloader middleware'
18
19    @classmethod
20    def _get_mwlist_from_settings(cls, settings):
21        return build_component_list(
22            settings.getwithbase('DOWNLOADER_MIDDLEWARES'))
23
24    def _add_middleware(self, mw):
25        if hasattr(mw, 'process_request'):
26            self.methods['process_request'].append(mw.process_request)
27        if hasattr(mw, 'process_response'):
28            self.methods['process_response'].appendleft(mw.process_response)
29        if hasattr(mw, 'process_exception'):
30            self.methods['process_exception'].appendleft(mw.process_exception)
31
32    def download(self, download_func, request, spider):
33        @defer.inlineCallbacks
34        def process_request(request):
35            for method in self.methods['process_request']:
36                response = yield deferred_from_coro(method(request=request, spider=spider))
37                if response is not None and not isinstance(response, (Response, Request)):
38                    raise _InvalidOutput(
39                        f"Middleware {method.__qualname__} must return None, Response or "
40                        f"Request, got {response.__class__.__name__}"
41                    )
42                if response:
43                    return response
44            return (yield download_func(request=request, spider=spider))
45
46        @defer.inlineCallbacks
47        def process_response(response):
48            if response is None:
49                raise TypeError("Received None in process_response")
50            elif isinstance(response, Request):
51                return response
52
53            for method in self.methods['process_response']:
54                response = yield deferred_from_coro(method(request=request, response=response, spider=spider))
55                if not isinstance(response, (Response, Request)):
56                    raise _InvalidOutput(
57                        f"Middleware {method.__qualname__} must return Response or Request, "
58                        f"got {type(response)}"
59                    )
60                if isinstance(response, Request):
61                    return response
62            return response
63
64        @defer.inlineCallbacks
65        def process_exception(failure):
66            exception = failure.value
67            for method in self.methods['process_exception']:
68                response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider))
69                if response is not None and not isinstance(response, (Response, Request)):
70                    raise _InvalidOutput(
71                        f"Middleware {method.__qualname__} must return None, Response or "
72                        f"Request, got {type(response)}"
73                    )
74                if response:
75                    return response
76            return failure
77
78        deferred = mustbe_deferred(process_request, request)
79        deferred.addErrback(process_exception)
80        deferred.addCallback(process_response)
81        return deferred
82