1# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
2
3# Copyright 2014-2021 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
4#
5# This file is part of qutebrowser.
6#
7# qutebrowser is free software: you can redistribute it and/or modify
8# it under the terms of the GNU General Public License as published by
9# the Free Software Foundation, either version 3 of the License, or
10# (at your option) any later version.
11#
12# qutebrowser is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15# GNU General Public License for more details.
16#
17# You should have received a copy of the GNU General Public License
18# along with qutebrowser.  If not, see <https://www.gnu.org/licenses/>.
19
20"""Our own QNetworkAccessManager."""
21
22import collections
23import html
24import dataclasses
25from typing import TYPE_CHECKING, Dict, MutableMapping, Optional, Set
26
27from PyQt5.QtCore import pyqtSlot, pyqtSignal, QUrl, QByteArray
28from PyQt5.QtNetwork import (QNetworkAccessManager, QNetworkReply, QSslSocket,
29                             QNetworkProxy)
30
31from qutebrowser.config import config
32from qutebrowser.utils import (message, log, usertypes, utils, objreg,
33                               urlutils, debug)
34from qutebrowser.browser import shared
35from qutebrowser.browser.network import proxy as proxymod
36from qutebrowser.extensions import interceptors
37from qutebrowser.browser.webkit import certificateerror, cookies, cache
38from qutebrowser.browser.webkit.network import (webkitqutescheme, networkreply,
39                                                filescheme)
40from qutebrowser.misc import objects
41
42if TYPE_CHECKING:
43    from qutebrowser.mainwindow import prompt
44
45
46HOSTBLOCK_ERROR_STRING = '%HOSTBLOCK%'
47_proxy_auth_cache: Dict['ProxyId', 'prompt.AuthInfo'] = {}
48
49
50@dataclasses.dataclass(frozen=True)
51class ProxyId:
52
53    """Information identifying a proxy server."""
54
55    type: QNetworkProxy.ProxyType
56    hostname: str
57    port: int
58
59
60def _is_secure_cipher(cipher):
61    """Check if a given SSL cipher (hopefully) isn't broken yet."""
62    tokens = [e.upper() for e in cipher.name().split('-')]
63    if cipher.usedBits() < 128:
64        # https://codereview.qt-project.org/#/c/75943/
65        return False
66    # OpenSSL should already protect against this in a better way
67    elif cipher.keyExchangeMethod() == 'DH' and utils.is_windows:
68        # https://weakdh.org/
69        return False
70    elif cipher.encryptionMethod().upper().startswith('RC4'):
71        # https://en.wikipedia.org/wiki/RC4#Security
72        # https://codereview.qt-project.org/#/c/148906/
73        return False
74    elif cipher.encryptionMethod().upper().startswith('DES'):
75        # https://en.wikipedia.org/wiki/Data_Encryption_Standard#Security_and_cryptanalysis
76        return False
77    elif 'MD5' in tokens:
78        # https://www.win.tue.nl/hashclash/rogue-ca/
79        return False
80    # OpenSSL should already protect against this in a better way
81    # elif (('CBC3' in tokens or 'CBC' in tokens) and (cipher.protocol() not in
82    #         [QSsl.TlsV1_0, QSsl.TlsV1_1, QSsl.TlsV1_2])):
83    #     # https://en.wikipedia.org/wiki/POODLE
84    #     return False
85    ### These things should never happen as those are already filtered out by
86    ### either the SSL libraries or Qt - but let's be sure.
87    elif cipher.authenticationMethod() in ['aNULL', 'NULL']:
88        # Ciphers without authentication.
89        return False
90    elif cipher.encryptionMethod() in ['eNULL', 'NULL']:
91        # Ciphers without encryption.
92        return False
93    elif 'EXP' in tokens or 'EXPORT' in tokens:
94        # Weak export-grade ciphers
95        return False
96    elif 'ADH' in tokens:
97        # No MITM protection
98        return False
99    ### This *should* happen ;)
100    else:
101        return True
102
103
104def init():
105    """Disable insecure SSL ciphers on old Qt versions."""
106    default_ciphers = QSslSocket.defaultCiphers()
107    log.init.vdebug(  # type: ignore[attr-defined]
108        "Default Qt ciphers: {}".format(
109            ', '.join(c.name() for c in default_ciphers)))
110
111    good_ciphers = []
112    bad_ciphers = []
113    for cipher in default_ciphers:
114        if _is_secure_cipher(cipher):
115            good_ciphers.append(cipher)
116        else:
117            bad_ciphers.append(cipher)
118
119    if bad_ciphers:
120        log.init.debug("Disabling bad ciphers: {}".format(
121            ', '.join(c.name() for c in bad_ciphers)))
122        QSslSocket.setDefaultCiphers(good_ciphers)
123
124
125_SavedErrorsType = MutableMapping[
126    urlutils.HostTupleType,
127    Set[certificateerror.CertificateErrorWrapper],
128]
129
130
131class NetworkManager(QNetworkAccessManager):
132
133    """Our own QNetworkAccessManager.
134
135    Attributes:
136        adopted_downloads: If downloads are running with this QNAM but the
137                           associated tab gets closed already, the NAM gets
138                           reparented to the DownloadManager. This counts the
139                           still running downloads, so the QNAM can clean
140                           itself up when this reaches zero again.
141        _scheme_handlers: A dictionary (scheme -> handler) of supported custom
142                          schemes.
143        _win_id: The window ID this NetworkManager is associated with.
144                 (or None for generic network managers)
145        _tab_id: The tab ID this NetworkManager is associated with.
146                 (or None for generic network managers)
147        _rejected_ssl_errors: A {QUrl: {SslError}} dict of rejected errors.
148        _accepted_ssl_errors: A {QUrl: {SslError}} dict of accepted errors.
149        _private: Whether we're in private browsing mode.
150        netrc_used: Whether netrc authentication was performed.
151
152    Signals:
153        shutting_down: Emitted when the QNAM is shutting down.
154    """
155
156    shutting_down = pyqtSignal()
157
158    def __init__(self, *, win_id, tab_id, private, parent=None):
159        log.init.debug("Initializing NetworkManager")
160        with log.disable_qt_msghandler():
161            # WORKAROUND for a hang when a message is printed - See:
162            # https://www.riverbankcomputing.com/pipermail/pyqt/2014-November/035045.html
163            #
164            # Still needed on Qt/PyQt 5.15.2 according to #6010.
165            super().__init__(parent)
166        log.init.debug("NetworkManager init done")
167        self.adopted_downloads = 0
168        self._win_id = win_id
169        self._tab_id = tab_id
170        self._private = private
171        self._scheme_handlers = {
172            'qute': webkitqutescheme.handler,
173            'file': filescheme.handler,
174        }
175        self._set_cookiejar()
176        self._set_cache()
177        self.sslErrors.connect(self.on_ssl_errors)
178        self._rejected_ssl_errors: _SavedErrorsType = collections.defaultdict(set)
179        self._accepted_ssl_errors: _SavedErrorsType = collections.defaultdict(set)
180        self.authenticationRequired.connect(self.on_authentication_required)
181        self.proxyAuthenticationRequired.connect(self.on_proxy_authentication_required)
182        self.netrc_used = False
183
184    def _set_cookiejar(self):
185        """Set the cookie jar of the NetworkManager correctly."""
186        if self._private:
187            cookie_jar = cookies.ram_cookie_jar
188        else:
189            cookie_jar = cookies.cookie_jar
190        assert cookie_jar is not None
191
192        # We have a shared cookie jar - we restore its parent so we don't
193        # take ownership of it.
194        self.setCookieJar(cookie_jar)
195        cookie_jar.setParent(objects.qapp)
196
197    def _set_cache(self):
198        """Set the cache of the NetworkManager correctly."""
199        if self._private:
200            return
201        # We have a shared cache - we restore its parent so we don't take
202        # ownership of it.
203        self.setCache(cache.diskcache)
204        cache.diskcache.setParent(objects.qapp)
205
206    def _get_abort_signals(self, owner=None):
207        """Get a list of signals which should abort a question."""
208        abort_on = [self.shutting_down]
209        if owner is not None:
210            abort_on.append(owner.destroyed)
211        # This might be a generic network manager, e.g. one belonging to a
212        # DownloadManager. In this case, just skip the webview thing.
213        if self._tab_id is not None:
214            assert self._win_id is not None
215            tab = objreg.get('tab', scope='tab', window=self._win_id,
216                             tab=self._tab_id)
217            abort_on.append(tab.load_started)
218        return abort_on
219
220    def _get_tab(self):
221        """Get the tab this NAM is associated with.
222
223        Return:
224            The tab if available, None otherwise.
225        """
226        # There are some scenarios where we can't figure out current_url:
227        # - There's a generic NetworkManager, e.g. for downloads
228        # - The download was in a tab which is now closed.
229        if self._tab_id is None:
230            return None
231
232        assert self._win_id is not None
233        try:
234            return objreg.get('tab', scope='tab', window=self._win_id, tab=self._tab_id)
235        except KeyError:
236            # https://github.com/qutebrowser/qutebrowser/issues/889
237            return None
238
239    def shutdown(self):
240        """Abort all running requests."""
241        self.setNetworkAccessible(QNetworkAccessManager.NotAccessible)
242        self.shutting_down.emit()
243
244    # No @pyqtSlot here, see
245    # https://github.com/qutebrowser/qutebrowser/issues/2213
246    def on_ssl_errors(self, reply, qt_errors):  # noqa: C901 pragma: no mccabe
247        """Decide if SSL errors should be ignored or not.
248
249        This slot is called on SSL/TLS errors by the self.sslErrors signal.
250
251        Args:
252            reply: The QNetworkReply that is encountering the errors.
253            qt_errors: A list of errors.
254        """
255        errors = certificateerror.CertificateErrorWrapper(qt_errors)
256        log.network.debug("Certificate errors: {!r}".format(errors))
257        try:
258            host_tpl: Optional[urlutils.HostTupleType] = urlutils.host_tuple(
259                reply.url())
260        except ValueError:
261            host_tpl = None
262            is_accepted = False
263            is_rejected = False
264        else:
265            assert host_tpl is not None
266            is_accepted = errors in self._accepted_ssl_errors[host_tpl]
267            is_rejected = errors in self._rejected_ssl_errors[host_tpl]
268
269        log.network.debug("Already accepted: {} / "
270                          "rejected {}".format(is_accepted, is_rejected))
271
272        if is_rejected:
273            return
274        elif is_accepted:
275            reply.ignoreSslErrors()
276            return
277
278        abort_on = self._get_abort_signals(reply)
279
280        tab = self._get_tab()
281        first_party_url = QUrl() if tab is None else tab.data.last_navigation.url
282
283        ignore = shared.ignore_certificate_error(
284            request_url=reply.url(),
285            first_party_url=first_party_url,
286            error=errors,
287            abort_on=abort_on,
288        )
289        if ignore:
290            reply.ignoreSslErrors()
291            if host_tpl is not None:
292                self._accepted_ssl_errors[host_tpl].add(errors)
293        elif host_tpl is not None:
294            self._rejected_ssl_errors[host_tpl].add(errors)
295
296    def clear_all_ssl_errors(self):
297        """Clear all remembered SSL errors."""
298        self._accepted_ssl_errors.clear()
299        self._rejected_ssl_errors.clear()
300
301    @pyqtSlot(QUrl)
302    def clear_rejected_ssl_errors(self, url):
303        """Clear the rejected SSL errors on a reload.
304
305        Args:
306            url: The URL to remove.
307        """
308        try:
309            del self._rejected_ssl_errors[url]
310        except KeyError:
311            pass
312
313    @pyqtSlot('QNetworkReply*', 'QAuthenticator*')
314    def on_authentication_required(self, reply, authenticator):
315        """Called when a website needs authentication."""
316        url = reply.url()
317        log.network.debug("Authentication requested for {}, netrc_used {}"
318                          .format(url.toDisplayString(), self.netrc_used))
319
320        netrc_success = False
321        if not self.netrc_used:
322            self.netrc_used = True
323            netrc_success = shared.netrc_authentication(url, authenticator)
324
325        if not netrc_success:
326            log.network.debug("Asking for credentials")
327            abort_on = self._get_abort_signals(reply)
328            shared.authentication_required(url, authenticator,
329                                           abort_on=abort_on)
330
331    @pyqtSlot('QNetworkProxy', 'QAuthenticator*')
332    def on_proxy_authentication_required(self, proxy, authenticator):
333        """Called when a proxy needs authentication."""
334        proxy_id = ProxyId(proxy.type(), proxy.hostName(), proxy.port())
335        if proxy_id in _proxy_auth_cache:
336            authinfo = _proxy_auth_cache[proxy_id]
337            authenticator.setUser(authinfo.user)
338            authenticator.setPassword(authinfo.password)
339        else:
340            msg = '<b>{}</b> says:<br/>{}'.format(
341                html.escape(proxy.hostName()),
342                html.escape(authenticator.realm()))
343            abort_on = self._get_abort_signals()
344            answer = message.ask(
345                title="Proxy authentication required", text=msg,
346                mode=usertypes.PromptMode.user_pwd, abort_on=abort_on)
347            if answer is not None:
348                authenticator.setUser(answer.user)
349                authenticator.setPassword(answer.password)
350                _proxy_auth_cache[proxy_id] = answer
351
352    @pyqtSlot()
353    def on_adopted_download_destroyed(self):
354        """Check if we can clean up if an adopted download was destroyed.
355
356        See the description for adopted_downloads for details.
357        """
358        self.adopted_downloads -= 1
359        log.downloads.debug("Adopted download destroyed, {} left.".format(
360            self.adopted_downloads))
361        assert self.adopted_downloads >= 0
362        if self.adopted_downloads == 0:
363            self.deleteLater()
364
365    @pyqtSlot(object)  # DownloadItem
366    def adopt_download(self, download):
367        """Adopt a new DownloadItem."""
368        self.adopted_downloads += 1
369        log.downloads.debug("Adopted download, {} adopted.".format(
370            self.adopted_downloads))
371        download.destroyed.connect(self.on_adopted_download_destroyed)
372        download.adopt_download.connect(self.adopt_download)
373
374    def set_referer(self, req, current_url):
375        """Set the referer header."""
376        referer_header_conf = config.val.content.headers.referer
377
378        try:
379            if referer_header_conf == 'never':
380                # Note: using ''.encode('ascii') sends a header with no value,
381                # instead of no header at all
382                req.setRawHeader('Referer'.encode('ascii'), QByteArray())
383            elif (referer_header_conf == 'same-domain' and
384                  not urlutils.same_domain(req.url(), current_url)):
385                req.setRawHeader('Referer'.encode('ascii'), QByteArray())
386            # If refer_header_conf is set to 'always', we leave the header
387            # alone as QtWebKit did set it.
388        except urlutils.InvalidUrlError:
389            # req.url() or current_url can be invalid - this happens on
390            # https://www.playstation.com/ for example.
391            pass
392
393    def createRequest(self, op, req, outgoing_data):
394        """Return a new QNetworkReply object.
395
396        Args:
397             op: Operation op
398             req: const QNetworkRequest & req
399             outgoing_data: QIODevice * outgoingData
400
401        Return:
402            A QNetworkReply.
403        """
404        if proxymod.application_factory is not None:
405            proxy_error = proxymod.application_factory.get_error()
406            if proxy_error is not None:
407                return networkreply.ErrorNetworkReply(
408                    req, proxy_error, QNetworkReply.UnknownProxyError,
409                    self)
410
411        if not req.url().isValid():
412            log.network.debug("Ignoring invalid requested URL: {}".format(
413                req.url().errorString()))
414            return networkreply.ErrorNetworkReply(
415                req, "Invalid request URL", QNetworkReply.HostNotFoundError,
416                self)
417
418        for header, value in shared.custom_headers(url=req.url()):
419            req.setRawHeader(header, value)
420
421        tab = self._get_tab()
422        current_url = QUrl()
423        if tab is not None:
424            try:
425                current_url = tab.url()
426            except RuntimeError:
427                # We could be in the middle of the webpage shutdown here.
428                pass
429
430        request = interceptors.Request(first_party_url=current_url,
431                                       request_url=req.url())
432        interceptors.run(request)
433        if request.is_blocked:
434            return networkreply.ErrorNetworkReply(
435                req, HOSTBLOCK_ERROR_STRING, QNetworkReply.ContentAccessDenied,
436                self)
437
438        if 'log-requests' in objects.debug_flags:
439            operation = debug.qenum_key(QNetworkAccessManager, op)
440            operation = operation.replace('Operation', '').upper()
441            log.network.debug("{} {}, first-party {}".format(
442                operation,
443                req.url().toDisplayString(),
444                current_url.toDisplayString()))
445
446        scheme = req.url().scheme()
447        if scheme in self._scheme_handlers:
448            result = self._scheme_handlers[scheme](req, op, current_url)
449            if result is not None:
450                result.setParent(self)
451                return result
452
453        self.set_referer(req, current_url)
454        return super().createRequest(op, req, outgoing_data)
455