1# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: 2 3# Copyright 2014-2021 Florian Bruhin (The Compiler) <mail@qutebrowser.org> 4# 5# This file is part of qutebrowser. 6# 7# qutebrowser is free software: you can redistribute it and/or modify 8# it under the terms of the GNU General Public License as published by 9# the Free Software Foundation, either version 3 of the License, or 10# (at your option) any later version. 11# 12# qutebrowser is distributed in the hope that it will be useful, 13# but WITHOUT ANY WARRANTY; without even the implied warranty of 14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15# GNU General Public License for more details. 16# 17# You should have received a copy of the GNU General Public License 18# along with qutebrowser. If not, see <https://www.gnu.org/licenses/>. 19 20"""Our own QNetworkAccessManager.""" 21 22import collections 23import html 24import dataclasses 25from typing import TYPE_CHECKING, Dict, MutableMapping, Optional, Set 26 27from PyQt5.QtCore import pyqtSlot, pyqtSignal, QUrl, QByteArray 28from PyQt5.QtNetwork import (QNetworkAccessManager, QNetworkReply, QSslSocket, 29 QNetworkProxy) 30 31from qutebrowser.config import config 32from qutebrowser.utils import (message, log, usertypes, utils, objreg, 33 urlutils, debug) 34from qutebrowser.browser import shared 35from qutebrowser.browser.network import proxy as proxymod 36from qutebrowser.extensions import interceptors 37from qutebrowser.browser.webkit import certificateerror, cookies, cache 38from qutebrowser.browser.webkit.network import (webkitqutescheme, networkreply, 39 filescheme) 40from qutebrowser.misc import objects 41 42if TYPE_CHECKING: 43 from qutebrowser.mainwindow import prompt 44 45 46HOSTBLOCK_ERROR_STRING = '%HOSTBLOCK%' 47_proxy_auth_cache: Dict['ProxyId', 'prompt.AuthInfo'] = {} 48 49 50@dataclasses.dataclass(frozen=True) 51class ProxyId: 52 53 """Information identifying a proxy server.""" 54 55 type: QNetworkProxy.ProxyType 56 hostname: str 57 port: int 58 59 60def _is_secure_cipher(cipher): 61 """Check if a given SSL cipher (hopefully) isn't broken yet.""" 62 tokens = [e.upper() for e in cipher.name().split('-')] 63 if cipher.usedBits() < 128: 64 # https://codereview.qt-project.org/#/c/75943/ 65 return False 66 # OpenSSL should already protect against this in a better way 67 elif cipher.keyExchangeMethod() == 'DH' and utils.is_windows: 68 # https://weakdh.org/ 69 return False 70 elif cipher.encryptionMethod().upper().startswith('RC4'): 71 # https://en.wikipedia.org/wiki/RC4#Security 72 # https://codereview.qt-project.org/#/c/148906/ 73 return False 74 elif cipher.encryptionMethod().upper().startswith('DES'): 75 # https://en.wikipedia.org/wiki/Data_Encryption_Standard#Security_and_cryptanalysis 76 return False 77 elif 'MD5' in tokens: 78 # https://www.win.tue.nl/hashclash/rogue-ca/ 79 return False 80 # OpenSSL should already protect against this in a better way 81 # elif (('CBC3' in tokens or 'CBC' in tokens) and (cipher.protocol() not in 82 # [QSsl.TlsV1_0, QSsl.TlsV1_1, QSsl.TlsV1_2])): 83 # # https://en.wikipedia.org/wiki/POODLE 84 # return False 85 ### These things should never happen as those are already filtered out by 86 ### either the SSL libraries or Qt - but let's be sure. 87 elif cipher.authenticationMethod() in ['aNULL', 'NULL']: 88 # Ciphers without authentication. 89 return False 90 elif cipher.encryptionMethod() in ['eNULL', 'NULL']: 91 # Ciphers without encryption. 92 return False 93 elif 'EXP' in tokens or 'EXPORT' in tokens: 94 # Weak export-grade ciphers 95 return False 96 elif 'ADH' in tokens: 97 # No MITM protection 98 return False 99 ### This *should* happen ;) 100 else: 101 return True 102 103 104def init(): 105 """Disable insecure SSL ciphers on old Qt versions.""" 106 default_ciphers = QSslSocket.defaultCiphers() 107 log.init.vdebug( # type: ignore[attr-defined] 108 "Default Qt ciphers: {}".format( 109 ', '.join(c.name() for c in default_ciphers))) 110 111 good_ciphers = [] 112 bad_ciphers = [] 113 for cipher in default_ciphers: 114 if _is_secure_cipher(cipher): 115 good_ciphers.append(cipher) 116 else: 117 bad_ciphers.append(cipher) 118 119 if bad_ciphers: 120 log.init.debug("Disabling bad ciphers: {}".format( 121 ', '.join(c.name() for c in bad_ciphers))) 122 QSslSocket.setDefaultCiphers(good_ciphers) 123 124 125_SavedErrorsType = MutableMapping[ 126 urlutils.HostTupleType, 127 Set[certificateerror.CertificateErrorWrapper], 128] 129 130 131class NetworkManager(QNetworkAccessManager): 132 133 """Our own QNetworkAccessManager. 134 135 Attributes: 136 adopted_downloads: If downloads are running with this QNAM but the 137 associated tab gets closed already, the NAM gets 138 reparented to the DownloadManager. This counts the 139 still running downloads, so the QNAM can clean 140 itself up when this reaches zero again. 141 _scheme_handlers: A dictionary (scheme -> handler) of supported custom 142 schemes. 143 _win_id: The window ID this NetworkManager is associated with. 144 (or None for generic network managers) 145 _tab_id: The tab ID this NetworkManager is associated with. 146 (or None for generic network managers) 147 _rejected_ssl_errors: A {QUrl: {SslError}} dict of rejected errors. 148 _accepted_ssl_errors: A {QUrl: {SslError}} dict of accepted errors. 149 _private: Whether we're in private browsing mode. 150 netrc_used: Whether netrc authentication was performed. 151 152 Signals: 153 shutting_down: Emitted when the QNAM is shutting down. 154 """ 155 156 shutting_down = pyqtSignal() 157 158 def __init__(self, *, win_id, tab_id, private, parent=None): 159 log.init.debug("Initializing NetworkManager") 160 with log.disable_qt_msghandler(): 161 # WORKAROUND for a hang when a message is printed - See: 162 # https://www.riverbankcomputing.com/pipermail/pyqt/2014-November/035045.html 163 # 164 # Still needed on Qt/PyQt 5.15.2 according to #6010. 165 super().__init__(parent) 166 log.init.debug("NetworkManager init done") 167 self.adopted_downloads = 0 168 self._win_id = win_id 169 self._tab_id = tab_id 170 self._private = private 171 self._scheme_handlers = { 172 'qute': webkitqutescheme.handler, 173 'file': filescheme.handler, 174 } 175 self._set_cookiejar() 176 self._set_cache() 177 self.sslErrors.connect(self.on_ssl_errors) 178 self._rejected_ssl_errors: _SavedErrorsType = collections.defaultdict(set) 179 self._accepted_ssl_errors: _SavedErrorsType = collections.defaultdict(set) 180 self.authenticationRequired.connect(self.on_authentication_required) 181 self.proxyAuthenticationRequired.connect(self.on_proxy_authentication_required) 182 self.netrc_used = False 183 184 def _set_cookiejar(self): 185 """Set the cookie jar of the NetworkManager correctly.""" 186 if self._private: 187 cookie_jar = cookies.ram_cookie_jar 188 else: 189 cookie_jar = cookies.cookie_jar 190 assert cookie_jar is not None 191 192 # We have a shared cookie jar - we restore its parent so we don't 193 # take ownership of it. 194 self.setCookieJar(cookie_jar) 195 cookie_jar.setParent(objects.qapp) 196 197 def _set_cache(self): 198 """Set the cache of the NetworkManager correctly.""" 199 if self._private: 200 return 201 # We have a shared cache - we restore its parent so we don't take 202 # ownership of it. 203 self.setCache(cache.diskcache) 204 cache.diskcache.setParent(objects.qapp) 205 206 def _get_abort_signals(self, owner=None): 207 """Get a list of signals which should abort a question.""" 208 abort_on = [self.shutting_down] 209 if owner is not None: 210 abort_on.append(owner.destroyed) 211 # This might be a generic network manager, e.g. one belonging to a 212 # DownloadManager. In this case, just skip the webview thing. 213 if self._tab_id is not None: 214 assert self._win_id is not None 215 tab = objreg.get('tab', scope='tab', window=self._win_id, 216 tab=self._tab_id) 217 abort_on.append(tab.load_started) 218 return abort_on 219 220 def _get_tab(self): 221 """Get the tab this NAM is associated with. 222 223 Return: 224 The tab if available, None otherwise. 225 """ 226 # There are some scenarios where we can't figure out current_url: 227 # - There's a generic NetworkManager, e.g. for downloads 228 # - The download was in a tab which is now closed. 229 if self._tab_id is None: 230 return None 231 232 assert self._win_id is not None 233 try: 234 return objreg.get('tab', scope='tab', window=self._win_id, tab=self._tab_id) 235 except KeyError: 236 # https://github.com/qutebrowser/qutebrowser/issues/889 237 return None 238 239 def shutdown(self): 240 """Abort all running requests.""" 241 self.setNetworkAccessible(QNetworkAccessManager.NotAccessible) 242 self.shutting_down.emit() 243 244 # No @pyqtSlot here, see 245 # https://github.com/qutebrowser/qutebrowser/issues/2213 246 def on_ssl_errors(self, reply, qt_errors): # noqa: C901 pragma: no mccabe 247 """Decide if SSL errors should be ignored or not. 248 249 This slot is called on SSL/TLS errors by the self.sslErrors signal. 250 251 Args: 252 reply: The QNetworkReply that is encountering the errors. 253 qt_errors: A list of errors. 254 """ 255 errors = certificateerror.CertificateErrorWrapper(qt_errors) 256 log.network.debug("Certificate errors: {!r}".format(errors)) 257 try: 258 host_tpl: Optional[urlutils.HostTupleType] = urlutils.host_tuple( 259 reply.url()) 260 except ValueError: 261 host_tpl = None 262 is_accepted = False 263 is_rejected = False 264 else: 265 assert host_tpl is not None 266 is_accepted = errors in self._accepted_ssl_errors[host_tpl] 267 is_rejected = errors in self._rejected_ssl_errors[host_tpl] 268 269 log.network.debug("Already accepted: {} / " 270 "rejected {}".format(is_accepted, is_rejected)) 271 272 if is_rejected: 273 return 274 elif is_accepted: 275 reply.ignoreSslErrors() 276 return 277 278 abort_on = self._get_abort_signals(reply) 279 280 tab = self._get_tab() 281 first_party_url = QUrl() if tab is None else tab.data.last_navigation.url 282 283 ignore = shared.ignore_certificate_error( 284 request_url=reply.url(), 285 first_party_url=first_party_url, 286 error=errors, 287 abort_on=abort_on, 288 ) 289 if ignore: 290 reply.ignoreSslErrors() 291 if host_tpl is not None: 292 self._accepted_ssl_errors[host_tpl].add(errors) 293 elif host_tpl is not None: 294 self._rejected_ssl_errors[host_tpl].add(errors) 295 296 def clear_all_ssl_errors(self): 297 """Clear all remembered SSL errors.""" 298 self._accepted_ssl_errors.clear() 299 self._rejected_ssl_errors.clear() 300 301 @pyqtSlot(QUrl) 302 def clear_rejected_ssl_errors(self, url): 303 """Clear the rejected SSL errors on a reload. 304 305 Args: 306 url: The URL to remove. 307 """ 308 try: 309 del self._rejected_ssl_errors[url] 310 except KeyError: 311 pass 312 313 @pyqtSlot('QNetworkReply*', 'QAuthenticator*') 314 def on_authentication_required(self, reply, authenticator): 315 """Called when a website needs authentication.""" 316 url = reply.url() 317 log.network.debug("Authentication requested for {}, netrc_used {}" 318 .format(url.toDisplayString(), self.netrc_used)) 319 320 netrc_success = False 321 if not self.netrc_used: 322 self.netrc_used = True 323 netrc_success = shared.netrc_authentication(url, authenticator) 324 325 if not netrc_success: 326 log.network.debug("Asking for credentials") 327 abort_on = self._get_abort_signals(reply) 328 shared.authentication_required(url, authenticator, 329 abort_on=abort_on) 330 331 @pyqtSlot('QNetworkProxy', 'QAuthenticator*') 332 def on_proxy_authentication_required(self, proxy, authenticator): 333 """Called when a proxy needs authentication.""" 334 proxy_id = ProxyId(proxy.type(), proxy.hostName(), proxy.port()) 335 if proxy_id in _proxy_auth_cache: 336 authinfo = _proxy_auth_cache[proxy_id] 337 authenticator.setUser(authinfo.user) 338 authenticator.setPassword(authinfo.password) 339 else: 340 msg = '<b>{}</b> says:<br/>{}'.format( 341 html.escape(proxy.hostName()), 342 html.escape(authenticator.realm())) 343 abort_on = self._get_abort_signals() 344 answer = message.ask( 345 title="Proxy authentication required", text=msg, 346 mode=usertypes.PromptMode.user_pwd, abort_on=abort_on) 347 if answer is not None: 348 authenticator.setUser(answer.user) 349 authenticator.setPassword(answer.password) 350 _proxy_auth_cache[proxy_id] = answer 351 352 @pyqtSlot() 353 def on_adopted_download_destroyed(self): 354 """Check if we can clean up if an adopted download was destroyed. 355 356 See the description for adopted_downloads for details. 357 """ 358 self.adopted_downloads -= 1 359 log.downloads.debug("Adopted download destroyed, {} left.".format( 360 self.adopted_downloads)) 361 assert self.adopted_downloads >= 0 362 if self.adopted_downloads == 0: 363 self.deleteLater() 364 365 @pyqtSlot(object) # DownloadItem 366 def adopt_download(self, download): 367 """Adopt a new DownloadItem.""" 368 self.adopted_downloads += 1 369 log.downloads.debug("Adopted download, {} adopted.".format( 370 self.adopted_downloads)) 371 download.destroyed.connect(self.on_adopted_download_destroyed) 372 download.adopt_download.connect(self.adopt_download) 373 374 def set_referer(self, req, current_url): 375 """Set the referer header.""" 376 referer_header_conf = config.val.content.headers.referer 377 378 try: 379 if referer_header_conf == 'never': 380 # Note: using ''.encode('ascii') sends a header with no value, 381 # instead of no header at all 382 req.setRawHeader('Referer'.encode('ascii'), QByteArray()) 383 elif (referer_header_conf == 'same-domain' and 384 not urlutils.same_domain(req.url(), current_url)): 385 req.setRawHeader('Referer'.encode('ascii'), QByteArray()) 386 # If refer_header_conf is set to 'always', we leave the header 387 # alone as QtWebKit did set it. 388 except urlutils.InvalidUrlError: 389 # req.url() or current_url can be invalid - this happens on 390 # https://www.playstation.com/ for example. 391 pass 392 393 def createRequest(self, op, req, outgoing_data): 394 """Return a new QNetworkReply object. 395 396 Args: 397 op: Operation op 398 req: const QNetworkRequest & req 399 outgoing_data: QIODevice * outgoingData 400 401 Return: 402 A QNetworkReply. 403 """ 404 if proxymod.application_factory is not None: 405 proxy_error = proxymod.application_factory.get_error() 406 if proxy_error is not None: 407 return networkreply.ErrorNetworkReply( 408 req, proxy_error, QNetworkReply.UnknownProxyError, 409 self) 410 411 if not req.url().isValid(): 412 log.network.debug("Ignoring invalid requested URL: {}".format( 413 req.url().errorString())) 414 return networkreply.ErrorNetworkReply( 415 req, "Invalid request URL", QNetworkReply.HostNotFoundError, 416 self) 417 418 for header, value in shared.custom_headers(url=req.url()): 419 req.setRawHeader(header, value) 420 421 tab = self._get_tab() 422 current_url = QUrl() 423 if tab is not None: 424 try: 425 current_url = tab.url() 426 except RuntimeError: 427 # We could be in the middle of the webpage shutdown here. 428 pass 429 430 request = interceptors.Request(first_party_url=current_url, 431 request_url=req.url()) 432 interceptors.run(request) 433 if request.is_blocked: 434 return networkreply.ErrorNetworkReply( 435 req, HOSTBLOCK_ERROR_STRING, QNetworkReply.ContentAccessDenied, 436 self) 437 438 if 'log-requests' in objects.debug_flags: 439 operation = debug.qenum_key(QNetworkAccessManager, op) 440 operation = operation.replace('Operation', '').upper() 441 log.network.debug("{} {}, first-party {}".format( 442 operation, 443 req.url().toDisplayString(), 444 current_url.toDisplayString())) 445 446 scheme = req.url().scheme() 447 if scheme in self._scheme_handlers: 448 result = self._scheme_handlers[scheme](req, op, current_url) 449 if result is not None: 450 result.setParent(self) 451 return result 452 453 self.set_referer(req, current_url) 454 return super().createRequest(op, req, outgoing_data) 455