1/** 2 * Copyright 2017 Google Inc. All rights reserved. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16import { EventEmitter } from './EventEmitter.js'; 17import { assert } from './assert.js'; 18import { helper, debugError } from './helper.js'; 19import { Protocol } from 'devtools-protocol'; 20import { CDPSession } from './Connection.js'; 21import { FrameManager } from './FrameManager.js'; 22import { HTTPRequest } from './HTTPRequest.js'; 23import { HTTPResponse } from './HTTPResponse.js'; 24 25/** 26 * @public 27 */ 28export interface Credentials { 29 username: string; 30 password: string; 31} 32 33/** 34 * @public 35 */ 36export interface NetworkConditions { 37 // Download speed (bytes/s) 38 download: number; 39 // Upload speed (bytes/s) 40 upload: number; 41 // Latency (ms) 42 latency: number; 43} 44/** 45 * @public 46 */ 47export interface InternalNetworkConditions extends NetworkConditions { 48 offline: boolean; 49} 50 51/** 52 * We use symbols to prevent any external parties listening to these events. 53 * They are internal to Puppeteer. 54 * 55 * @internal 56 */ 57export const NetworkManagerEmittedEvents = { 58 Request: Symbol('NetworkManager.Request'), 59 RequestServedFromCache: Symbol('NetworkManager.RequestServedFromCache'), 60 Response: Symbol('NetworkManager.Response'), 61 RequestFailed: Symbol('NetworkManager.RequestFailed'), 62 RequestFinished: Symbol('NetworkManager.RequestFinished'), 63} as const; 64 65/** 66 * @internal 67 */ 68export class NetworkManager extends EventEmitter { 69 _client: CDPSession; 70 _ignoreHTTPSErrors: boolean; 71 _frameManager: FrameManager; 72 73 /* 74 * There are four possible orders of events: 75 * A. `_onRequestWillBeSent` 76 * B. `_onRequestWillBeSent`, `_onRequestPaused` 77 * C. `_onRequestPaused`, `_onRequestWillBeSent` 78 * D. `_onRequestPaused`, `_onRequestWillBeSent`, `_onRequestPaused` 79 * (see crbug.com/1196004) 80 * 81 * For `_onRequest` we need the event from `_onRequestWillBeSent` and 82 * optionally the `interceptionId` from `_onRequestPaused`. 83 * 84 * If request interception is disabled, call `_onRequest` once per call to 85 * `_onRequestWillBeSent`. 86 * If request interception is enabled, call `_onRequest` once per call to 87 * `_onRequestPaused` (once per `interceptionId`). 88 * 89 * Events are stored to allow for subsequent events to call `_onRequest`. 90 * 91 * Note that (chains of) redirect requests have the same `requestId` (!) as 92 * the original request. We have to anticipate series of events like these: 93 * A. `_onRequestWillBeSent`, 94 * `_onRequestWillBeSent`, ... 95 * B. `_onRequestWillBeSent`, `_onRequestPaused`, 96 * `_onRequestWillBeSent`, `_onRequestPaused`, ... 97 * C. `_onRequestWillBeSent`, `_onRequestPaused`, 98 * `_onRequestPaused`, `_onRequestWillBeSent`, ... 99 * D. `_onRequestPaused`, `_onRequestWillBeSent`, 100 * `_onRequestPaused`, `_onRequestWillBeSent`, `_onRequestPaused`, ... 101 * (see crbug.com/1196004) 102 */ 103 _requestIdToRequestWillBeSentEvent = new Map< 104 string, 105 Protocol.Network.RequestWillBeSentEvent 106 >(); 107 _requestIdToRequestPausedEvent = new Map< 108 string, 109 Protocol.Fetch.RequestPausedEvent 110 >(); 111 _requestIdToRequest = new Map<string, HTTPRequest>(); 112 113 _extraHTTPHeaders: Record<string, string> = {}; 114 _credentials?: Credentials = null; 115 _attemptedAuthentications = new Set<string>(); 116 _userRequestInterceptionEnabled = false; 117 _protocolRequestInterceptionEnabled = false; 118 _userCacheDisabled = false; 119 _emulatedNetworkConditions: InternalNetworkConditions = { 120 offline: false, 121 upload: -1, 122 download: -1, 123 latency: 0, 124 }; 125 126 constructor( 127 client: CDPSession, 128 ignoreHTTPSErrors: boolean, 129 frameManager: FrameManager 130 ) { 131 super(); 132 this._client = client; 133 this._ignoreHTTPSErrors = ignoreHTTPSErrors; 134 this._frameManager = frameManager; 135 136 this._client.on('Fetch.requestPaused', this._onRequestPaused.bind(this)); 137 this._client.on('Fetch.authRequired', this._onAuthRequired.bind(this)); 138 this._client.on( 139 'Network.requestWillBeSent', 140 this._onRequestWillBeSent.bind(this) 141 ); 142 this._client.on( 143 'Network.requestServedFromCache', 144 this._onRequestServedFromCache.bind(this) 145 ); 146 this._client.on( 147 'Network.responseReceived', 148 this._onResponseReceived.bind(this) 149 ); 150 this._client.on( 151 'Network.loadingFinished', 152 this._onLoadingFinished.bind(this) 153 ); 154 this._client.on('Network.loadingFailed', this._onLoadingFailed.bind(this)); 155 } 156 157 async initialize(): Promise<void> { 158 await this._client.send('Network.enable'); 159 if (this._ignoreHTTPSErrors) 160 await this._client.send('Security.setIgnoreCertificateErrors', { 161 ignore: true, 162 }); 163 } 164 165 async authenticate(credentials?: Credentials): Promise<void> { 166 this._credentials = credentials; 167 await this._updateProtocolRequestInterception(); 168 } 169 170 async setExtraHTTPHeaders( 171 extraHTTPHeaders: Record<string, string> 172 ): Promise<void> { 173 this._extraHTTPHeaders = {}; 174 for (const key of Object.keys(extraHTTPHeaders)) { 175 const value = extraHTTPHeaders[key]; 176 assert( 177 helper.isString(value), 178 `Expected value of header "${key}" to be String, but "${typeof value}" is found.` 179 ); 180 this._extraHTTPHeaders[key.toLowerCase()] = value; 181 } 182 await this._client.send('Network.setExtraHTTPHeaders', { 183 headers: this._extraHTTPHeaders, 184 }); 185 } 186 187 extraHTTPHeaders(): Record<string, string> { 188 return Object.assign({}, this._extraHTTPHeaders); 189 } 190 191 async setOfflineMode(value: boolean): Promise<void> { 192 this._emulatedNetworkConditions.offline = value; 193 await this._updateNetworkConditions(); 194 } 195 196 async emulateNetworkConditions( 197 networkConditions: NetworkConditions | null 198 ): Promise<void> { 199 this._emulatedNetworkConditions.upload = networkConditions 200 ? networkConditions.upload 201 : -1; 202 this._emulatedNetworkConditions.download = networkConditions 203 ? networkConditions.download 204 : -1; 205 this._emulatedNetworkConditions.latency = networkConditions 206 ? networkConditions.latency 207 : 0; 208 209 await this._updateNetworkConditions(); 210 } 211 212 async _updateNetworkConditions(): Promise<void> { 213 await this._client.send('Network.emulateNetworkConditions', { 214 offline: this._emulatedNetworkConditions.offline, 215 latency: this._emulatedNetworkConditions.latency, 216 uploadThroughput: this._emulatedNetworkConditions.upload, 217 downloadThroughput: this._emulatedNetworkConditions.download, 218 }); 219 } 220 221 async setUserAgent(userAgent: string): Promise<void> { 222 await this._client.send('Network.setUserAgentOverride', { userAgent }); 223 } 224 225 async setCacheEnabled(enabled: boolean): Promise<void> { 226 this._userCacheDisabled = !enabled; 227 await this._updateProtocolCacheDisabled(); 228 } 229 230 async setRequestInterception(value: boolean): Promise<void> { 231 this._userRequestInterceptionEnabled = value; 232 await this._updateProtocolRequestInterception(); 233 } 234 235 async _updateProtocolRequestInterception(): Promise<void> { 236 const enabled = this._userRequestInterceptionEnabled || !!this._credentials; 237 if (enabled === this._protocolRequestInterceptionEnabled) return; 238 this._protocolRequestInterceptionEnabled = enabled; 239 if (enabled) { 240 await Promise.all([ 241 this._updateProtocolCacheDisabled(), 242 this._client.send('Fetch.enable', { 243 handleAuthRequests: true, 244 patterns: [{ urlPattern: '*' }], 245 }), 246 ]); 247 } else { 248 await Promise.all([ 249 this._updateProtocolCacheDisabled(), 250 this._client.send('Fetch.disable'), 251 ]); 252 } 253 } 254 255 _cacheDisabled(): boolean { 256 return this._userCacheDisabled; 257 } 258 259 async _updateProtocolCacheDisabled(): Promise<void> { 260 await this._client.send('Network.setCacheDisabled', { 261 cacheDisabled: this._cacheDisabled(), 262 }); 263 } 264 265 _onRequestWillBeSent(event: Protocol.Network.RequestWillBeSentEvent): void { 266 // Request interception doesn't happen for data URLs with Network Service. 267 if ( 268 this._userRequestInterceptionEnabled && 269 !event.request.url.startsWith('data:') 270 ) { 271 const requestId = event.requestId; 272 const requestPausedEvent = 273 this._requestIdToRequestPausedEvent.get(requestId); 274 275 this._requestIdToRequestWillBeSentEvent.set(requestId, event); 276 277 if (requestPausedEvent) { 278 const interceptionId = requestPausedEvent.requestId; 279 this._onRequest(event, interceptionId); 280 this._requestIdToRequestPausedEvent.delete(requestId); 281 } 282 283 return; 284 } 285 this._onRequest(event, null); 286 } 287 288 _onAuthRequired(event: Protocol.Fetch.AuthRequiredEvent): void { 289 /* TODO(jacktfranklin): This is defined in protocol.d.ts but not 290 * in an easily referrable way - we should look at exposing it. 291 */ 292 type AuthResponse = 'Default' | 'CancelAuth' | 'ProvideCredentials'; 293 let response: AuthResponse = 'Default'; 294 if (this._attemptedAuthentications.has(event.requestId)) { 295 response = 'CancelAuth'; 296 } else if (this._credentials) { 297 response = 'ProvideCredentials'; 298 this._attemptedAuthentications.add(event.requestId); 299 } 300 const { username, password } = this._credentials || { 301 username: undefined, 302 password: undefined, 303 }; 304 this._client 305 .send('Fetch.continueWithAuth', { 306 requestId: event.requestId, 307 authChallengeResponse: { response, username, password }, 308 }) 309 .catch(debugError); 310 } 311 312 _onRequestPaused(event: Protocol.Fetch.RequestPausedEvent): void { 313 if ( 314 !this._userRequestInterceptionEnabled && 315 this._protocolRequestInterceptionEnabled 316 ) { 317 this._client 318 .send('Fetch.continueRequest', { 319 requestId: event.requestId, 320 }) 321 .catch(debugError); 322 } 323 324 const requestId = event.networkId; 325 const interceptionId = event.requestId; 326 327 if (!requestId) { 328 return; 329 } 330 331 let requestWillBeSentEvent = 332 this._requestIdToRequestWillBeSentEvent.get(requestId); 333 334 // redirect requests have the same `requestId`, 335 if ( 336 requestWillBeSentEvent && 337 (requestWillBeSentEvent.request.url !== event.request.url || 338 requestWillBeSentEvent.request.method !== event.request.method) 339 ) { 340 this._requestIdToRequestWillBeSentEvent.delete(requestId); 341 requestWillBeSentEvent = null; 342 } 343 344 if (requestWillBeSentEvent) { 345 this._onRequest(requestWillBeSentEvent, interceptionId); 346 this._requestIdToRequestWillBeSentEvent.delete(requestId); 347 } else { 348 this._requestIdToRequestPausedEvent.set(requestId, event); 349 } 350 } 351 352 _onRequest( 353 event: Protocol.Network.RequestWillBeSentEvent, 354 interceptionId?: string 355 ): void { 356 let redirectChain = []; 357 if (event.redirectResponse) { 358 const request = this._requestIdToRequest.get(event.requestId); 359 // If we connect late to the target, we could have missed the 360 // requestWillBeSent event. 361 if (request) { 362 this._handleRequestRedirect(request, event.redirectResponse); 363 redirectChain = request._redirectChain; 364 } 365 } 366 const frame = event.frameId 367 ? this._frameManager.frame(event.frameId) 368 : null; 369 const request = new HTTPRequest( 370 this._client, 371 frame, 372 interceptionId, 373 this._userRequestInterceptionEnabled, 374 event, 375 redirectChain 376 ); 377 this._requestIdToRequest.set(event.requestId, request); 378 this.emit(NetworkManagerEmittedEvents.Request, request); 379 } 380 381 _onRequestServedFromCache( 382 event: Protocol.Network.RequestServedFromCacheEvent 383 ): void { 384 const request = this._requestIdToRequest.get(event.requestId); 385 if (request) request._fromMemoryCache = true; 386 this.emit(NetworkManagerEmittedEvents.RequestServedFromCache, request); 387 } 388 389 _handleRequestRedirect( 390 request: HTTPRequest, 391 responsePayload: Protocol.Network.Response 392 ): void { 393 const response = new HTTPResponse(this._client, request, responsePayload); 394 request._response = response; 395 request._redirectChain.push(request); 396 response._resolveBody( 397 new Error('Response body is unavailable for redirect responses') 398 ); 399 this._forgetRequest(request, false); 400 this.emit(NetworkManagerEmittedEvents.Response, response); 401 this.emit(NetworkManagerEmittedEvents.RequestFinished, request); 402 } 403 404 _onResponseReceived(event: Protocol.Network.ResponseReceivedEvent): void { 405 const request = this._requestIdToRequest.get(event.requestId); 406 // FileUpload sends a response without a matching request. 407 if (!request) return; 408 const response = new HTTPResponse(this._client, request, event.response); 409 request._response = response; 410 this.emit(NetworkManagerEmittedEvents.Response, response); 411 } 412 413 _forgetRequest(request: HTTPRequest, events: boolean): void { 414 const requestId = request._requestId; 415 const interceptionId = request._interceptionId; 416 417 this._requestIdToRequest.delete(requestId); 418 this._attemptedAuthentications.delete(interceptionId); 419 420 if (events) { 421 this._requestIdToRequestWillBeSentEvent.delete(requestId); 422 this._requestIdToRequestPausedEvent.delete(requestId); 423 } 424 } 425 426 _onLoadingFinished(event: Protocol.Network.LoadingFinishedEvent): void { 427 const request = this._requestIdToRequest.get(event.requestId); 428 // For certain requestIds we never receive requestWillBeSent event. 429 // @see https://crbug.com/750469 430 if (!request) return; 431 432 // Under certain conditions we never get the Network.responseReceived 433 // event from protocol. @see https://crbug.com/883475 434 if (request.response()) request.response()._resolveBody(null); 435 this._forgetRequest(request, true); 436 this.emit(NetworkManagerEmittedEvents.RequestFinished, request); 437 } 438 439 _onLoadingFailed(event: Protocol.Network.LoadingFailedEvent): void { 440 const request = this._requestIdToRequest.get(event.requestId); 441 // For certain requestIds we never receive requestWillBeSent event. 442 // @see https://crbug.com/750469 443 if (!request) return; 444 request._failureText = event.errorText; 445 const response = request.response(); 446 if (response) response._resolveBody(null); 447 this._forgetRequest(request, true); 448 this.emit(NetworkManagerEmittedEvents.RequestFailed, request); 449 } 450} 451