1/**
2 * Copyright 2017 Google Inc. All rights reserved.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *     http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16import { EventEmitter } from './EventEmitter.js';
17import { assert } from './assert.js';
18import { helper, debugError } from './helper.js';
19import { Protocol } from 'devtools-protocol';
20import { CDPSession } from './Connection.js';
21import { FrameManager } from './FrameManager.js';
22import { HTTPRequest } from './HTTPRequest.js';
23import { HTTPResponse } from './HTTPResponse.js';
24
25/**
26 * @public
27 */
28export interface Credentials {
29  username: string;
30  password: string;
31}
32
33/**
34 * @public
35 */
36export interface NetworkConditions {
37  // Download speed (bytes/s)
38  download: number;
39  // Upload speed (bytes/s)
40  upload: number;
41  // Latency (ms)
42  latency: number;
43}
44/**
45 * @public
46 */
47export interface InternalNetworkConditions extends NetworkConditions {
48  offline: boolean;
49}
50
51/**
52 * We use symbols to prevent any external parties listening to these events.
53 * They are internal to Puppeteer.
54 *
55 * @internal
56 */
57export const NetworkManagerEmittedEvents = {
58  Request: Symbol('NetworkManager.Request'),
59  RequestServedFromCache: Symbol('NetworkManager.RequestServedFromCache'),
60  Response: Symbol('NetworkManager.Response'),
61  RequestFailed: Symbol('NetworkManager.RequestFailed'),
62  RequestFinished: Symbol('NetworkManager.RequestFinished'),
63} as const;
64
65/**
66 * @internal
67 */
68export class NetworkManager extends EventEmitter {
69  _client: CDPSession;
70  _ignoreHTTPSErrors: boolean;
71  _frameManager: FrameManager;
72
73  /*
74   * There are four possible orders of events:
75   *  A. `_onRequestWillBeSent`
76   *  B. `_onRequestWillBeSent`, `_onRequestPaused`
77   *  C. `_onRequestPaused`, `_onRequestWillBeSent`
78   *  D. `_onRequestPaused`, `_onRequestWillBeSent`, `_onRequestPaused`
79   *     (see crbug.com/1196004)
80   *
81   * For `_onRequest` we need the event from `_onRequestWillBeSent` and
82   * optionally the `interceptionId` from `_onRequestPaused`.
83   *
84   * If request interception is disabled, call `_onRequest` once per call to
85   * `_onRequestWillBeSent`.
86   * If request interception is enabled, call `_onRequest` once per call to
87   * `_onRequestPaused` (once per `interceptionId`).
88   *
89   * Events are stored to allow for subsequent events to call `_onRequest`.
90   *
91   * Note that (chains of) redirect requests have the same `requestId` (!) as
92   * the original request. We have to anticipate series of events like these:
93   *  A. `_onRequestWillBeSent`,
94   *     `_onRequestWillBeSent`, ...
95   *  B. `_onRequestWillBeSent`, `_onRequestPaused`,
96   *     `_onRequestWillBeSent`, `_onRequestPaused`, ...
97   *  C. `_onRequestWillBeSent`, `_onRequestPaused`,
98   *     `_onRequestPaused`, `_onRequestWillBeSent`, ...
99   *  D. `_onRequestPaused`, `_onRequestWillBeSent`,
100   *     `_onRequestPaused`, `_onRequestWillBeSent`, `_onRequestPaused`, ...
101   *     (see crbug.com/1196004)
102   */
103  _requestIdToRequestWillBeSentEvent = new Map<
104    string,
105    Protocol.Network.RequestWillBeSentEvent
106  >();
107  _requestIdToRequestPausedEvent = new Map<
108    string,
109    Protocol.Fetch.RequestPausedEvent
110  >();
111  _requestIdToRequest = new Map<string, HTTPRequest>();
112
113  _extraHTTPHeaders: Record<string, string> = {};
114  _credentials?: Credentials = null;
115  _attemptedAuthentications = new Set<string>();
116  _userRequestInterceptionEnabled = false;
117  _protocolRequestInterceptionEnabled = false;
118  _userCacheDisabled = false;
119  _emulatedNetworkConditions: InternalNetworkConditions = {
120    offline: false,
121    upload: -1,
122    download: -1,
123    latency: 0,
124  };
125
126  constructor(
127    client: CDPSession,
128    ignoreHTTPSErrors: boolean,
129    frameManager: FrameManager
130  ) {
131    super();
132    this._client = client;
133    this._ignoreHTTPSErrors = ignoreHTTPSErrors;
134    this._frameManager = frameManager;
135
136    this._client.on('Fetch.requestPaused', this._onRequestPaused.bind(this));
137    this._client.on('Fetch.authRequired', this._onAuthRequired.bind(this));
138    this._client.on(
139      'Network.requestWillBeSent',
140      this._onRequestWillBeSent.bind(this)
141    );
142    this._client.on(
143      'Network.requestServedFromCache',
144      this._onRequestServedFromCache.bind(this)
145    );
146    this._client.on(
147      'Network.responseReceived',
148      this._onResponseReceived.bind(this)
149    );
150    this._client.on(
151      'Network.loadingFinished',
152      this._onLoadingFinished.bind(this)
153    );
154    this._client.on('Network.loadingFailed', this._onLoadingFailed.bind(this));
155  }
156
157  async initialize(): Promise<void> {
158    await this._client.send('Network.enable');
159    if (this._ignoreHTTPSErrors)
160      await this._client.send('Security.setIgnoreCertificateErrors', {
161        ignore: true,
162      });
163  }
164
165  async authenticate(credentials?: Credentials): Promise<void> {
166    this._credentials = credentials;
167    await this._updateProtocolRequestInterception();
168  }
169
170  async setExtraHTTPHeaders(
171    extraHTTPHeaders: Record<string, string>
172  ): Promise<void> {
173    this._extraHTTPHeaders = {};
174    for (const key of Object.keys(extraHTTPHeaders)) {
175      const value = extraHTTPHeaders[key];
176      assert(
177        helper.isString(value),
178        `Expected value of header "${key}" to be String, but "${typeof value}" is found.`
179      );
180      this._extraHTTPHeaders[key.toLowerCase()] = value;
181    }
182    await this._client.send('Network.setExtraHTTPHeaders', {
183      headers: this._extraHTTPHeaders,
184    });
185  }
186
187  extraHTTPHeaders(): Record<string, string> {
188    return Object.assign({}, this._extraHTTPHeaders);
189  }
190
191  async setOfflineMode(value: boolean): Promise<void> {
192    this._emulatedNetworkConditions.offline = value;
193    await this._updateNetworkConditions();
194  }
195
196  async emulateNetworkConditions(
197    networkConditions: NetworkConditions | null
198  ): Promise<void> {
199    this._emulatedNetworkConditions.upload = networkConditions
200      ? networkConditions.upload
201      : -1;
202    this._emulatedNetworkConditions.download = networkConditions
203      ? networkConditions.download
204      : -1;
205    this._emulatedNetworkConditions.latency = networkConditions
206      ? networkConditions.latency
207      : 0;
208
209    await this._updateNetworkConditions();
210  }
211
212  async _updateNetworkConditions(): Promise<void> {
213    await this._client.send('Network.emulateNetworkConditions', {
214      offline: this._emulatedNetworkConditions.offline,
215      latency: this._emulatedNetworkConditions.latency,
216      uploadThroughput: this._emulatedNetworkConditions.upload,
217      downloadThroughput: this._emulatedNetworkConditions.download,
218    });
219  }
220
221  async setUserAgent(userAgent: string): Promise<void> {
222    await this._client.send('Network.setUserAgentOverride', { userAgent });
223  }
224
225  async setCacheEnabled(enabled: boolean): Promise<void> {
226    this._userCacheDisabled = !enabled;
227    await this._updateProtocolCacheDisabled();
228  }
229
230  async setRequestInterception(value: boolean): Promise<void> {
231    this._userRequestInterceptionEnabled = value;
232    await this._updateProtocolRequestInterception();
233  }
234
235  async _updateProtocolRequestInterception(): Promise<void> {
236    const enabled = this._userRequestInterceptionEnabled || !!this._credentials;
237    if (enabled === this._protocolRequestInterceptionEnabled) return;
238    this._protocolRequestInterceptionEnabled = enabled;
239    if (enabled) {
240      await Promise.all([
241        this._updateProtocolCacheDisabled(),
242        this._client.send('Fetch.enable', {
243          handleAuthRequests: true,
244          patterns: [{ urlPattern: '*' }],
245        }),
246      ]);
247    } else {
248      await Promise.all([
249        this._updateProtocolCacheDisabled(),
250        this._client.send('Fetch.disable'),
251      ]);
252    }
253  }
254
255  _cacheDisabled(): boolean {
256    return this._userCacheDisabled;
257  }
258
259  async _updateProtocolCacheDisabled(): Promise<void> {
260    await this._client.send('Network.setCacheDisabled', {
261      cacheDisabled: this._cacheDisabled(),
262    });
263  }
264
265  _onRequestWillBeSent(event: Protocol.Network.RequestWillBeSentEvent): void {
266    // Request interception doesn't happen for data URLs with Network Service.
267    if (
268      this._userRequestInterceptionEnabled &&
269      !event.request.url.startsWith('data:')
270    ) {
271      const requestId = event.requestId;
272      const requestPausedEvent =
273        this._requestIdToRequestPausedEvent.get(requestId);
274
275      this._requestIdToRequestWillBeSentEvent.set(requestId, event);
276
277      if (requestPausedEvent) {
278        const interceptionId = requestPausedEvent.requestId;
279        this._onRequest(event, interceptionId);
280        this._requestIdToRequestPausedEvent.delete(requestId);
281      }
282
283      return;
284    }
285    this._onRequest(event, null);
286  }
287
288  _onAuthRequired(event: Protocol.Fetch.AuthRequiredEvent): void {
289    /* TODO(jacktfranklin): This is defined in protocol.d.ts but not
290     * in an easily referrable way - we should look at exposing it.
291     */
292    type AuthResponse = 'Default' | 'CancelAuth' | 'ProvideCredentials';
293    let response: AuthResponse = 'Default';
294    if (this._attemptedAuthentications.has(event.requestId)) {
295      response = 'CancelAuth';
296    } else if (this._credentials) {
297      response = 'ProvideCredentials';
298      this._attemptedAuthentications.add(event.requestId);
299    }
300    const { username, password } = this._credentials || {
301      username: undefined,
302      password: undefined,
303    };
304    this._client
305      .send('Fetch.continueWithAuth', {
306        requestId: event.requestId,
307        authChallengeResponse: { response, username, password },
308      })
309      .catch(debugError);
310  }
311
312  _onRequestPaused(event: Protocol.Fetch.RequestPausedEvent): void {
313    if (
314      !this._userRequestInterceptionEnabled &&
315      this._protocolRequestInterceptionEnabled
316    ) {
317      this._client
318        .send('Fetch.continueRequest', {
319          requestId: event.requestId,
320        })
321        .catch(debugError);
322    }
323
324    const requestId = event.networkId;
325    const interceptionId = event.requestId;
326
327    if (!requestId) {
328      return;
329    }
330
331    let requestWillBeSentEvent =
332      this._requestIdToRequestWillBeSentEvent.get(requestId);
333
334    // redirect requests have the same `requestId`,
335    if (
336      requestWillBeSentEvent &&
337      (requestWillBeSentEvent.request.url !== event.request.url ||
338        requestWillBeSentEvent.request.method !== event.request.method)
339    ) {
340      this._requestIdToRequestWillBeSentEvent.delete(requestId);
341      requestWillBeSentEvent = null;
342    }
343
344    if (requestWillBeSentEvent) {
345      this._onRequest(requestWillBeSentEvent, interceptionId);
346      this._requestIdToRequestWillBeSentEvent.delete(requestId);
347    } else {
348      this._requestIdToRequestPausedEvent.set(requestId, event);
349    }
350  }
351
352  _onRequest(
353    event: Protocol.Network.RequestWillBeSentEvent,
354    interceptionId?: string
355  ): void {
356    let redirectChain = [];
357    if (event.redirectResponse) {
358      const request = this._requestIdToRequest.get(event.requestId);
359      // If we connect late to the target, we could have missed the
360      // requestWillBeSent event.
361      if (request) {
362        this._handleRequestRedirect(request, event.redirectResponse);
363        redirectChain = request._redirectChain;
364      }
365    }
366    const frame = event.frameId
367      ? this._frameManager.frame(event.frameId)
368      : null;
369    const request = new HTTPRequest(
370      this._client,
371      frame,
372      interceptionId,
373      this._userRequestInterceptionEnabled,
374      event,
375      redirectChain
376    );
377    this._requestIdToRequest.set(event.requestId, request);
378    this.emit(NetworkManagerEmittedEvents.Request, request);
379  }
380
381  _onRequestServedFromCache(
382    event: Protocol.Network.RequestServedFromCacheEvent
383  ): void {
384    const request = this._requestIdToRequest.get(event.requestId);
385    if (request) request._fromMemoryCache = true;
386    this.emit(NetworkManagerEmittedEvents.RequestServedFromCache, request);
387  }
388
389  _handleRequestRedirect(
390    request: HTTPRequest,
391    responsePayload: Protocol.Network.Response
392  ): void {
393    const response = new HTTPResponse(this._client, request, responsePayload);
394    request._response = response;
395    request._redirectChain.push(request);
396    response._resolveBody(
397      new Error('Response body is unavailable for redirect responses')
398    );
399    this._forgetRequest(request, false);
400    this.emit(NetworkManagerEmittedEvents.Response, response);
401    this.emit(NetworkManagerEmittedEvents.RequestFinished, request);
402  }
403
404  _onResponseReceived(event: Protocol.Network.ResponseReceivedEvent): void {
405    const request = this._requestIdToRequest.get(event.requestId);
406    // FileUpload sends a response without a matching request.
407    if (!request) return;
408    const response = new HTTPResponse(this._client, request, event.response);
409    request._response = response;
410    this.emit(NetworkManagerEmittedEvents.Response, response);
411  }
412
413  _forgetRequest(request: HTTPRequest, events: boolean): void {
414    const requestId = request._requestId;
415    const interceptionId = request._interceptionId;
416
417    this._requestIdToRequest.delete(requestId);
418    this._attemptedAuthentications.delete(interceptionId);
419
420    if (events) {
421      this._requestIdToRequestWillBeSentEvent.delete(requestId);
422      this._requestIdToRequestPausedEvent.delete(requestId);
423    }
424  }
425
426  _onLoadingFinished(event: Protocol.Network.LoadingFinishedEvent): void {
427    const request = this._requestIdToRequest.get(event.requestId);
428    // For certain requestIds we never receive requestWillBeSent event.
429    // @see https://crbug.com/750469
430    if (!request) return;
431
432    // Under certain conditions we never get the Network.responseReceived
433    // event from protocol. @see https://crbug.com/883475
434    if (request.response()) request.response()._resolveBody(null);
435    this._forgetRequest(request, true);
436    this.emit(NetworkManagerEmittedEvents.RequestFinished, request);
437  }
438
439  _onLoadingFailed(event: Protocol.Network.LoadingFailedEvent): void {
440    const request = this._requestIdToRequest.get(event.requestId);
441    // For certain requestIds we never receive requestWillBeSent event.
442    // @see https://crbug.com/750469
443    if (!request) return;
444    request._failureText = event.errorText;
445    const response = request.response();
446    if (response) response._resolveBody(null);
447    this._forgetRequest(request, true);
448    this.emit(NetworkManagerEmittedEvents.RequestFailed, request);
449  }
450}
451