1/** 2 * Copyright 2020 Google Inc. All rights reserved. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16import { ProtocolMapping } from 'devtools-protocol/types/protocol-mapping.js'; 17 18import { EventEmitter } from './EventEmitter.js'; 19import { Frame } from './FrameManager.js'; 20import { HTTPResponse } from './HTTPResponse.js'; 21import { assert } from './assert.js'; 22import { helper, debugError } from './helper.js'; 23import { Protocol } from 'devtools-protocol'; 24import { ProtocolError } from './Errors.js'; 25 26/** 27 * @public 28 */ 29export interface ContinueRequestOverrides { 30 /** 31 * If set, the request URL will change. This is not a redirect. 32 */ 33 url?: string; 34 method?: string; 35 postData?: string; 36 headers?: Record<string, string>; 37} 38 39/** 40 * @public 41 */ 42export interface InterceptResolutionState { 43 action: InterceptResolutionAction; 44 priority?: number; 45} 46 47/** 48 * Required response data to fulfill a request with. 49 * 50 * @public 51 */ 52export interface ResponseForRequest { 53 status: number; 54 /** 55 * Optional response headers. All values are converted to strings. 56 */ 57 headers: Record<string, unknown>; 58 contentType: string; 59 body: string | Buffer; 60} 61 62/** 63 * Resource types for HTTPRequests as perceived by the rendering engine. 64 * 65 * @public 66 */ 67export type ResourceType = Lowercase<Protocol.Network.ResourceType>; 68 69/** 70 * The default cooperative request interception resolution priority 71 * 72 * @public 73 */ 74export const DEFAULT_INTERCEPT_RESOLUTION_PRIORITY = 0; 75 76interface CDPSession extends EventEmitter { 77 send<T extends keyof ProtocolMapping.Commands>( 78 method: T, 79 ...paramArgs: ProtocolMapping.Commands[T]['paramsType'] 80 ): Promise<ProtocolMapping.Commands[T]['returnType']>; 81} 82 83/** 84 * 85 * Represents an HTTP request sent by a page. 86 * @remarks 87 * 88 * Whenever the page sends a request, such as for a network resource, the 89 * following events are emitted by Puppeteer's `page`: 90 * 91 * - `request`: emitted when the request is issued by the page. 92 * - `requestfinished` - emitted when the response body is downloaded and the 93 * request is complete. 94 * 95 * If request fails at some point, then instead of `requestfinished` event the 96 * `requestfailed` event is emitted. 97 * 98 * All of these events provide an instance of `HTTPRequest` representing the 99 * request that occurred: 100 * 101 * ``` 102 * page.on('request', request => ...) 103 * ``` 104 * 105 * NOTE: HTTP Error responses, such as 404 or 503, are still successful 106 * responses from HTTP standpoint, so request will complete with 107 * `requestfinished` event. 108 * 109 * If request gets a 'redirect' response, the request is successfully finished 110 * with the `requestfinished` event, and a new request is issued to a 111 * redirected url. 112 * 113 * @public 114 */ 115export class HTTPRequest { 116 /** 117 * @internal 118 */ 119 _requestId: string; 120 /** 121 * @internal 122 */ 123 _interceptionId: string; 124 /** 125 * @internal 126 */ 127 _failureText = null; 128 /** 129 * @internal 130 */ 131 _response: HTTPResponse | null = null; 132 /** 133 * @internal 134 */ 135 _fromMemoryCache = false; 136 /** 137 * @internal 138 */ 139 _redirectChain: HTTPRequest[]; 140 141 private _client: CDPSession; 142 private _isNavigationRequest: boolean; 143 private _allowInterception: boolean; 144 private _interceptionHandled = false; 145 private _url: string; 146 private _resourceType: ResourceType; 147 148 private _method: string; 149 private _postData?: string; 150 private _headers: Record<string, string> = {}; 151 private _frame: Frame; 152 private _continueRequestOverrides: ContinueRequestOverrides; 153 private _responseForRequest: Partial<ResponseForRequest>; 154 private _abortErrorReason: Protocol.Network.ErrorReason; 155 private _interceptResolutionState: InterceptResolutionState; 156 private _interceptHandlers: Array<() => void | PromiseLike<any>>; 157 private _initiator: Protocol.Network.Initiator; 158 159 /** 160 * @internal 161 */ 162 constructor( 163 client: CDPSession, 164 frame: Frame, 165 interceptionId: string, 166 allowInterception: boolean, 167 event: Protocol.Network.RequestWillBeSentEvent, 168 redirectChain: HTTPRequest[] 169 ) { 170 this._client = client; 171 this._requestId = event.requestId; 172 this._isNavigationRequest = 173 event.requestId === event.loaderId && event.type === 'Document'; 174 this._interceptionId = interceptionId; 175 this._allowInterception = allowInterception; 176 this._url = event.request.url; 177 this._resourceType = event.type.toLowerCase() as ResourceType; 178 this._method = event.request.method; 179 this._postData = event.request.postData; 180 this._frame = frame; 181 this._redirectChain = redirectChain; 182 this._continueRequestOverrides = {}; 183 this._interceptResolutionState = { action: InterceptResolutionAction.None }; 184 this._interceptHandlers = []; 185 this._initiator = event.initiator; 186 187 for (const key of Object.keys(event.request.headers)) 188 this._headers[key.toLowerCase()] = event.request.headers[key]; 189 } 190 191 /** 192 * @returns the URL of the request 193 */ 194 url(): string { 195 return this._url; 196 } 197 198 /** 199 * @returns the `ContinueRequestOverrides` that will be used 200 * if the interception is allowed to continue (ie, `abort()` and 201 * `respond()` aren't called). 202 */ 203 continueRequestOverrides(): ContinueRequestOverrides { 204 assert(this._allowInterception, 'Request Interception is not enabled!'); 205 return this._continueRequestOverrides; 206 } 207 208 /** 209 * @returns The `ResponseForRequest` that gets used if the 210 * interception is allowed to respond (ie, `abort()` is not called). 211 */ 212 responseForRequest(): Partial<ResponseForRequest> { 213 assert(this._allowInterception, 'Request Interception is not enabled!'); 214 return this._responseForRequest; 215 } 216 217 /** 218 * @returns the most recent reason for aborting the request 219 */ 220 abortErrorReason(): Protocol.Network.ErrorReason { 221 assert(this._allowInterception, 'Request Interception is not enabled!'); 222 return this._abortErrorReason; 223 } 224 225 /** 226 * @returns An InterceptResolutionState object describing the current resolution 227 * action and priority. 228 * 229 * InterceptResolutionState contains: 230 * action: InterceptResolutionAction 231 * priority?: number 232 * 233 * InterceptResolutionAction is one of: `abort`, `respond`, `continue`, 234 * `disabled`, `none`, or `already-handled`. 235 */ 236 interceptResolutionState(): InterceptResolutionState { 237 if (!this._allowInterception) 238 return { action: InterceptResolutionAction.Disabled }; 239 if (this._interceptionHandled) 240 return { action: InterceptResolutionAction.AlreadyHandled }; 241 return { ...this._interceptResolutionState }; 242 } 243 244 /** 245 * @returns `true` if the intercept resolution has already been handled, 246 * `false` otherwise. 247 */ 248 isInterceptResolutionHandled(): boolean { 249 return this._interceptionHandled; 250 } 251 252 /** 253 * Adds an async request handler to the processing queue. 254 * Deferred handlers are not guaranteed to execute in any particular order, 255 * but they are guarnateed to resolve before the request interception 256 * is finalized. 257 */ 258 enqueueInterceptAction( 259 pendingHandler: () => void | PromiseLike<unknown> 260 ): void { 261 this._interceptHandlers.push(pendingHandler); 262 } 263 264 /** 265 * Awaits pending interception handlers and then decides how to fulfill 266 * the request interception. 267 */ 268 async finalizeInterceptions(): Promise<void> { 269 await this._interceptHandlers.reduce( 270 (promiseChain, interceptAction) => promiseChain.then(interceptAction), 271 Promise.resolve() 272 ); 273 const { action } = this.interceptResolutionState(); 274 switch (action) { 275 case 'abort': 276 return this._abort(this._abortErrorReason); 277 case 'respond': 278 return this._respond(this._responseForRequest); 279 case 'continue': 280 return this._continue(this._continueRequestOverrides); 281 } 282 } 283 284 /** 285 * Contains the request's resource type as it was perceived by the rendering 286 * engine. 287 */ 288 resourceType(): ResourceType { 289 return this._resourceType; 290 } 291 292 /** 293 * @returns the method used (`GET`, `POST`, etc.) 294 */ 295 method(): string { 296 return this._method; 297 } 298 299 /** 300 * @returns the request's post body, if any. 301 */ 302 postData(): string | undefined { 303 return this._postData; 304 } 305 306 /** 307 * @returns an object with HTTP headers associated with the request. All 308 * header names are lower-case. 309 */ 310 headers(): Record<string, string> { 311 return this._headers; 312 } 313 314 /** 315 * @returns A matching `HTTPResponse` object, or null if the response has not 316 * been received yet. 317 */ 318 response(): HTTPResponse | null { 319 return this._response; 320 } 321 322 /** 323 * @returns the frame that initiated the request, or null if navigating to 324 * error pages. 325 */ 326 frame(): Frame | null { 327 return this._frame; 328 } 329 330 /** 331 * @returns true if the request is the driver of the current frame's navigation. 332 */ 333 isNavigationRequest(): boolean { 334 return this._isNavigationRequest; 335 } 336 337 /** 338 * @returns the initiator of the request. 339 */ 340 initiator(): Protocol.Network.Initiator { 341 return this._initiator; 342 } 343 344 /** 345 * A `redirectChain` is a chain of requests initiated to fetch a resource. 346 * @remarks 347 * 348 * `redirectChain` is shared between all the requests of the same chain. 349 * 350 * For example, if the website `http://example.com` has a single redirect to 351 * `https://example.com`, then the chain will contain one request: 352 * 353 * ```js 354 * const response = await page.goto('http://example.com'); 355 * const chain = response.request().redirectChain(); 356 * console.log(chain.length); // 1 357 * console.log(chain[0].url()); // 'http://example.com' 358 * ``` 359 * 360 * If the website `https://google.com` has no redirects, then the chain will be empty: 361 * 362 * ```js 363 * const response = await page.goto('https://google.com'); 364 * const chain = response.request().redirectChain(); 365 * console.log(chain.length); // 0 366 * ``` 367 * 368 * @returns the chain of requests - if a server responds with at least a 369 * single redirect, this chain will contain all requests that were redirected. 370 */ 371 redirectChain(): HTTPRequest[] { 372 return this._redirectChain.slice(); 373 } 374 375 /** 376 * Access information about the request's failure. 377 * 378 * @remarks 379 * 380 * @example 381 * 382 * Example of logging all failed requests: 383 * 384 * ```js 385 * page.on('requestfailed', request => { 386 * console.log(request.url() + ' ' + request.failure().errorText); 387 * }); 388 * ``` 389 * 390 * @returns `null` unless the request failed. If the request fails this can 391 * return an object with `errorText` containing a human-readable error 392 * message, e.g. `net::ERR_FAILED`. It is not guaranteeded that there will be 393 * failure text if the request fails. 394 */ 395 failure(): { errorText: string } | null { 396 if (!this._failureText) return null; 397 return { 398 errorText: this._failureText, 399 }; 400 } 401 402 /** 403 * Continues request with optional request overrides. 404 * 405 * @remarks 406 * 407 * To use this, request 408 * interception should be enabled with {@link Page.setRequestInterception}. 409 * 410 * Exception is immediately thrown if the request interception is not enabled. 411 * 412 * @example 413 * ```js 414 * await page.setRequestInterception(true); 415 * page.on('request', request => { 416 * // Override headers 417 * const headers = Object.assign({}, request.headers(), { 418 * foo: 'bar', // set "foo" header 419 * origin: undefined, // remove "origin" header 420 * }); 421 * request.continue({headers}); 422 * }); 423 * ``` 424 * 425 * @param overrides - optional overrides to apply to the request. 426 * @param priority - If provided, intercept is resolved using 427 * cooperative handling rules. Otherwise, intercept is resolved 428 * immediately. 429 */ 430 async continue( 431 overrides: ContinueRequestOverrides = {}, 432 priority?: number 433 ): Promise<void> { 434 // Request interception is not supported for data: urls. 435 if (this._url.startsWith('data:')) return; 436 assert(this._allowInterception, 'Request Interception is not enabled!'); 437 assert(!this._interceptionHandled, 'Request is already handled!'); 438 if (priority === undefined) { 439 return this._continue(overrides); 440 } 441 this._continueRequestOverrides = overrides; 442 if ( 443 priority > this._interceptResolutionState.priority || 444 this._interceptResolutionState.priority === undefined 445 ) { 446 this._interceptResolutionState = { 447 action: InterceptResolutionAction.Continue, 448 priority, 449 }; 450 return; 451 } 452 if (priority === this._interceptResolutionState.priority) { 453 if ( 454 this._interceptResolutionState.action === 'abort' || 455 this._interceptResolutionState.action === 'respond' 456 ) { 457 return; 458 } 459 this._interceptResolutionState.action = 460 InterceptResolutionAction.Continue; 461 } 462 return; 463 } 464 465 private async _continue( 466 overrides: ContinueRequestOverrides = {} 467 ): Promise<void> { 468 const { url, method, postData, headers } = overrides; 469 this._interceptionHandled = true; 470 471 const postDataBinaryBase64 = postData 472 ? Buffer.from(postData).toString('base64') 473 : undefined; 474 475 await this._client 476 .send('Fetch.continueRequest', { 477 requestId: this._interceptionId, 478 url, 479 method, 480 postData: postDataBinaryBase64, 481 headers: headers ? headersArray(headers) : undefined, 482 }) 483 .catch((error) => { 484 this._interceptionHandled = false; 485 return handleError(error); 486 }); 487 } 488 489 /** 490 * Fulfills a request with the given response. 491 * 492 * @remarks 493 * 494 * To use this, request 495 * interception should be enabled with {@link Page.setRequestInterception}. 496 * 497 * Exception is immediately thrown if the request interception is not enabled. 498 * 499 * @example 500 * An example of fulfilling all requests with 404 responses: 501 * ```js 502 * await page.setRequestInterception(true); 503 * page.on('request', request => { 504 * request.respond({ 505 * status: 404, 506 * contentType: 'text/plain', 507 * body: 'Not Found!' 508 * }); 509 * }); 510 * ``` 511 * 512 * NOTE: Mocking responses for dataURL requests is not supported. 513 * Calling `request.respond` for a dataURL request is a noop. 514 * 515 * @param response - the response to fulfill the request with. 516 * @param priority - If provided, intercept is resolved using 517 * cooperative handling rules. Otherwise, intercept is resolved 518 * immediately. 519 */ 520 async respond( 521 response: Partial<ResponseForRequest>, 522 priority?: number 523 ): Promise<void> { 524 // Mocking responses for dataURL requests is not currently supported. 525 if (this._url.startsWith('data:')) return; 526 assert(this._allowInterception, 'Request Interception is not enabled!'); 527 assert(!this._interceptionHandled, 'Request is already handled!'); 528 if (priority === undefined) { 529 return this._respond(response); 530 } 531 this._responseForRequest = response; 532 if ( 533 priority > this._interceptResolutionState.priority || 534 this._interceptResolutionState.priority === undefined 535 ) { 536 this._interceptResolutionState = { 537 action: InterceptResolutionAction.Respond, 538 priority, 539 }; 540 return; 541 } 542 if (priority === this._interceptResolutionState.priority) { 543 if (this._interceptResolutionState.action === 'abort') { 544 return; 545 } 546 this._interceptResolutionState.action = InterceptResolutionAction.Respond; 547 } 548 } 549 550 private async _respond(response: Partial<ResponseForRequest>): Promise<void> { 551 this._interceptionHandled = true; 552 553 const responseBody: Buffer | null = 554 response.body && helper.isString(response.body) 555 ? Buffer.from(response.body) 556 : (response.body as Buffer) || null; 557 558 const responseHeaders: Record<string, string> = {}; 559 if (response.headers) { 560 for (const header of Object.keys(response.headers)) 561 responseHeaders[header.toLowerCase()] = String( 562 response.headers[header] 563 ); 564 } 565 if (response.contentType) 566 responseHeaders['content-type'] = response.contentType; 567 if (responseBody && !('content-length' in responseHeaders)) 568 responseHeaders['content-length'] = String( 569 Buffer.byteLength(responseBody) 570 ); 571 572 await this._client 573 .send('Fetch.fulfillRequest', { 574 requestId: this._interceptionId, 575 responseCode: response.status || 200, 576 responsePhrase: STATUS_TEXTS[response.status || 200], 577 responseHeaders: headersArray(responseHeaders), 578 body: responseBody ? responseBody.toString('base64') : undefined, 579 }) 580 .catch((error) => { 581 this._interceptionHandled = false; 582 return handleError(error); 583 }); 584 } 585 586 /** 587 * Aborts a request. 588 * 589 * @remarks 590 * To use this, request interception should be enabled with 591 * {@link Page.setRequestInterception}. If it is not enabled, this method will 592 * throw an exception immediately. 593 * 594 * @param errorCode - optional error code to provide. 595 * @param priority - If provided, intercept is resolved using 596 * cooperative handling rules. Otherwise, intercept is resolved 597 * immediately. 598 */ 599 async abort( 600 errorCode: ErrorCode = 'failed', 601 priority?: number 602 ): Promise<void> { 603 // Request interception is not supported for data: urls. 604 if (this._url.startsWith('data:')) return; 605 const errorReason = errorReasons[errorCode]; 606 assert(errorReason, 'Unknown error code: ' + errorCode); 607 assert(this._allowInterception, 'Request Interception is not enabled!'); 608 assert(!this._interceptionHandled, 'Request is already handled!'); 609 if (priority === undefined) { 610 return this._abort(errorReason); 611 } 612 this._abortErrorReason = errorReason; 613 if ( 614 priority >= this._interceptResolutionState.priority || 615 this._interceptResolutionState.priority === undefined 616 ) { 617 this._interceptResolutionState = { 618 action: InterceptResolutionAction.Abort, 619 priority, 620 }; 621 return; 622 } 623 } 624 625 private async _abort( 626 errorReason: Protocol.Network.ErrorReason 627 ): Promise<void> { 628 this._interceptionHandled = true; 629 await this._client 630 .send('Fetch.failRequest', { 631 requestId: this._interceptionId, 632 errorReason, 633 }) 634 .catch(handleError); 635 } 636} 637 638/** 639 * @public 640 */ 641export enum InterceptResolutionAction { 642 Abort = 'abort', 643 Respond = 'respond', 644 Continue = 'continue', 645 Disabled = 'disabled', 646 None = 'none', 647 AlreadyHandled = 'already-handled', 648} 649 650/** 651 * @public 652 * 653 * @deprecated please use {@link InterceptResolutionAction} instead. 654 */ 655export type InterceptResolutionStrategy = InterceptResolutionAction; 656 657/** 658 * @public 659 */ 660export type ErrorCode = 661 | 'aborted' 662 | 'accessdenied' 663 | 'addressunreachable' 664 | 'blockedbyclient' 665 | 'blockedbyresponse' 666 | 'connectionaborted' 667 | 'connectionclosed' 668 | 'connectionfailed' 669 | 'connectionrefused' 670 | 'connectionreset' 671 | 'internetdisconnected' 672 | 'namenotresolved' 673 | 'timedout' 674 | 'failed'; 675 676const errorReasons: Record<ErrorCode, Protocol.Network.ErrorReason> = { 677 aborted: 'Aborted', 678 accessdenied: 'AccessDenied', 679 addressunreachable: 'AddressUnreachable', 680 blockedbyclient: 'BlockedByClient', 681 blockedbyresponse: 'BlockedByResponse', 682 connectionaborted: 'ConnectionAborted', 683 connectionclosed: 'ConnectionClosed', 684 connectionfailed: 'ConnectionFailed', 685 connectionrefused: 'ConnectionRefused', 686 connectionreset: 'ConnectionReset', 687 internetdisconnected: 'InternetDisconnected', 688 namenotresolved: 'NameNotResolved', 689 timedout: 'TimedOut', 690 failed: 'Failed', 691} as const; 692 693/** 694 * @public 695 */ 696export type ActionResult = 'continue' | 'abort' | 'respond'; 697 698function headersArray( 699 headers: Record<string, string> 700): Array<{ name: string; value: string }> { 701 const result = []; 702 for (const name in headers) { 703 if (!Object.is(headers[name], undefined)) 704 result.push({ name, value: headers[name] + '' }); 705 } 706 return result; 707} 708 709async function handleError(error: ProtocolError) { 710 if (['Invalid header'].includes(error.originalMessage)) { 711 throw error; 712 } 713 // In certain cases, protocol will return error if the request was 714 // already canceled or the page was closed. We should tolerate these 715 // errors. 716 debugError(error); 717} 718 719// List taken from 720// https://www.iana.org/assignments/http-status-codes/http-status-codes.xhtml 721// with extra 306 and 418 codes. 722const STATUS_TEXTS = { 723 '100': 'Continue', 724 '101': 'Switching Protocols', 725 '102': 'Processing', 726 '103': 'Early Hints', 727 '200': 'OK', 728 '201': 'Created', 729 '202': 'Accepted', 730 '203': 'Non-Authoritative Information', 731 '204': 'No Content', 732 '205': 'Reset Content', 733 '206': 'Partial Content', 734 '207': 'Multi-Status', 735 '208': 'Already Reported', 736 '226': 'IM Used', 737 '300': 'Multiple Choices', 738 '301': 'Moved Permanently', 739 '302': 'Found', 740 '303': 'See Other', 741 '304': 'Not Modified', 742 '305': 'Use Proxy', 743 '306': 'Switch Proxy', 744 '307': 'Temporary Redirect', 745 '308': 'Permanent Redirect', 746 '400': 'Bad Request', 747 '401': 'Unauthorized', 748 '402': 'Payment Required', 749 '403': 'Forbidden', 750 '404': 'Not Found', 751 '405': 'Method Not Allowed', 752 '406': 'Not Acceptable', 753 '407': 'Proxy Authentication Required', 754 '408': 'Request Timeout', 755 '409': 'Conflict', 756 '410': 'Gone', 757 '411': 'Length Required', 758 '412': 'Precondition Failed', 759 '413': 'Payload Too Large', 760 '414': 'URI Too Long', 761 '415': 'Unsupported Media Type', 762 '416': 'Range Not Satisfiable', 763 '417': 'Expectation Failed', 764 '418': "I'm a teapot", 765 '421': 'Misdirected Request', 766 '422': 'Unprocessable Entity', 767 '423': 'Locked', 768 '424': 'Failed Dependency', 769 '425': 'Too Early', 770 '426': 'Upgrade Required', 771 '428': 'Precondition Required', 772 '429': 'Too Many Requests', 773 '431': 'Request Header Fields Too Large', 774 '451': 'Unavailable For Legal Reasons', 775 '500': 'Internal Server Error', 776 '501': 'Not Implemented', 777 '502': 'Bad Gateway', 778 '503': 'Service Unavailable', 779 '504': 'Gateway Timeout', 780 '505': 'HTTP Version Not Supported', 781 '506': 'Variant Also Negotiates', 782 '507': 'Insufficient Storage', 783 '508': 'Loop Detected', 784 '510': 'Not Extended', 785 '511': 'Network Authentication Required', 786} as const; 787