1/** 2 * Copyright 2017 Google Inc. All rights reserved. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17import * as os from 'os'; 18import * as fs from 'fs'; 19import * as path from 'path'; 20import * as util from 'util'; 21import * as childProcess from 'child_process'; 22import * as https from 'https'; 23import * as http from 'http'; 24 25import { Product } from '../common/Product.js'; 26import extractZip from 'extract-zip'; 27import { debug } from '../common/Debug.js'; 28import { promisify } from 'util'; 29import removeRecursive from 'rimraf'; 30import * as URL from 'url'; 31import createHttpsProxyAgent, { 32 HttpsProxyAgent, 33 HttpsProxyAgentOptions, 34} from 'https-proxy-agent'; 35import { getProxyForUrl } from 'proxy-from-env'; 36import { assert } from '../common/assert.js'; 37 38const debugFetcher = debug('puppeteer:fetcher'); 39 40const downloadURLs = { 41 chrome: { 42 linux: '%s/chromium-browser-snapshots/Linux_x64/%d/%s.zip', 43 mac: '%s/chromium-browser-snapshots/Mac/%d/%s.zip', 44 win32: '%s/chromium-browser-snapshots/Win/%d/%s.zip', 45 win64: '%s/chromium-browser-snapshots/Win_x64/%d/%s.zip', 46 }, 47 firefox: { 48 linux: '%s/firefox-%s.en-US.%s-x86_64.tar.bz2', 49 mac: '%s/firefox-%s.en-US.%s.dmg', 50 win32: '%s/firefox-%s.en-US.%s.zip', 51 win64: '%s/firefox-%s.en-US.%s.zip', 52 }, 53} as const; 54 55const browserConfig = { 56 chrome: { 57 host: 'https://storage.googleapis.com', 58 destination: '.local-chromium', 59 }, 60 firefox: { 61 host: 'https://archive.mozilla.org/pub/firefox/nightly/latest-mozilla-central', 62 destination: '.local-firefox', 63 }, 64} as const; 65 66/** 67 * Supported platforms. 68 * @public 69 */ 70export type Platform = 'linux' | 'mac' | 'win32' | 'win64'; 71 72function archiveName( 73 product: Product, 74 platform: Platform, 75 revision: string 76): string { 77 if (product === 'chrome') { 78 if (platform === 'linux') return 'chrome-linux'; 79 if (platform === 'mac') return 'chrome-mac'; 80 if (platform === 'win32' || platform === 'win64') { 81 // Windows archive name changed at r591479. 82 return parseInt(revision, 10) > 591479 ? 'chrome-win' : 'chrome-win32'; 83 } 84 } else if (product === 'firefox') { 85 return platform; 86 } 87} 88 89/** 90 * @internal 91 */ 92function downloadURL( 93 product: Product, 94 platform: Platform, 95 host: string, 96 revision: string 97): string { 98 const url = util.format( 99 downloadURLs[product][platform], 100 host, 101 revision, 102 archiveName(product, platform, revision) 103 ); 104 return url; 105} 106 107/** 108 * @internal 109 */ 110function handleArm64(): void { 111 fs.stat('/usr/bin/chromium-browser', function (err, stats) { 112 if (stats === undefined) { 113 fs.stat('/usr/bin/chromium', function (err, stats) { 114 if (stats === undefined) { 115 console.error( 116 'The chromium binary is not available for arm64.' + 117 '\nIf you are on Ubuntu, you can install with: ' + 118 '\n\n sudo apt install chromium\n' + 119 '\n\n sudo apt install chromium-browser\n' 120 ); 121 throw new Error(); 122 } 123 }); 124 } 125 }); 126} 127const readdirAsync = promisify(fs.readdir.bind(fs)); 128const mkdirAsync = promisify(fs.mkdir.bind(fs)); 129const unlinkAsync = promisify(fs.unlink.bind(fs)); 130const chmodAsync = promisify(fs.chmod.bind(fs)); 131 132function existsAsync(filePath: string): Promise<boolean> { 133 return new Promise((resolve) => { 134 fs.access(filePath, (err) => resolve(!err)); 135 }); 136} 137 138/** 139 * @public 140 */ 141export interface BrowserFetcherOptions { 142 platform?: Platform; 143 product?: string; 144 path?: string; 145 host?: string; 146} 147 148/** 149 * @public 150 */ 151export interface BrowserFetcherRevisionInfo { 152 folderPath: string; 153 executablePath: string; 154 url: string; 155 local: boolean; 156 revision: string; 157 product: string; 158} 159/** 160 * BrowserFetcher can download and manage different versions of Chromium and Firefox. 161 * 162 * @remarks 163 * BrowserFetcher operates on revision strings that specify a precise version of Chromium, e.g. `"533271"`. Revision strings can be obtained from {@link http://omahaproxy.appspot.com/ | omahaproxy.appspot.com}. 164 * In the Firefox case, BrowserFetcher downloads Firefox Nightly and 165 * operates on version numbers such as `"75"`. 166 * 167 * @example 168 * An example of using BrowserFetcher to download a specific version of Chromium 169 * and running Puppeteer against it: 170 * 171 * ```js 172 * const browserFetcher = puppeteer.createBrowserFetcher(); 173 * const revisionInfo = await browserFetcher.download('533271'); 174 * const browser = await puppeteer.launch({executablePath: revisionInfo.executablePath}) 175 * ``` 176 * 177 * **NOTE** BrowserFetcher is not designed to work concurrently with other 178 * instances of BrowserFetcher that share the same downloads directory. 179 * 180 * @public 181 */ 182 183export class BrowserFetcher { 184 private _product: Product; 185 private _downloadsFolder: string; 186 private _downloadHost: string; 187 private _platform: Platform; 188 189 /** 190 * @internal 191 */ 192 constructor(projectRoot: string, options: BrowserFetcherOptions = {}) { 193 this._product = (options.product || 'chrome').toLowerCase() as Product; 194 assert( 195 this._product === 'chrome' || this._product === 'firefox', 196 `Unknown product: "${options.product}"` 197 ); 198 199 this._downloadsFolder = 200 options.path || 201 path.join(projectRoot, browserConfig[this._product].destination); 202 this._downloadHost = options.host || browserConfig[this._product].host; 203 this.setPlatform(options.platform); 204 assert( 205 downloadURLs[this._product][this._platform], 206 'Unsupported platform: ' + this._platform 207 ); 208 } 209 210 private setPlatform(platformFromOptions?: Platform): void { 211 if (platformFromOptions) { 212 this._platform = platformFromOptions; 213 return; 214 } 215 216 const platform = os.platform(); 217 if (platform === 'darwin') this._platform = 'mac'; 218 else if (platform === 'linux') this._platform = 'linux'; 219 else if (platform === 'win32') 220 this._platform = os.arch() === 'x64' ? 'win64' : 'win32'; 221 else assert(this._platform, 'Unsupported platform: ' + platform); 222 } 223 224 /** 225 * @returns Returns the current `Platform`, which is one of `mac`, `linux`, 226 * `win32` or `win64`. 227 */ 228 platform(): Platform { 229 return this._platform; 230 } 231 232 /** 233 * @returns Returns the current `Product`, which is one of `chrome` or 234 * `firefox`. 235 */ 236 product(): Product { 237 return this._product; 238 } 239 240 /** 241 * @returns The download host being used. 242 */ 243 host(): string { 244 return this._downloadHost; 245 } 246 247 /** 248 * Initiates a HEAD request to check if the revision is available. 249 * @remarks 250 * This method is affected by the current `product`. 251 * @param revision - The revision to check availability for. 252 * @returns A promise that resolves to `true` if the revision could be downloaded 253 * from the host. 254 */ 255 canDownload(revision: string): Promise<boolean> { 256 const url = downloadURL( 257 this._product, 258 this._platform, 259 this._downloadHost, 260 revision 261 ); 262 return new Promise((resolve) => { 263 const request = httpRequest(url, 'HEAD', (response) => { 264 resolve(response.statusCode === 200); 265 }); 266 request.on('error', (error) => { 267 console.error(error); 268 resolve(false); 269 }); 270 }); 271 } 272 273 /** 274 * Initiates a GET request to download the revision from the host. 275 * @remarks 276 * This method is affected by the current `product`. 277 * @param revision - The revision to download. 278 * @param progressCallback - A function that will be called with two arguments: 279 * How many bytes have been downloaded and the total number of bytes of the download. 280 * @returns A promise with revision information when the revision is downloaded 281 * and extracted. 282 */ 283 async download( 284 revision: string, 285 progressCallback: (x: number, y: number) => void = (): void => {} 286 ): Promise<BrowserFetcherRevisionInfo> { 287 const url = downloadURL( 288 this._product, 289 this._platform, 290 this._downloadHost, 291 revision 292 ); 293 const fileName = url.split('/').pop(); 294 const archivePath = path.join(this._downloadsFolder, fileName); 295 const outputPath = this._getFolderPath(revision); 296 if (await existsAsync(outputPath)) return this.revisionInfo(revision); 297 if (!(await existsAsync(this._downloadsFolder))) 298 await mkdirAsync(this._downloadsFolder); 299 300 // Use Intel x86 builds on Apple M1 until native macOS arm64 301 // Chromium builds are available. 302 if (os.platform() !== 'darwin' && os.arch() === 'arm64') { 303 handleArm64(); 304 return; 305 } 306 try { 307 await downloadFile(url, archivePath, progressCallback); 308 await install(archivePath, outputPath); 309 } finally { 310 if (await existsAsync(archivePath)) await unlinkAsync(archivePath); 311 } 312 const revisionInfo = this.revisionInfo(revision); 313 if (revisionInfo) await chmodAsync(revisionInfo.executablePath, 0o755); 314 return revisionInfo; 315 } 316 317 /** 318 * @remarks 319 * This method is affected by the current `product`. 320 * @returns A promise with a list of all revision strings (for the current `product`) 321 * available locally on disk. 322 */ 323 async localRevisions(): Promise<string[]> { 324 if (!(await existsAsync(this._downloadsFolder))) return []; 325 const fileNames = await readdirAsync(this._downloadsFolder); 326 return fileNames 327 .map((fileName) => parseFolderPath(this._product, fileName)) 328 .filter((entry) => entry && entry.platform === this._platform) 329 .map((entry) => entry.revision); 330 } 331 332 /** 333 * @remarks 334 * This method is affected by the current `product`. 335 * @param revision - A revision to remove for the current `product`. 336 * @returns A promise that resolves when the revision has been removes or 337 * throws if the revision has not been downloaded. 338 */ 339 async remove(revision: string): Promise<void> { 340 const folderPath = this._getFolderPath(revision); 341 assert( 342 await existsAsync(folderPath), 343 `Failed to remove: revision ${revision} is not downloaded` 344 ); 345 await new Promise((fulfill) => removeRecursive(folderPath, fulfill)); 346 } 347 348 /** 349 * @param revision - The revision to get info for. 350 * @returns The revision info for the given revision. 351 */ 352 revisionInfo(revision: string): BrowserFetcherRevisionInfo { 353 const folderPath = this._getFolderPath(revision); 354 let executablePath = ''; 355 if (this._product === 'chrome') { 356 if (this._platform === 'mac') 357 executablePath = path.join( 358 folderPath, 359 archiveName(this._product, this._platform, revision), 360 'Chromium.app', 361 'Contents', 362 'MacOS', 363 'Chromium' 364 ); 365 else if (this._platform === 'linux') 366 executablePath = path.join( 367 folderPath, 368 archiveName(this._product, this._platform, revision), 369 'chrome' 370 ); 371 else if (this._platform === 'win32' || this._platform === 'win64') 372 executablePath = path.join( 373 folderPath, 374 archiveName(this._product, this._platform, revision), 375 'chrome.exe' 376 ); 377 else throw new Error('Unsupported platform: ' + this._platform); 378 } else if (this._product === 'firefox') { 379 if (this._platform === 'mac') 380 executablePath = path.join( 381 folderPath, 382 'Firefox Nightly.app', 383 'Contents', 384 'MacOS', 385 'firefox' 386 ); 387 else if (this._platform === 'linux') 388 executablePath = path.join(folderPath, 'firefox', 'firefox'); 389 else if (this._platform === 'win32' || this._platform === 'win64') 390 executablePath = path.join(folderPath, 'firefox', 'firefox.exe'); 391 else throw new Error('Unsupported platform: ' + this._platform); 392 } else throw new Error('Unsupported product: ' + this._product); 393 const url = downloadURL( 394 this._product, 395 this._platform, 396 this._downloadHost, 397 revision 398 ); 399 const local = fs.existsSync(folderPath); 400 debugFetcher({ 401 revision, 402 executablePath, 403 folderPath, 404 local, 405 url, 406 product: this._product, 407 }); 408 return { 409 revision, 410 executablePath, 411 folderPath, 412 local, 413 url, 414 product: this._product, 415 }; 416 } 417 418 /** 419 * @internal 420 */ 421 _getFolderPath(revision: string): string { 422 return path.resolve(this._downloadsFolder, `${this._platform}-${revision}`); 423 } 424} 425 426function parseFolderPath( 427 product: Product, 428 folderPath: string 429): { product: string; platform: string; revision: string } | null { 430 const name = path.basename(folderPath); 431 const splits = name.split('-'); 432 if (splits.length !== 2) return null; 433 const [platform, revision] = splits; 434 if (!downloadURLs[product][platform]) return null; 435 return { product, platform, revision }; 436} 437 438/** 439 * @internal 440 */ 441function downloadFile( 442 url: string, 443 destinationPath: string, 444 progressCallback: (x: number, y: number) => void 445): Promise<void> { 446 debugFetcher(`Downloading binary from ${url}`); 447 let fulfill, reject; 448 let downloadedBytes = 0; 449 let totalBytes = 0; 450 451 const promise = new Promise<void>((x, y) => { 452 fulfill = x; 453 reject = y; 454 }); 455 456 const request = httpRequest(url, 'GET', (response) => { 457 if (response.statusCode !== 200) { 458 const error = new Error( 459 `Download failed: server returned code ${response.statusCode}. URL: ${url}` 460 ); 461 // consume response data to free up memory 462 response.resume(); 463 reject(error); 464 return; 465 } 466 const file = fs.createWriteStream(destinationPath); 467 file.on('finish', () => fulfill()); 468 file.on('error', (error) => reject(error)); 469 response.pipe(file); 470 totalBytes = parseInt( 471 /** @type {string} */ response.headers['content-length'], 472 10 473 ); 474 if (progressCallback) response.on('data', onData); 475 }); 476 request.on('error', (error) => reject(error)); 477 return promise; 478 479 function onData(chunk: string): void { 480 downloadedBytes += chunk.length; 481 progressCallback(downloadedBytes, totalBytes); 482 } 483} 484 485function install(archivePath: string, folderPath: string): Promise<unknown> { 486 debugFetcher(`Installing ${archivePath} to ${folderPath}`); 487 if (archivePath.endsWith('.zip')) 488 return extractZip(archivePath, { dir: folderPath }); 489 else if (archivePath.endsWith('.tar.bz2')) 490 return extractTar(archivePath, folderPath); 491 else if (archivePath.endsWith('.dmg')) 492 return mkdirAsync(folderPath).then(() => 493 installDMG(archivePath, folderPath) 494 ); 495 else throw new Error(`Unsupported archive format: ${archivePath}`); 496} 497 498/** 499 * @internal 500 */ 501function extractTar(tarPath: string, folderPath: string): Promise<unknown> { 502 // eslint-disable-next-line @typescript-eslint/no-var-requires 503 const tar = require('tar-fs'); 504 // eslint-disable-next-line @typescript-eslint/no-var-requires 505 const bzip = require('unbzip2-stream'); 506 return new Promise((fulfill, reject) => { 507 const tarStream = tar.extract(folderPath); 508 tarStream.on('error', reject); 509 tarStream.on('finish', fulfill); 510 const readStream = fs.createReadStream(tarPath); 511 readStream.pipe(bzip()).pipe(tarStream); 512 }); 513} 514 515/** 516 * @internal 517 */ 518function installDMG(dmgPath: string, folderPath: string): Promise<void> { 519 let mountPath; 520 521 function mountAndCopy(fulfill: () => void, reject: (Error) => void): void { 522 const mountCommand = `hdiutil attach -nobrowse -noautoopen "${dmgPath}"`; 523 childProcess.exec(mountCommand, (err, stdout) => { 524 if (err) return reject(err); 525 const volumes = stdout.match(/\/Volumes\/(.*)/m); 526 if (!volumes) 527 return reject(new Error(`Could not find volume path in ${stdout}`)); 528 mountPath = volumes[0]; 529 readdirAsync(mountPath) 530 .then((fileNames) => { 531 const appName = fileNames.find( 532 (item) => typeof item === 'string' && item.endsWith('.app') 533 ); 534 if (!appName) 535 return reject(new Error(`Cannot find app in ${mountPath}`)); 536 const copyPath = path.join(mountPath, appName); 537 debugFetcher(`Copying ${copyPath} to ${folderPath}`); 538 childProcess.exec(`cp -R "${copyPath}" "${folderPath}"`, (err) => { 539 if (err) reject(err); 540 else fulfill(); 541 }); 542 }) 543 .catch(reject); 544 }); 545 } 546 547 function unmount(): void { 548 if (!mountPath) return; 549 const unmountCommand = `hdiutil detach "${mountPath}" -quiet`; 550 debugFetcher(`Unmounting ${mountPath}`); 551 childProcess.exec(unmountCommand, (err) => { 552 if (err) console.error(`Error unmounting dmg: ${err}`); 553 }); 554 } 555 556 return new Promise<void>(mountAndCopy) 557 .catch((error) => { 558 console.error(error); 559 }) 560 .finally(unmount); 561} 562 563function httpRequest( 564 url: string, 565 method: string, 566 response: (x: http.IncomingMessage) => void 567): http.ClientRequest { 568 const urlParsed = URL.parse(url); 569 570 type Options = Partial<URL.UrlWithStringQuery> & { 571 method?: string; 572 agent?: HttpsProxyAgent; 573 rejectUnauthorized?: boolean; 574 }; 575 576 let options: Options = { 577 ...urlParsed, 578 method, 579 }; 580 581 const proxyURL = getProxyForUrl(url); 582 if (proxyURL) { 583 if (url.startsWith('http:')) { 584 const proxy = URL.parse(proxyURL); 585 options = { 586 path: options.href, 587 host: proxy.hostname, 588 port: proxy.port, 589 }; 590 } else { 591 const parsedProxyURL = URL.parse(proxyURL); 592 593 const proxyOptions = { 594 ...parsedProxyURL, 595 secureProxy: parsedProxyURL.protocol === 'https:', 596 } as HttpsProxyAgentOptions; 597 598 options.agent = createHttpsProxyAgent(proxyOptions); 599 options.rejectUnauthorized = false; 600 } 601 } 602 603 const requestCallback = (res: http.IncomingMessage): void => { 604 if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) 605 httpRequest(res.headers.location, method, response); 606 else response(res); 607 }; 608 const request = 609 options.protocol === 'https:' 610 ? https.request(options, requestCallback) 611 : http.request(options, requestCallback); 612 request.end(); 613 return request; 614} 615