1#!/usr/bin/python3 -u 2# -*- coding: utf-8 -*- 3 4# This Source Code Form is subject to the terms of the Mozilla Public 5# License, v. 2.0. If a copy of the MPL was not distributed with this 6# file, You can obtain one at http://mozilla.org/MPL/2.0/. 7 8""" 9This script downloads the latest chromium build (or a manually 10defined version) for a given platform. It then uploads the build, 11with the revision of the build stored in a REVISION file. 12""" 13 14from __future__ import absolute_import, print_function 15 16import argparse 17import errno 18import os 19import shutil 20import subprocess 21import requests 22import tempfile 23 24 25LAST_CHANGE_URL = ( 26 # formatted with platform 27 "https://www.googleapis.com/download/storage/v1/b/" 28 "chromium-browser-snapshots/o/{}%2FLAST_CHANGE?alt=media" 29) 30 31CHROMIUM_BASE_URL = ( 32 # formatted with (platform/revision/archive) 33 "https://www.googleapis.com/download/storage/v1/b/" 34 "chromium-browser-snapshots/o/{}%2F{}%2F{}?alt=media" 35) 36 37 38CHROMIUM_INFO = { 39 "linux": { 40 "platform": "Linux_x64", 41 "chromium": "chrome-linux.zip", 42 "result": "chromium-linux.tar.bz2", 43 "chromedriver": "chromedriver_linux64.zip", 44 }, 45 "win32": { 46 "platform": "Win", 47 "chromium": "chrome-win.zip", 48 "result": "chromium-win32.tar.bz2", 49 "chromedriver": "chromedriver_win32.zip", 50 }, 51 "win64": { 52 "platform": "Win", 53 "chromium": "chrome-win.zip", 54 "result": "chromium-win64.tar.bz2", 55 "chromedriver": "chromedriver_win32.zip", 56 }, 57 "mac": { 58 "platform": "Mac", 59 "chromium": "chrome-mac.zip", 60 "result": "chromium-mac.tar.bz2", 61 "chromedriver": "chromedriver_mac64.zip", 62 }, 63} 64 65 66def log(msg): 67 print("build-chromium: %s" % msg) 68 69 70def fetch_file(url, filepath): 71 """Download a file from the given url to a given file.""" 72 size = 4096 73 r = requests.get(url, stream=True) 74 r.raise_for_status() 75 76 with open(filepath, "wb") as fd: 77 for chunk in r.iter_content(size): 78 fd.write(chunk) 79 80 81def unzip(zippath, target): 82 """Unzips an archive to the target location.""" 83 log("Unpacking archive at: %s to: %s" % (zippath, target)) 84 unzip_command = ["unzip", "-q", "-o", zippath, "-d", target] 85 subprocess.check_call(unzip_command) 86 87 88def fetch_chromium_revision(platform): 89 """Get the revision of the latest chromium build. """ 90 chromium_platform = CHROMIUM_INFO[platform]["platform"] 91 revision_url = LAST_CHANGE_URL.format(chromium_platform) 92 93 log("Getting revision number for latest %s chromium build..." % chromium_platform) 94 95 # Expecting a file with a single number indicating the latest 96 # chromium build with a chromedriver that we can download 97 r = requests.get(revision_url, timeout=30) 98 r.raise_for_status() 99 100 chromium_revision = r.content.decode("utf-8") 101 return chromium_revision.strip() 102 103 104def fetch_chromium_build(platform, revision, zippath): 105 """Download a chromium build for a given revision, or the latest. """ 106 if not revision: 107 revision = fetch_chromium_revision(platform) 108 109 download_platform = CHROMIUM_INFO[platform]["platform"] 110 download_url = CHROMIUM_BASE_URL.format( 111 download_platform, revision, CHROMIUM_INFO[platform]["chromium"] 112 ) 113 114 log("Downloading %s chromium build revision %s..." % (download_platform, revision)) 115 116 fetch_file(download_url, zippath) 117 return revision 118 119 120def fetch_chromedriver(platform, revision, chromium_dir): 121 """Get the chromedriver for the given revision and repackage it.""" 122 download_url = CHROMIUM_BASE_URL.format( 123 CHROMIUM_INFO[platform]["platform"], 124 revision, 125 CHROMIUM_INFO[platform]["chromedriver"], 126 ) 127 128 tmpzip = os.path.join(tempfile.mkdtemp(), "cd-tmp.zip") 129 log("Downloading chromedriver from %s" % download_url) 130 fetch_file(download_url, tmpzip) 131 132 tmppath = tempfile.mkdtemp() 133 unzip(tmpzip, tmppath) 134 135 # Find the chromedriver then copy it to the chromium directory 136 cd_path = None 137 for dirpath, _, filenames in os.walk(tmppath): 138 for filename in filenames: 139 if filename == "chromedriver" or filename == "chromedriver.exe": 140 cd_path = os.path.join(dirpath, filename) 141 break 142 if cd_path is not None: 143 break 144 if cd_path is None: 145 raise Exception("Could not find chromedriver binary in %s" % tmppath) 146 log("Copying chromedriver from: %s to: %s" % (cd_path, chromium_dir)) 147 shutil.copy(cd_path, chromium_dir) 148 149 150def build_chromium_archive(platform, revision=None): 151 """ 152 Download and store a chromium build for a given platform. 153 154 Retrieves either the latest version, or uses a pre-defined version if 155 the `--revision` option is given a revision. 156 """ 157 upload_dir = os.environ.get("UPLOAD_DIR") 158 if upload_dir: 159 # Create the upload directory if it doesn't exist. 160 try: 161 log("Creating upload directory in %s..." % os.path.abspath(upload_dir)) 162 os.makedirs(upload_dir) 163 except OSError as e: 164 if e.errno != errno.EEXIST: 165 raise 166 167 # Make a temporary location for the file 168 tmppath = tempfile.mkdtemp() 169 tmpzip = os.path.join(tmppath, "tmp-chromium.zip") 170 171 revision = fetch_chromium_build(platform, revision, tmpzip) 172 173 # Unpack archive in `tmpzip` to store the revision number and 174 # the chromedriver 175 unzip(tmpzip, tmppath) 176 177 dirs = [ 178 d 179 for d in os.listdir(tmppath) 180 if os.path.isdir(os.path.join(tmppath, d)) and d.startswith("chrome-") 181 ] 182 183 if len(dirs) > 1: 184 raise Exception( 185 "Too many directories starting with `chrome-` after extracting." 186 ) 187 elif len(dirs) == 0: 188 raise Exception( 189 "Could not find any directories after extraction of chromium zip." 190 ) 191 192 chromium_dir = os.path.join(tmppath, dirs[0]) 193 revision_file = os.path.join(chromium_dir, ".REVISION") 194 with open(revision_file, "w+") as f: 195 f.write(str(revision)) 196 197 # Get and store the chromedriver 198 fetch_chromedriver(platform, revision, chromium_dir) 199 200 tar_file = CHROMIUM_INFO[platform]["result"] 201 tar_command = ["tar", "cjf", tar_file, "-C", tmppath, dirs[0]] 202 log("Added revision to %s file." % revision_file) 203 204 log("Tarring with the command: %s" % str(tar_command)) 205 subprocess.check_call(tar_command) 206 207 upload_dir = os.environ.get("UPLOAD_DIR") 208 if upload_dir: 209 # Move the tarball to the output directory for upload. 210 log("Moving %s to the upload directory..." % tar_file) 211 shutil.copy(tar_file, os.path.join(upload_dir, tar_file)) 212 213 shutil.rmtree(tmppath) 214 215 216def parse_args(): 217 """Read command line arguments and return options.""" 218 parser = argparse.ArgumentParser() 219 parser.add_argument( 220 "--platform", help="Platform version of chromium to build.", required=True 221 ) 222 parser.add_argument( 223 "--revision", 224 help="Revision of chromium to build to get. " 225 "(Defaults to the newest chromium build).", 226 default=None, 227 ) 228 229 return parser.parse_args() 230 231 232if __name__ == "__main__": 233 args = vars(parse_args()) 234 build_chromium_archive(**args) 235