1#!/usr/bin/python3 -u
2# -*- coding: utf-8 -*-
3
4# This Source Code Form is subject to the terms of the Mozilla Public
5# License, v. 2.0. If a copy of the MPL was not distributed with this
6# file, You can obtain one at http://mozilla.org/MPL/2.0/.
7
8"""
9This script downloads the latest chromium build (or a manually
10defined version) for a given platform. It then uploads the build,
11with the revision of the build stored in a REVISION file.
12"""
13
14from __future__ import absolute_import, print_function
15
16import argparse
17import errno
18import os
19import shutil
20import subprocess
21import requests
22import tempfile
23
24
25LAST_CHANGE_URL = (
26    # formatted with platform
27    "https://www.googleapis.com/download/storage/v1/b/"
28    "chromium-browser-snapshots/o/{}%2FLAST_CHANGE?alt=media"
29)
30
31CHROMIUM_BASE_URL = (
32    # formatted with (platform/revision/archive)
33    "https://www.googleapis.com/download/storage/v1/b/"
34    "chromium-browser-snapshots/o/{}%2F{}%2F{}?alt=media"
35)
36
37
38CHROMIUM_INFO = {
39    "linux": {
40        "platform": "Linux_x64",
41        "chromium": "chrome-linux.zip",
42        "result": "chromium-linux.tar.bz2",
43        "chromedriver": "chromedriver_linux64.zip",
44    },
45    "win32": {
46        "platform": "Win",
47        "chromium": "chrome-win.zip",
48        "result": "chromium-win32.tar.bz2",
49        "chromedriver": "chromedriver_win32.zip",
50    },
51    "win64": {
52        "platform": "Win",
53        "chromium": "chrome-win.zip",
54        "result": "chromium-win64.tar.bz2",
55        "chromedriver": "chromedriver_win32.zip",
56    },
57    "mac": {
58        "platform": "Mac",
59        "chromium": "chrome-mac.zip",
60        "result": "chromium-mac.tar.bz2",
61        "chromedriver": "chromedriver_mac64.zip",
62    },
63}
64
65
66def log(msg):
67    print("build-chromium: %s" % msg)
68
69
70def fetch_file(url, filepath):
71    """Download a file from the given url to a given file."""
72    size = 4096
73    r = requests.get(url, stream=True)
74    r.raise_for_status()
75
76    with open(filepath, "wb") as fd:
77        for chunk in r.iter_content(size):
78            fd.write(chunk)
79
80
81def unzip(zippath, target):
82    """Unzips an archive to the target location."""
83    log("Unpacking archive at: %s to: %s" % (zippath, target))
84    unzip_command = ["unzip", "-q", "-o", zippath, "-d", target]
85    subprocess.check_call(unzip_command)
86
87
88def fetch_chromium_revision(platform):
89    """Get the revision of the latest chromium build. """
90    chromium_platform = CHROMIUM_INFO[platform]["platform"]
91    revision_url = LAST_CHANGE_URL.format(chromium_platform)
92
93    log("Getting revision number for latest %s chromium build..." % chromium_platform)
94
95    # Expecting a file with a single number indicating the latest
96    # chromium build with a chromedriver that we can download
97    r = requests.get(revision_url, timeout=30)
98    r.raise_for_status()
99
100    chromium_revision = r.content.decode("utf-8")
101    return chromium_revision.strip()
102
103
104def fetch_chromium_build(platform, revision, zippath):
105    """Download a chromium build for a given revision, or the latest. """
106    if not revision:
107        revision = fetch_chromium_revision(platform)
108
109    download_platform = CHROMIUM_INFO[platform]["platform"]
110    download_url = CHROMIUM_BASE_URL.format(
111        download_platform, revision, CHROMIUM_INFO[platform]["chromium"]
112    )
113
114    log("Downloading %s chromium build revision %s..." % (download_platform, revision))
115
116    fetch_file(download_url, zippath)
117    return revision
118
119
120def fetch_chromedriver(platform, revision, chromium_dir):
121    """Get the chromedriver for the given revision and repackage it."""
122    download_url = CHROMIUM_BASE_URL.format(
123        CHROMIUM_INFO[platform]["platform"],
124        revision,
125        CHROMIUM_INFO[platform]["chromedriver"],
126    )
127
128    tmpzip = os.path.join(tempfile.mkdtemp(), "cd-tmp.zip")
129    log("Downloading chromedriver from %s" % download_url)
130    fetch_file(download_url, tmpzip)
131
132    tmppath = tempfile.mkdtemp()
133    unzip(tmpzip, tmppath)
134
135    # Find the chromedriver then copy it to the chromium directory
136    cd_path = None
137    for dirpath, _, filenames in os.walk(tmppath):
138        for filename in filenames:
139            if filename == "chromedriver" or filename == "chromedriver.exe":
140                cd_path = os.path.join(dirpath, filename)
141                break
142        if cd_path is not None:
143            break
144    if cd_path is None:
145        raise Exception("Could not find chromedriver binary in %s" % tmppath)
146    log("Copying chromedriver from: %s to: %s" % (cd_path, chromium_dir))
147    shutil.copy(cd_path, chromium_dir)
148
149
150def build_chromium_archive(platform, revision=None):
151    """
152    Download and store a chromium build for a given platform.
153
154    Retrieves either the latest version, or uses a pre-defined version if
155    the `--revision` option is given a revision.
156    """
157    upload_dir = os.environ.get("UPLOAD_DIR")
158    if upload_dir:
159        # Create the upload directory if it doesn't exist.
160        try:
161            log("Creating upload directory in %s..." % os.path.abspath(upload_dir))
162            os.makedirs(upload_dir)
163        except OSError as e:
164            if e.errno != errno.EEXIST:
165                raise
166
167    # Make a temporary location for the file
168    tmppath = tempfile.mkdtemp()
169    tmpzip = os.path.join(tmppath, "tmp-chromium.zip")
170
171    revision = fetch_chromium_build(platform, revision, tmpzip)
172
173    # Unpack archive in `tmpzip` to store the revision number and
174    # the chromedriver
175    unzip(tmpzip, tmppath)
176
177    dirs = [
178        d
179        for d in os.listdir(tmppath)
180        if os.path.isdir(os.path.join(tmppath, d)) and d.startswith("chrome-")
181    ]
182
183    if len(dirs) > 1:
184        raise Exception(
185            "Too many directories starting with `chrome-` after extracting."
186        )
187    elif len(dirs) == 0:
188        raise Exception(
189            "Could not find any directories after extraction of chromium zip."
190        )
191
192    chromium_dir = os.path.join(tmppath, dirs[0])
193    revision_file = os.path.join(chromium_dir, ".REVISION")
194    with open(revision_file, "w+") as f:
195        f.write(str(revision))
196
197    # Get and store the chromedriver
198    fetch_chromedriver(platform, revision, chromium_dir)
199
200    tar_file = CHROMIUM_INFO[platform]["result"]
201    tar_command = ["tar", "cjf", tar_file, "-C", tmppath, dirs[0]]
202    log("Added revision to %s file." % revision_file)
203
204    log("Tarring with the command: %s" % str(tar_command))
205    subprocess.check_call(tar_command)
206
207    upload_dir = os.environ.get("UPLOAD_DIR")
208    if upload_dir:
209        # Move the tarball to the output directory for upload.
210        log("Moving %s to the upload directory..." % tar_file)
211        shutil.copy(tar_file, os.path.join(upload_dir, tar_file))
212
213    shutil.rmtree(tmppath)
214
215
216def parse_args():
217    """Read command line arguments and return options."""
218    parser = argparse.ArgumentParser()
219    parser.add_argument(
220        "--platform", help="Platform version of chromium to build.", required=True
221    )
222    parser.add_argument(
223        "--revision",
224        help="Revision of chromium to build to get. "
225        "(Defaults to the newest chromium build).",
226        default=None,
227    )
228
229    return parser.parse_args()
230
231
232if __name__ == "__main__":
233    args = vars(parse_args())
234    build_chromium_archive(**args)
235