1#!/usr/bin/env python3
2#
3# This Source Code Form is subject to the terms of the Mozilla Public
4# License, v. 2.0. If a copy of the MPL was not distributed with this
5# file, You can obtain one at http://mozilla.org/MPL/2.0/.
6#
7# This script uploads a symbol archive file from a path or URL passed on the commandline
8# to the symbol server at https://symbols.mozilla.org/ .
9#
10# Using this script requires you to have generated an authentication
11# token in the symbol server web interface. You must store the token in a Taskcluster
12# secret as the JSON blob `{"token": "<token>"}` and set the `SYMBOL_SECRET`
13# environment variable to the name of the Taskcluster secret. Alternately,
14# you can put the token in a file and set `SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE`
15# environment variable to the path to the file.
16
17from __future__ import absolute_import, print_function, unicode_literals
18
19import argparse
20import logging
21import os
22import redo
23import requests
24import shutil
25import sys
26from mozbuild.base import MozbuildObject
27
28log = logging.getLogger("upload-symbols")
29log.setLevel(logging.INFO)
30
31DEFAULT_URL = "https://symbols.mozilla.org/upload/"
32MAX_RETRIES = 7
33
34
35def print_error(r):
36    if r.status_code < 400:
37        log.error("Error: bad auth token? ({0}: {1})".format(r.status_code, r.reason))
38    else:
39        log.error("Error: got HTTP response {0}: {1}".format(r.status_code, r.reason))
40
41    log.error(
42        "Response body:\n{sep}\n{body}\n{sep}\n".format(sep="=" * 20, body=r.text)
43    )
44
45
46def get_taskcluster_secret(secret_name):
47    secrets_url = "http://taskcluster/secrets/v1/secret/{}".format(secret_name)
48    log.info(
49        'Using symbol upload token from the secrets service: "{}"'.format(secrets_url)
50    )
51    res = requests.get(secrets_url)
52    res.raise_for_status()
53    secret = res.json()
54    auth_token = secret["secret"]["token"]
55
56    return auth_token
57
58
59def main():
60    config = MozbuildObject.from_environment()
61    config.activate_virtualenv()
62
63    logging.basicConfig()
64    parser = argparse.ArgumentParser(
65        description="Upload symbols in ZIP using token from Taskcluster secrets service."
66    )
67    parser.add_argument(
68        "archive", help="Symbols archive file - URL or path to local file"
69    )
70    parser.add_argument(
71        "--ignore-missing", help="No error on missing files", action="store_true"
72    )
73    args = parser.parse_args()
74
75    def check_file_exists(url):
76        for i, _ in enumerate(redo.retrier(attempts=MAX_RETRIES), start=1):
77            try:
78                resp = requests.head(url, allow_redirects=True)
79                return resp.status_code == requests.codes.ok
80            except requests.exceptions.RequestException as e:
81                log.error("Error: {0}".format(e))
82            log.info("Retrying...")
83        return False
84
85    zip_path = args.archive
86
87    if args.archive.endswith(".tar.zst"):
88        from mozpack.files import File
89        from mozpack.mozjar import JarWriter
90        import gzip
91        import tarfile
92        import tempfile
93
94        config._ensure_zstd()
95        import zstandard
96
97        def prepare_zip_from(archive, tmpdir):
98            if archive.startswith("http"):
99                resp = requests.get(archive, allow_redirects=True, stream=True)
100                resp.raise_for_status()
101                reader = resp.raw
102                # Work around taskcluster generic-worker possibly gzipping the tar.zst.
103                if resp.headers.get("Content-Encoding") == "gzip":
104                    reader = gzip.GzipFile(fileobj=reader)
105            else:
106                reader = open(archive, "rb")
107
108            ctx = zstandard.ZstdDecompressor()
109            uncompressed = ctx.stream_reader(reader)
110            with tarfile.open(
111                mode="r|", fileobj=uncompressed, bufsize=1024 * 1024
112            ) as tar:
113                while True:
114                    info = tar.next()
115                    if info is None:
116                        break
117                    log.info(info.name)
118                    data = tar.extractfile(info)
119                    path = os.path.join(tmpdir, info.name.lstrip("/"))
120                    if info.name.endswith(".dbg"):
121                        os.makedirs(os.path.dirname(path), exist_ok=True)
122                        with open(path, "wb") as fh:
123                            with gzip.GzipFile(
124                                fileobj=fh, mode="wb", compresslevel=5
125                            ) as c:
126                                shutil.copyfileobj(data, c)
127                        jar.add(info.name + ".gz", File(path), compress=False)
128                    elif info.name.endswith(".dSYM.tar"):
129                        import bz2
130
131                        os.makedirs(os.path.dirname(path), exist_ok=True)
132                        with open(path, "wb") as fh:
133                            c = bz2.BZ2Compressor()
134                            while True:
135                                buf = data.read(16384)
136                                if not buf:
137                                    break
138                                fh.write(c.compress(buf))
139                            fh.write(c.flush())
140                        jar.add(info.name + ".bz2", File(path), compress=False)
141                    elif info.name.endswith((".pdb", ".exe", ".dll")):
142                        import subprocess
143
144                        makecab = os.environ.get("MAKECAB", "makecab")
145                        os.makedirs(os.path.dirname(path), exist_ok=True)
146                        with open(path, "wb") as fh:
147                            shutil.copyfileobj(data, fh)
148
149                        subprocess.check_call(
150                            [makecab, "-D", "CompressionType=MSZIP", path, path + "_"],
151                            stdout=subprocess.DEVNULL,
152                            stderr=subprocess.STDOUT,
153                        )
154
155                        jar.add(info.name[:-1] + "_", File(path + "_"), compress=False)
156                    else:
157                        jar.add(info.name, data)
158            reader.close()
159
160        tmpdir = tempfile.TemporaryDirectory()
161        zip_path = os.path.join(tmpdir.name, "symbols.zip")
162        log.info(
163            'Preparing symbol archive "{0}" from "{1}"'.format(zip_path, args.archive)
164        )
165        is_existing = False
166        try:
167            for i, _ in enumerate(redo.retrier(attempts=MAX_RETRIES), start=1):
168                with JarWriter(zip_path, compress_level=5) as jar:
169                    try:
170                        prepare_zip_from(args.archive, tmpdir.name)
171                        is_existing = True
172                        break
173                    except requests.exceptions.RequestException as e:
174                        log.error("Error: {0}".format(e))
175                    log.info("Retrying...")
176        except Exception:
177            os.remove(zip_path)
178            raise
179
180    elif args.archive.startswith("http"):
181        is_existing = check_file_exists(args.archive)
182    else:
183        is_existing = os.path.isfile(args.archive)
184
185    if not is_existing:
186        if args.ignore_missing:
187            log.info('Archive file "{0}" does not exist!'.format(args.archive))
188            return 0
189        else:
190            log.error('Error: archive file "{0}" does not exist!'.format(args.archive))
191            return 1
192
193    secret_name = os.environ.get("SYMBOL_SECRET")
194    if secret_name is not None:
195        auth_token = get_taskcluster_secret(secret_name)
196    elif "SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE" in os.environ:
197        token_file = os.environ["SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE"]
198
199        if not os.path.isfile(token_file):
200            log.error(
201                'SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE "{0}" does not exist!'.format(
202                    token_file
203                )
204            )
205            return 1
206        auth_token = open(token_file, "r").read().strip()
207    else:
208        log.error(
209            "You must set the SYMBOL_SECRET or SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE "
210            "environment variables!"
211        )
212        return 1
213
214    # Allow overwriting of the upload url with an environmental variable
215    if "SOCORRO_SYMBOL_UPLOAD_URL" in os.environ:
216        url = os.environ["SOCORRO_SYMBOL_UPLOAD_URL"]
217    else:
218        url = DEFAULT_URL
219
220    log.info('Uploading symbol file "{0}" to "{1}"'.format(zip_path, url))
221
222    for i, _ in enumerate(redo.retrier(attempts=MAX_RETRIES), start=1):
223        log.info("Attempt %d of %d..." % (i, MAX_RETRIES))
224        try:
225            if zip_path.startswith("http"):
226                zip_arg = {"data": {"url": zip_path}}
227            else:
228                zip_arg = {"files": {"symbols.zip": open(zip_path, "rb")}}
229            r = requests.post(
230                url,
231                headers={"Auth-Token": auth_token},
232                allow_redirects=False,
233                # Allow a longer read timeout because uploading by URL means the server
234                # has to fetch the entire zip file, which can take a while. The load balancer
235                # in front of symbols.mozilla.org has a 300 second timeout, so we'll use that.
236                timeout=(300, 300),
237                **zip_arg
238            )
239            # 429 or any 5XX is likely to be a transient failure.
240            # Break out for success or other error codes.
241            if r.ok or (r.status_code < 500 and r.status_code != 429):
242                break
243            print_error(r)
244        except requests.exceptions.RequestException as e:
245            log.error("Error: {0}".format(e))
246        log.info("Retrying...")
247    else:
248        log.warn("Maximum retries hit, giving up!")
249        return 1
250
251    if r.status_code >= 200 and r.status_code < 300:
252        log.info("Uploaded successfully!")
253        return 0
254
255    print_error(r)
256    return 1
257
258
259if __name__ == "__main__":
260    sys.exit(main())
261