1# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
2# vim: set filetype=python:
3# This Source Code Form is subject to the terms of the Mozilla Public
4# License, v. 2.0. If a copy of the MPL was not distributed with this
5# file, You can obtain one at http://mozilla.org/MPL/2.0/.
6
7option(
8    env="MOZ_FETCHES_DIR",
9    nargs=1,
10    when="MOZ_AUTOMATION",
11    help="Directory containing fetched artifacts",
12)
13
14
15@depends("MOZ_FETCHES_DIR", when="MOZ_AUTOMATION")
16def moz_fetches_dir(value):
17    if value:
18        return value[0]
19
20
21@depends(vcs_checkout_type, milestone.is_nightly, "MOZ_AUTOMATION")
22def bootstrap_default(vcs_checkout_type, is_nightly, automation):
23    if automation:
24        return False
25    # We only enable if building off a VCS checkout of central.
26    if is_nightly and vcs_checkout_type:
27        return True
28
29
30option(
31    "--enable-bootstrap",
32    default=bootstrap_default,
33    help="{Automatically bootstrap or update some toolchains|Disable bootstrap or update of toolchains}",
34)
35
36
37@depends(developer_options, "--enable-bootstrap", moz_fetches_dir)
38def bootstrap_search_path_order(developer_options, bootstrap, moz_fetches_dir):
39    if moz_fetches_dir:
40        log.debug("Prioritizing MOZ_FETCHES_DIR in toolchain path.")
41        return "prepend"
42
43    if bootstrap:
44        log.debug(
45            "Prioritizing mozbuild state dir in toolchain paths because "
46            "bootstrap mode is enabled."
47        )
48        return "prepend"
49
50    if developer_options:
51        log.debug(
52            "Prioritizing mozbuild state dir in toolchain paths because "
53            "you are not building in release mode."
54        )
55        return "prepend"
56
57    log.debug(
58        "Prioritizing system over mozbuild state dir in "
59        "toolchain paths because you are building in "
60        "release mode."
61    )
62    return "append"
63
64
65toolchains_base_dir = moz_fetches_dir | mozbuild_state_path
66
67
68@dependable
69@imports("os")
70@imports(_from="os", _import="environ")
71def original_path():
72    return environ["PATH"].split(os.pathsep)
73
74
75@depends(host, when="--enable-bootstrap")
76@imports("os")
77@imports("traceback")
78@imports(_from="mozbuild.toolchains", _import="toolchain_task_definitions")
79@imports(_from="__builtin__", _import="Exception")
80def bootstrap_toolchain_tasks(host):
81    prefix = {
82        ("x86_64", "GNU", "Linux"): "linux64",
83        ("x86_64", "OSX", "Darwin"): "macosx64",
84        ("aarch64", "OSX", "Darwin"): "macosx64-aarch64",
85        ("x86_64", "WINNT", "WINNT"): "win64",
86        ("aarch64", "WINNT", "WINNT"): "win64-aarch64",
87    }.get((host.cpu, host.os, host.kernel))
88    try:
89        tasks = toolchain_task_definitions()
90    except Exception as e:
91        message = traceback.format_exc()
92        log.warning(str(e))
93        log.debug(message)
94        return None
95    # We only want to use toolchains annotated with "local-toolchain". We also limit the
96    # amount of data to what we use, so that trace logs can be more useful.
97    tasks = {
98        k: {
99            "index": t.optimization["index-search"],
100            "artifact": t.attributes["toolchain-artifact"],
101        }
102        for k, t in tasks.items()
103        if t.attributes.get("local-toolchain") and "index-search" in t.optimization
104    }
105
106    return namespace(prefix=prefix, tasks=tasks)
107
108
109@template
110def bootstrap_path(path, **kwargs):
111    when = kwargs.pop("when", None)
112    if kwargs:
113        configure_error("bootstrap_path only takes `when` as a keyword argument")
114
115    @depends(
116        "--enable-bootstrap",
117        toolchains_base_dir,
118        bootstrap_toolchain_tasks,
119        build_environment,
120        dependable(path),
121        when=when,
122    )
123    @imports("os")
124    @imports("subprocess")
125    @imports("sys")
126    @imports(_from="mozbuild.util", _import="ensureParentDir")
127    @imports(_from="importlib", _import="import_module")
128    @imports(_from="__builtin__", _import="open")
129    @imports(_from="__builtin__", _import="Exception")
130    def bootstrap_path(bootstrap, toolchains_base_dir, tasks, build_env, path):
131        if not path:
132            return
133        path_parts = path.split("/")
134
135        def try_bootstrap(exists):
136            if not tasks:
137                return False
138            prefixes = [""]
139            if tasks.prefix:
140                prefixes.insert(0, "{}-".format(tasks.prefix))
141            for prefix in prefixes:
142                label = "toolchain-{}{}".format(prefix, path_parts[0])
143                task = tasks.tasks.get(label)
144                if task:
145                    break
146            log.debug("Trying to bootstrap %s", label)
147            if not task:
148                return False
149            task_index = task["index"]
150            log.debug("Resolved %s to %s", label, task_index[0])
151            task_index = task_index[0].split(".")[-1]
152            artifact = task["artifact"]
153            # `mach artifact toolchain` doesn't support authentication for
154            # private artifacts.
155            if not artifact.startswith("public/"):
156                log.debug("Cannot bootstrap %s: not a public artifact", label)
157                return False
158            index_file = os.path.join(toolchains_base_dir, "indices", path_parts[0])
159            try:
160                with open(index_file) as fh:
161                    index = fh.read().strip()
162            except Exception:
163                index = None
164            if index == task_index and exists:
165                log.debug("%s is up-to-date", label)
166                return True
167            # Manually import with import_module so that we can gracefully disable bootstrap
168            # when e.g. building from a js standalone tarball, that doesn't contain the
169            # taskgraph code. In those cases, `mach artifact toolchain --from-build` would
170            # also fail.
171            try:
172                IndexSearch = import_module(
173                    "gecko_taskgraph.optimize.strategies"
174                ).IndexSearch
175            except Exception:
176                log.debug("Cannot bootstrap %s: missing taskgraph module", label)
177                return False
178            task_id = IndexSearch().should_replace_task(task, {}, None, task["index"])
179            if task_id:
180                # If we found the task in the index, use the `mach artifact toolchain`
181                # fast path.
182                flags = ["--from-task", f"{task_id}:{artifact}"]
183            else:
184                # Otherwise, use the slower path, which will print a better error than
185                # we would be able to.
186                flags = ["--from-build", label]
187
188            log.info(
189                "%s bootstrapped toolchain in %s",
190                "Updating" if exists else "Installing",
191                os.path.join(toolchains_base_dir, path_parts[0]),
192            )
193            os.makedirs(toolchains_base_dir, exist_ok=True)
194            subprocess.run(
195                [
196                    sys.executable,
197                    os.path.join(build_env.topsrcdir, "mach"),
198                    "--log-no-times",
199                    "artifact",
200                    "toolchain",
201                ]
202                + flags,
203                cwd=toolchains_base_dir,
204                check=True,
205            )
206            ensureParentDir(index_file)
207            with open(index_file, "w") as fh:
208                fh.write(task_index)
209            return True
210
211        path = os.path.join(toolchains_base_dir, *path_parts)
212        if bootstrap:
213            try:
214                if not try_bootstrap(os.path.exists(path)):
215                    # If there aren't toolchain artifacts to use for this build,
216                    # don't return a path.
217                    return None
218            except Exception as e:
219                log.error("%s", e)
220                die("If you can't fix the above, retry with --disable-bootstrap.")
221        # We re-test whether the path exists because it may have been created by
222        # try_bootstrap. Automation will not have gone through the bootstrap
223        # process, but we want to return the path if it exists.
224        if os.path.exists(path):
225            return path
226
227    return bootstrap_path
228
229
230@template
231def bootstrap_search_path(path, paths=original_path, **kwargs):
232    @depends(
233        bootstrap_path(path, **kwargs),
234        bootstrap_search_path_order,
235        paths,
236        original_path,
237    )
238    def bootstrap_search_path(path, order, paths, original_path):
239        if paths is None:
240            paths = original_path
241        if not path:
242            return paths
243        if order == "prepend":
244            return [path] + paths
245        return paths + [path]
246
247    return bootstrap_search_path
248