1# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- 2# vim: set filetype=python: 3# This Source Code Form is subject to the terms of the Mozilla Public 4# License, v. 2.0. If a copy of the MPL was not distributed with this 5# file, You can obtain one at http://mozilla.org/MPL/2.0/. 6 7option( 8 env="MOZ_FETCHES_DIR", 9 nargs=1, 10 when="MOZ_AUTOMATION", 11 help="Directory containing fetched artifacts", 12) 13 14 15@depends("MOZ_FETCHES_DIR", when="MOZ_AUTOMATION") 16def moz_fetches_dir(value): 17 if value: 18 return value[0] 19 20 21@depends(vcs_checkout_type, milestone.is_nightly, "MOZ_AUTOMATION") 22def bootstrap_default(vcs_checkout_type, is_nightly, automation): 23 if automation: 24 return False 25 # We only enable if building off a VCS checkout of central. 26 if is_nightly and vcs_checkout_type: 27 return True 28 29 30option( 31 "--enable-bootstrap", 32 default=bootstrap_default, 33 help="{Automatically bootstrap or update some toolchains|Disable bootstrap or update of toolchains}", 34) 35 36 37@depends(developer_options, "--enable-bootstrap", moz_fetches_dir) 38def bootstrap_search_path_order(developer_options, bootstrap, moz_fetches_dir): 39 if moz_fetches_dir: 40 log.debug("Prioritizing MOZ_FETCHES_DIR in toolchain path.") 41 return "prepend" 42 43 if bootstrap: 44 log.debug( 45 "Prioritizing mozbuild state dir in toolchain paths because " 46 "bootstrap mode is enabled." 47 ) 48 return "prepend" 49 50 if developer_options: 51 log.debug( 52 "Prioritizing mozbuild state dir in toolchain paths because " 53 "you are not building in release mode." 54 ) 55 return "prepend" 56 57 log.debug( 58 "Prioritizing system over mozbuild state dir in " 59 "toolchain paths because you are building in " 60 "release mode." 61 ) 62 return "append" 63 64 65toolchains_base_dir = moz_fetches_dir | mozbuild_state_path 66 67 68@dependable 69@imports("os") 70@imports(_from="os", _import="environ") 71def original_path(): 72 return environ["PATH"].split(os.pathsep) 73 74 75@depends(host, when="--enable-bootstrap") 76@imports("os") 77@imports("traceback") 78@imports(_from="mozbuild.toolchains", _import="toolchain_task_definitions") 79@imports(_from="__builtin__", _import="Exception") 80def bootstrap_toolchain_tasks(host): 81 prefix = { 82 ("x86_64", "GNU", "Linux"): "linux64", 83 ("x86_64", "OSX", "Darwin"): "macosx64", 84 ("aarch64", "OSX", "Darwin"): "macosx64-aarch64", 85 ("x86_64", "WINNT", "WINNT"): "win64", 86 ("aarch64", "WINNT", "WINNT"): "win64-aarch64", 87 }.get((host.cpu, host.os, host.kernel)) 88 try: 89 tasks = toolchain_task_definitions() 90 except Exception as e: 91 message = traceback.format_exc() 92 log.warning(str(e)) 93 log.debug(message) 94 return None 95 # We only want to use toolchains annotated with "local-toolchain". We also limit the 96 # amount of data to what we use, so that trace logs can be more useful. 97 tasks = { 98 k: { 99 "index": t.optimization["index-search"], 100 "artifact": t.attributes["toolchain-artifact"], 101 } 102 for k, t in tasks.items() 103 if t.attributes.get("local-toolchain") and "index-search" in t.optimization 104 } 105 106 return namespace(prefix=prefix, tasks=tasks) 107 108 109@template 110def bootstrap_path(path, **kwargs): 111 when = kwargs.pop("when", None) 112 if kwargs: 113 configure_error("bootstrap_path only takes `when` as a keyword argument") 114 115 @depends( 116 "--enable-bootstrap", 117 toolchains_base_dir, 118 bootstrap_toolchain_tasks, 119 build_environment, 120 dependable(path), 121 when=when, 122 ) 123 @imports("os") 124 @imports("subprocess") 125 @imports("sys") 126 @imports(_from="mozbuild.util", _import="ensureParentDir") 127 @imports(_from="importlib", _import="import_module") 128 @imports(_from="__builtin__", _import="open") 129 @imports(_from="__builtin__", _import="Exception") 130 def bootstrap_path(bootstrap, toolchains_base_dir, tasks, build_env, path): 131 if not path: 132 return 133 path_parts = path.split("/") 134 135 def try_bootstrap(exists): 136 if not tasks: 137 return False 138 prefixes = [""] 139 if tasks.prefix: 140 prefixes.insert(0, "{}-".format(tasks.prefix)) 141 for prefix in prefixes: 142 label = "toolchain-{}{}".format(prefix, path_parts[0]) 143 task = tasks.tasks.get(label) 144 if task: 145 break 146 log.debug("Trying to bootstrap %s", label) 147 if not task: 148 return False 149 task_index = task["index"] 150 log.debug("Resolved %s to %s", label, task_index[0]) 151 task_index = task_index[0].split(".")[-1] 152 artifact = task["artifact"] 153 # `mach artifact toolchain` doesn't support authentication for 154 # private artifacts. 155 if not artifact.startswith("public/"): 156 log.debug("Cannot bootstrap %s: not a public artifact", label) 157 return False 158 index_file = os.path.join(toolchains_base_dir, "indices", path_parts[0]) 159 try: 160 with open(index_file) as fh: 161 index = fh.read().strip() 162 except Exception: 163 index = None 164 if index == task_index and exists: 165 log.debug("%s is up-to-date", label) 166 return True 167 # Manually import with import_module so that we can gracefully disable bootstrap 168 # when e.g. building from a js standalone tarball, that doesn't contain the 169 # taskgraph code. In those cases, `mach artifact toolchain --from-build` would 170 # also fail. 171 try: 172 IndexSearch = import_module( 173 "gecko_taskgraph.optimize.strategies" 174 ).IndexSearch 175 except Exception: 176 log.debug("Cannot bootstrap %s: missing taskgraph module", label) 177 return False 178 task_id = IndexSearch().should_replace_task(task, {}, None, task["index"]) 179 if task_id: 180 # If we found the task in the index, use the `mach artifact toolchain` 181 # fast path. 182 flags = ["--from-task", f"{task_id}:{artifact}"] 183 else: 184 # Otherwise, use the slower path, which will print a better error than 185 # we would be able to. 186 flags = ["--from-build", label] 187 188 log.info( 189 "%s bootstrapped toolchain in %s", 190 "Updating" if exists else "Installing", 191 os.path.join(toolchains_base_dir, path_parts[0]), 192 ) 193 os.makedirs(toolchains_base_dir, exist_ok=True) 194 subprocess.run( 195 [ 196 sys.executable, 197 os.path.join(build_env.topsrcdir, "mach"), 198 "--log-no-times", 199 "artifact", 200 "toolchain", 201 ] 202 + flags, 203 cwd=toolchains_base_dir, 204 check=True, 205 ) 206 ensureParentDir(index_file) 207 with open(index_file, "w") as fh: 208 fh.write(task_index) 209 return True 210 211 path = os.path.join(toolchains_base_dir, *path_parts) 212 if bootstrap: 213 try: 214 if not try_bootstrap(os.path.exists(path)): 215 # If there aren't toolchain artifacts to use for this build, 216 # don't return a path. 217 return None 218 except Exception as e: 219 log.error("%s", e) 220 die("If you can't fix the above, retry with --disable-bootstrap.") 221 # We re-test whether the path exists because it may have been created by 222 # try_bootstrap. Automation will not have gone through the bootstrap 223 # process, but we want to return the path if it exists. 224 if os.path.exists(path): 225 return path 226 227 return bootstrap_path 228 229 230@template 231def bootstrap_search_path(path, paths=original_path, **kwargs): 232 @depends( 233 bootstrap_path(path, **kwargs), 234 bootstrap_search_path_order, 235 paths, 236 original_path, 237 ) 238 def bootstrap_search_path(path, order, paths, original_path): 239 if paths is None: 240 paths = original_path 241 if not path: 242 return paths 243 if order == "prepend": 244 return [path] + paths 245 return paths + [path] 246 247 return bootstrap_search_path 248