1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5from __future__ import absolute_import, print_function, unicode_literals 6 7import io 8import json 9import logging 10import mozpack.path as mozpath 11import multiprocessing 12import os 13import six 14import subprocess 15import sys 16import errno 17 18from mach.mixin.process import ProcessExecutionMixin 19from mozboot.mozconfig import MozconfigFindException 20from mozfile import which 21from mozversioncontrol import ( 22 get_repository_from_build_config, 23 get_repository_object, 24 GitRepository, 25 HgRepository, 26 InvalidRepoPath, 27 MissingConfigureInfo, 28 MissingVCSTool, 29) 30 31from .backend.configenvironment import ( 32 ConfigEnvironment, 33 ConfigStatusFailure, 34) 35from .configure import ConfigureSandbox 36from .controller.clobber import Clobberer 37from .mozconfig import ( 38 MozconfigLoadException, 39 MozconfigLoader, 40) 41from .util import ( 42 memoize, 43 memoized_property, 44) 45 46try: 47 import psutil 48except Exception: 49 psutil = None 50 51 52def ancestors(path): 53 """Emit the parent directories of a path.""" 54 while path: 55 yield path 56 newpath = os.path.dirname(path) 57 if newpath == path: 58 break 59 path = newpath 60 61 62def samepath(path1, path2): 63 # Under Python 3 (but NOT Python 2), MozillaBuild exposes the 64 # os.path.samefile function despite it not working, so only use it if we're 65 # not running under Windows. 66 if hasattr(os.path, "samefile") and os.name != "nt": 67 return os.path.samefile(path1, path2) 68 return os.path.normcase(os.path.realpath(path1)) == os.path.normcase( 69 os.path.realpath(path2) 70 ) 71 72 73class BadEnvironmentException(Exception): 74 """Base class for errors raised when the build environment is not sane.""" 75 76 77class BuildEnvironmentNotFoundException(BadEnvironmentException, AttributeError): 78 """Raised when we could not find a build environment.""" 79 80 81class ObjdirMismatchException(BadEnvironmentException): 82 """Raised when the current dir is an objdir and doesn't match the mozconfig.""" 83 84 def __init__(self, objdir1, objdir2): 85 self.objdir1 = objdir1 86 self.objdir2 = objdir2 87 88 def __str__(self): 89 return "Objdir mismatch: %s != %s" % (self.objdir1, self.objdir2) 90 91 92class BinaryNotFoundException(Exception): 93 """Raised when the binary is not found in the expected location.""" 94 95 def __init__(self, path): 96 self.path = path 97 98 def __str__(self): 99 return "Binary expected at {} does not exist.".format(self.path) 100 101 def help(self): 102 return "It looks like your program isn't built. You can run |./mach build| to build it." 103 104 105class MozbuildObject(ProcessExecutionMixin): 106 """Base class providing basic functionality useful to many modules. 107 108 Modules in this package typically require common functionality such as 109 accessing the current config, getting the location of the source directory, 110 running processes, etc. This classes provides that functionality. Other 111 modules can inherit from this class to obtain this functionality easily. 112 """ 113 114 def __init__( 115 self, 116 topsrcdir, 117 settings, 118 log_manager, 119 topobjdir=None, 120 mozconfig=MozconfigLoader.AUTODETECT, 121 virtualenv_name=None, 122 ): 123 """Create a new Mozbuild object instance. 124 125 Instances are bound to a source directory, a ConfigSettings instance, 126 and a LogManager instance. The topobjdir may be passed in as well. If 127 it isn't, it will be calculated from the active mozconfig. 128 """ 129 self.topsrcdir = mozpath.normsep(topsrcdir) 130 self.settings = settings 131 132 self.populate_logger() 133 self.log_manager = log_manager 134 135 self._make = None 136 self._topobjdir = mozpath.normsep(topobjdir) if topobjdir else topobjdir 137 self._mozconfig = mozconfig 138 self._config_environment = None 139 self._virtualenv_name = virtualenv_name or "common" 140 self._virtualenv_manager = None 141 142 @classmethod 143 def from_environment(cls, cwd=None, detect_virtualenv_mozinfo=True, **kwargs): 144 """Create a MozbuildObject by detecting the proper one from the env. 145 146 This examines environment state like the current working directory and 147 creates a MozbuildObject from the found source directory, mozconfig, etc. 148 149 The role of this function is to identify a topsrcdir, topobjdir, and 150 mozconfig file. 151 152 If the current working directory is inside a known objdir, we always 153 use the topsrcdir and mozconfig associated with that objdir. 154 155 If the current working directory is inside a known srcdir, we use that 156 topsrcdir and look for mozconfigs using the default mechanism, which 157 looks inside environment variables. 158 159 If the current Python interpreter is running from a virtualenv inside 160 an objdir, we use that as our objdir. 161 162 If we're not inside a srcdir or objdir, an exception is raised. 163 164 detect_virtualenv_mozinfo determines whether we should look for a 165 mozinfo.json file relative to the virtualenv directory. This was 166 added to facilitate testing. Callers likely shouldn't change the 167 default. 168 """ 169 170 cwd = os.path.realpath(cwd or os.getcwd()) 171 topsrcdir = None 172 topobjdir = None 173 mozconfig = MozconfigLoader.AUTODETECT 174 175 def load_mozinfo(path): 176 info = json.load(io.open(path, "rt", encoding="utf-8")) 177 topsrcdir = info.get("topsrcdir") 178 topobjdir = os.path.dirname(path) 179 mozconfig = info.get("mozconfig") 180 return topsrcdir, topobjdir, mozconfig 181 182 for dir_path in ancestors(cwd): 183 # If we find a mozinfo.json, we are in the objdir. 184 mozinfo_path = os.path.join(dir_path, "mozinfo.json") 185 if os.path.isfile(mozinfo_path): 186 topsrcdir, topobjdir, mozconfig = load_mozinfo(mozinfo_path) 187 break 188 189 # We choose an arbitrary file as an indicator that this is a 190 # srcdir. We go with ourself because why not! 191 our_path = os.path.join( 192 dir_path, "python", "mozbuild", "mozbuild", "base.py" 193 ) 194 if os.path.isfile(our_path): 195 topsrcdir = dir_path 196 break 197 198 # See if we're running from a Python virtualenv that's inside an objdir. 199 mozinfo_path = os.path.join(os.path.dirname(sys.prefix), "../mozinfo.json") 200 if detect_virtualenv_mozinfo and os.path.isfile(mozinfo_path): 201 topsrcdir, topobjdir, mozconfig = load_mozinfo(mozinfo_path) 202 203 if not topsrcdir: 204 topsrcdir = os.path.abspath( 205 os.path.join(os.path.dirname(__file__), "..", "..", "..") 206 ) 207 208 topsrcdir = mozpath.normsep(topsrcdir) 209 if topobjdir: 210 topobjdir = mozpath.normsep(os.path.normpath(topobjdir)) 211 212 if topsrcdir == topobjdir: 213 raise BadEnvironmentException( 214 "The object directory appears " 215 "to be the same as your source directory (%s). This build " 216 "configuration is not supported." % topsrcdir 217 ) 218 219 # If we can't resolve topobjdir, oh well. We'll figure out when we need 220 # one. 221 return cls( 222 topsrcdir, None, None, topobjdir=topobjdir, mozconfig=mozconfig, **kwargs 223 ) 224 225 def resolve_mozconfig_topobjdir(self, default=None): 226 topobjdir = self.mozconfig["topobjdir"] or default 227 if not topobjdir: 228 return None 229 230 if "@CONFIG_GUESS@" in topobjdir: 231 topobjdir = topobjdir.replace("@CONFIG_GUESS@", self.resolve_config_guess()) 232 233 if not os.path.isabs(topobjdir): 234 topobjdir = os.path.abspath(os.path.join(self.topsrcdir, topobjdir)) 235 236 return mozpath.normsep(os.path.normpath(topobjdir)) 237 238 def build_out_of_date(self, output, dep_file): 239 if not os.path.isfile(output): 240 print(" Output reference file not found: %s" % output) 241 return True 242 if not os.path.isfile(dep_file): 243 print(" Dependency file not found: %s" % dep_file) 244 return True 245 246 deps = [] 247 with io.open(dep_file, "r", encoding="utf-8", newline="\n") as fh: 248 deps = fh.read().splitlines() 249 250 mtime = os.path.getmtime(output) 251 for f in deps: 252 try: 253 dep_mtime = os.path.getmtime(f) 254 except OSError as e: 255 if e.errno == errno.ENOENT: 256 print(" Input not found: %s" % f) 257 return True 258 raise 259 if dep_mtime > mtime: 260 print(" %s is out of date with respect to %s" % (output, f)) 261 return True 262 return False 263 264 def backend_out_of_date(self, backend_file): 265 if not os.path.isfile(backend_file): 266 return True 267 268 # Check if any of our output files have been removed since 269 # we last built the backend, re-generate the backend if 270 # so. 271 outputs = [] 272 with io.open(backend_file, "r", encoding="utf-8", newline="\n") as fh: 273 outputs = fh.read().splitlines() 274 for output in outputs: 275 if not os.path.isfile(mozpath.join(self.topobjdir, output)): 276 return True 277 278 dep_file = "%s.in" % backend_file 279 return self.build_out_of_date(backend_file, dep_file) 280 281 @property 282 def topobjdir(self): 283 if self._topobjdir is None: 284 self._topobjdir = self.resolve_mozconfig_topobjdir( 285 default="obj-@CONFIG_GUESS@" 286 ) 287 288 return self._topobjdir 289 290 @property 291 def virtualenv_manager(self): 292 from mach.site import CommandSiteManager 293 from mozboot.util import get_state_dir 294 295 if self._virtualenv_manager is None: 296 self._virtualenv_manager = CommandSiteManager.from_environment( 297 self.topsrcdir, 298 lambda: get_state_dir( 299 specific_to_topsrcdir=True, topsrcdir=self.topsrcdir 300 ), 301 self._virtualenv_name, 302 os.path.join(self.topobjdir, "_virtualenvs"), 303 ) 304 305 return self._virtualenv_manager 306 307 @staticmethod 308 @memoize 309 def get_mozconfig_and_target(topsrcdir, path, env_mozconfig): 310 # env_mozconfig is only useful for unittests, which change the value of 311 # the environment variable, which has an impact on autodetection (when 312 # path is MozconfigLoader.AUTODETECT), and memoization wouldn't account 313 # for it without the explicit (unused) argument. 314 out = six.StringIO() 315 env = os.environ 316 if path and path != MozconfigLoader.AUTODETECT: 317 env = dict(env) 318 env["MOZCONFIG"] = path 319 320 # We use python configure to get mozconfig content and the value for 321 # --target (from mozconfig if necessary, guessed otherwise). 322 323 # Modified configure sandbox that replaces '--help' dependencies with 324 # `always`, such that depends functions with a '--help' dependency are 325 # not automatically executed when including files. We don't want all of 326 # those from init.configure to execute, only a subset. 327 class ReducedConfigureSandbox(ConfigureSandbox): 328 def depends_impl(self, *args, **kwargs): 329 args = tuple( 330 a 331 if not isinstance(a, six.string_types) or a != "--help" 332 else self._always.sandboxed 333 for a in args 334 ) 335 return super(ReducedConfigureSandbox, self).depends_impl( 336 *args, **kwargs 337 ) 338 339 # This may be called recursively from configure itself for $reasons, 340 # so avoid logging to the same logger (configure uses "moz.configure") 341 logger = logging.getLogger("moz.configure.reduced") 342 handler = logging.StreamHandler(out) 343 logger.addHandler(handler) 344 # If this were true, logging would still propagate to "moz.configure". 345 logger.propagate = False 346 sandbox = ReducedConfigureSandbox( 347 {}, 348 environ=env, 349 argv=["mach", "--help"], 350 logger=logger, 351 ) 352 base_dir = os.path.join(topsrcdir, "build", "moz.configure") 353 try: 354 sandbox.include_file(os.path.join(base_dir, "init.configure")) 355 # Force mozconfig options injection before getting the target. 356 sandbox._value_for(sandbox["mozconfig_options"]) 357 return ( 358 sandbox._value_for(sandbox["mozconfig"]), 359 sandbox._value_for(sandbox["real_target"]), 360 ) 361 except SystemExit: 362 print(out.getvalue()) 363 raise 364 365 @property 366 def mozconfig_and_target(self): 367 return self.get_mozconfig_and_target( 368 self.topsrcdir, self._mozconfig, os.environ.get("MOZCONFIG") 369 ) 370 371 @property 372 def mozconfig(self): 373 """Returns information about the current mozconfig file. 374 375 This a dict as returned by MozconfigLoader.read_mozconfig() 376 """ 377 return self.mozconfig_and_target[0] 378 379 @property 380 def config_environment(self): 381 """Returns the ConfigEnvironment for the current build configuration. 382 383 This property is only available once configure has executed. 384 385 If configure's output is not available, this will raise. 386 """ 387 if self._config_environment: 388 return self._config_environment 389 390 config_status = os.path.join(self.topobjdir, "config.status") 391 392 if not os.path.exists(config_status) or not os.path.getsize(config_status): 393 raise BuildEnvironmentNotFoundException( 394 "config.status not available. Run configure." 395 ) 396 397 try: 398 self._config_environment = ConfigEnvironment.from_config_status( 399 config_status 400 ) 401 except ConfigStatusFailure as e: 402 six.raise_from( 403 BuildEnvironmentNotFoundException( 404 "config.status is outdated or broken. Run configure." 405 ), 406 e, 407 ) 408 409 return self._config_environment 410 411 @property 412 def defines(self): 413 return self.config_environment.defines 414 415 @property 416 def substs(self): 417 return self.config_environment.substs 418 419 @property 420 def distdir(self): 421 return os.path.join(self.topobjdir, "dist") 422 423 @property 424 def bindir(self): 425 return os.path.join(self.topobjdir, "dist", "bin") 426 427 @property 428 def includedir(self): 429 return os.path.join(self.topobjdir, "dist", "include") 430 431 @property 432 def statedir(self): 433 return os.path.join(self.topobjdir, ".mozbuild") 434 435 @property 436 def platform(self): 437 """Returns current platform and architecture name""" 438 import mozinfo 439 440 platform_name = None 441 bits = str(mozinfo.info["bits"]) 442 if mozinfo.isLinux: 443 platform_name = "linux" + bits 444 elif mozinfo.isWin: 445 platform_name = "win" + bits 446 elif mozinfo.isMac: 447 platform_name = "macosx" + bits 448 449 return platform_name, bits + "bit" 450 451 @memoized_property 452 def repository(self): 453 """Get a `mozversioncontrol.Repository` object for the 454 top source directory.""" 455 # We try to obtain a repo using the configured VCS info first. 456 # If we don't have a configure context, fall back to auto-detection. 457 try: 458 return get_repository_from_build_config(self) 459 except ( 460 BuildEnvironmentNotFoundException, 461 MissingConfigureInfo, 462 MissingVCSTool, 463 ): 464 pass 465 466 return get_repository_object(self.topsrcdir) 467 468 def reload_config_environment(self): 469 """Force config.status to be re-read and return the new value 470 of ``self.config_environment``. 471 """ 472 self._config_environment = None 473 return self.config_environment 474 475 def mozbuild_reader( 476 self, config_mode="build", vcs_revision=None, vcs_check_clean=True 477 ): 478 """Obtain a ``BuildReader`` for evaluating moz.build files. 479 480 Given arguments, returns a ``mozbuild.frontend.reader.BuildReader`` 481 that can be used to evaluate moz.build files for this repo. 482 483 ``config_mode`` is either ``build`` or ``empty``. If ``build``, 484 ``self.config_environment`` is used. This requires a configured build 485 system to work. If ``empty``, an empty config is used. ``empty`` is 486 appropriate for file-based traversal mode where ``Files`` metadata is 487 read. 488 489 If ``vcs_revision`` is defined, it specifies a version control revision 490 to use to obtain files content. The default is to use the filesystem. 491 This mode is only supported with Mercurial repositories. 492 493 If ``vcs_revision`` is not defined and the version control checkout is 494 sparse, this implies ``vcs_revision='.'``. 495 496 If ``vcs_revision`` is ``.`` (denotes the parent of the working 497 directory), we will verify that the working directory is clean unless 498 ``vcs_check_clean`` is False. This prevents confusion due to uncommitted 499 file changes not being reflected in the reader. 500 """ 501 from mozbuild.frontend.reader import ( 502 default_finder, 503 BuildReader, 504 EmptyConfig, 505 ) 506 from mozpack.files import MercurialRevisionFinder 507 508 if config_mode == "build": 509 config = self.config_environment 510 elif config_mode == "empty": 511 config = EmptyConfig(self.topsrcdir) 512 else: 513 raise ValueError("unknown config_mode value: %s" % config_mode) 514 515 try: 516 repo = self.repository 517 except InvalidRepoPath: 518 repo = None 519 520 if repo and not vcs_revision and repo.sparse_checkout_present(): 521 vcs_revision = "." 522 523 if vcs_revision is None: 524 finder = default_finder 525 else: 526 # If we failed to detect the repo prior, check again to raise its 527 # exception. 528 if not repo: 529 self.repository 530 assert False 531 532 if repo.name != "hg": 533 raise Exception("do not support VCS reading mode for %s" % repo.name) 534 535 if vcs_revision == "." and vcs_check_clean: 536 with repo: 537 if not repo.working_directory_clean(): 538 raise Exception( 539 "working directory is not clean; " 540 "refusing to use a VCS-based finder" 541 ) 542 543 finder = MercurialRevisionFinder( 544 self.topsrcdir, rev=vcs_revision, recognize_repo_paths=True 545 ) 546 547 return BuildReader(config, finder=finder) 548 549 def is_clobber_needed(self): 550 if not os.path.exists(self.topobjdir): 551 return False 552 return Clobberer(self.topsrcdir, self.topobjdir).clobber_needed() 553 554 def get_binary_path(self, what="app", validate_exists=True, where="default"): 555 """Obtain the path to a compiled binary for this build configuration. 556 557 The what argument is the program or tool being sought after. See the 558 code implementation for supported values. 559 560 If validate_exists is True (the default), we will ensure the found path 561 exists before returning, raising an exception if it doesn't. 562 563 If where is 'staged-package', we will return the path to the binary in 564 the package staging directory. 565 566 If no arguments are specified, we will return the main binary for the 567 configured XUL application. 568 """ 569 570 if where not in ("default", "staged-package"): 571 raise Exception("Don't know location %s" % where) 572 573 substs = self.substs 574 575 stem = self.distdir 576 if where == "staged-package": 577 stem = os.path.join(stem, substs["MOZ_APP_NAME"]) 578 579 if substs["OS_ARCH"] == "Darwin" and "MOZ_MACBUNDLE_NAME" in substs: 580 stem = os.path.join(stem, substs["MOZ_MACBUNDLE_NAME"], "Contents", "MacOS") 581 elif where == "default": 582 stem = os.path.join(stem, "bin") 583 584 leaf = None 585 586 leaf = (substs["MOZ_APP_NAME"] if what == "app" else what) + substs[ 587 "BIN_SUFFIX" 588 ] 589 path = os.path.join(stem, leaf) 590 591 if validate_exists and not os.path.exists(path): 592 raise BinaryNotFoundException(path) 593 594 return path 595 596 def resolve_config_guess(self): 597 return self.mozconfig_and_target[1].alias 598 599 def notify(self, msg): 600 """Show a desktop notification with the supplied message 601 602 On Linux and Mac, this will show a desktop notification with the message, 603 but on Windows we can only flash the screen. 604 """ 605 if "MOZ_NOSPAM" in os.environ or "MOZ_AUTOMATION" in os.environ: 606 return 607 608 try: 609 if sys.platform.startswith("darwin"): 610 notifier = which("terminal-notifier") 611 if not notifier: 612 raise Exception( 613 "Install terminal-notifier to get " 614 "a notification when the build finishes." 615 ) 616 self.run_process( 617 [ 618 notifier, 619 "-title", 620 "Mozilla Build System", 621 "-group", 622 "mozbuild", 623 "-message", 624 msg, 625 ], 626 ensure_exit_code=False, 627 ) 628 elif sys.platform.startswith("win"): 629 from ctypes import Structure, windll, POINTER, sizeof, WINFUNCTYPE 630 from ctypes.wintypes import DWORD, HANDLE, BOOL, UINT 631 632 class FLASHWINDOW(Structure): 633 _fields_ = [ 634 ("cbSize", UINT), 635 ("hwnd", HANDLE), 636 ("dwFlags", DWORD), 637 ("uCount", UINT), 638 ("dwTimeout", DWORD), 639 ] 640 641 FlashWindowExProto = WINFUNCTYPE(BOOL, POINTER(FLASHWINDOW)) 642 FlashWindowEx = FlashWindowExProto(("FlashWindowEx", windll.user32)) 643 FLASHW_CAPTION = 0x01 644 FLASHW_TRAY = 0x02 645 FLASHW_TIMERNOFG = 0x0C 646 647 # GetConsoleWindows returns NULL if no console is attached. We 648 # can't flash nothing. 649 console = windll.kernel32.GetConsoleWindow() 650 if not console: 651 return 652 653 params = FLASHWINDOW( 654 sizeof(FLASHWINDOW), 655 console, 656 FLASHW_CAPTION | FLASHW_TRAY | FLASHW_TIMERNOFG, 657 3, 658 0, 659 ) 660 FlashWindowEx(params) 661 else: 662 notifier = which("notify-send") 663 if not notifier: 664 raise Exception( 665 "Install notify-send (usually part of " 666 "the libnotify package) to get a notification when " 667 "the build finishes." 668 ) 669 self.run_process( 670 [ 671 notifier, 672 "--app-name=Mozilla Build System", 673 "Mozilla Build System", 674 msg, 675 ], 676 ensure_exit_code=False, 677 ) 678 except Exception as e: 679 self.log( 680 logging.WARNING, 681 "notifier-failed", 682 {"error": str(e)}, 683 "Notification center failed: {error}", 684 ) 685 686 def _ensure_objdir_exists(self): 687 if os.path.isdir(self.statedir): 688 return 689 690 os.makedirs(self.statedir) 691 692 def _ensure_state_subdir_exists(self, subdir): 693 path = os.path.join(self.statedir, subdir) 694 695 if os.path.isdir(path): 696 return 697 698 os.makedirs(path) 699 700 def _get_state_filename(self, filename, subdir=None): 701 path = self.statedir 702 703 if subdir: 704 path = os.path.join(path, subdir) 705 706 return os.path.join(path, filename) 707 708 def _wrap_path_argument(self, arg): 709 return PathArgument(arg, self.topsrcdir, self.topobjdir) 710 711 def _run_make( 712 self, 713 directory=None, 714 filename=None, 715 target=None, 716 log=True, 717 srcdir=False, 718 line_handler=None, 719 append_env=None, 720 explicit_env=None, 721 ignore_errors=False, 722 ensure_exit_code=0, 723 silent=True, 724 print_directory=True, 725 pass_thru=False, 726 num_jobs=0, 727 job_size=0, 728 keep_going=False, 729 ): 730 """Invoke make. 731 732 directory -- Relative directory to look for Makefile in. 733 filename -- Explicit makefile to run. 734 target -- Makefile target(s) to make. Can be a string or iterable of 735 strings. 736 srcdir -- If True, invoke make from the source directory tree. 737 Otherwise, make will be invoked from the object directory. 738 silent -- If True (the default), run make in silent mode. 739 print_directory -- If True (the default), have make print directories 740 while doing traversal. 741 """ 742 self._ensure_objdir_exists() 743 744 args = [self.substs["GMAKE"]] 745 746 if directory: 747 args.extend(["-C", directory.replace(os.sep, "/")]) 748 749 if filename: 750 args.extend(["-f", filename]) 751 752 if num_jobs == 0 and self.mozconfig["make_flags"]: 753 flags = iter(self.mozconfig["make_flags"]) 754 for flag in flags: 755 if flag == "-j": 756 try: 757 flag = flags.next() 758 except StopIteration: 759 break 760 try: 761 num_jobs = int(flag) 762 except ValueError: 763 args.append(flag) 764 elif flag.startswith("-j"): 765 try: 766 num_jobs = int(flag[2:]) 767 except (ValueError, IndexError): 768 break 769 else: 770 args.append(flag) 771 772 if num_jobs == 0: 773 if job_size == 0: 774 job_size = 2.0 if self.substs.get("CC_TYPE") == "gcc" else 1.0 # GiB 775 776 cpus = multiprocessing.cpu_count() 777 if not psutil or not job_size: 778 num_jobs = cpus 779 else: 780 mem_gb = psutil.virtual_memory().total / 1024 ** 3 781 from_mem = round(mem_gb / job_size) 782 num_jobs = max(1, min(cpus, from_mem)) 783 print( 784 " Parallelism determined by memory: using %d jobs for %d cores " 785 "based on %.1f GiB RAM and estimated job size of %.1f GiB" 786 % (num_jobs, cpus, mem_gb, job_size) 787 ) 788 789 args.append("-j%d" % num_jobs) 790 791 if ignore_errors: 792 args.append("-k") 793 794 if silent: 795 args.append("-s") 796 797 # Print entering/leaving directory messages. Some consumers look at 798 # these to measure progress. 799 if print_directory: 800 args.append("-w") 801 802 if keep_going: 803 args.append("-k") 804 805 if isinstance(target, list): 806 args.extend(target) 807 elif target: 808 args.append(target) 809 810 fn = self._run_command_in_objdir 811 812 if srcdir: 813 fn = self._run_command_in_srcdir 814 815 append_env = dict(append_env or ()) 816 append_env["MACH"] = "1" 817 818 params = { 819 "args": args, 820 "line_handler": line_handler, 821 "append_env": append_env, 822 "explicit_env": explicit_env, 823 "log_level": logging.INFO, 824 "require_unix_environment": False, 825 "ensure_exit_code": ensure_exit_code, 826 "pass_thru": pass_thru, 827 # Make manages its children, so mozprocess doesn't need to bother. 828 # Having mozprocess manage children can also have side-effects when 829 # building on Windows. See bug 796840. 830 "ignore_children": True, 831 } 832 833 if log: 834 params["log_name"] = "make" 835 836 return fn(**params) 837 838 def _run_command_in_srcdir(self, **args): 839 return self.run_process(cwd=self.topsrcdir, **args) 840 841 def _run_command_in_objdir(self, **args): 842 return self.run_process(cwd=self.topobjdir, **args) 843 844 def _is_windows(self): 845 return os.name in ("nt", "ce") 846 847 def _is_osx(self): 848 return "darwin" in str(sys.platform).lower() 849 850 def _spawn(self, cls): 851 """Create a new MozbuildObject-derived class instance from ourselves. 852 853 This is used as a convenience method to create other 854 MozbuildObject-derived class instances. It can only be used on 855 classes that have the same constructor arguments as us. 856 """ 857 858 return cls( 859 self.topsrcdir, self.settings, self.log_manager, topobjdir=self.topobjdir 860 ) 861 862 def activate_virtualenv(self): 863 self.virtualenv_manager.activate() 864 865 def _set_log_level(self, verbose): 866 self.log_manager.terminal_handler.setLevel( 867 logging.INFO if not verbose else logging.DEBUG 868 ) 869 870 def _ensure_zstd(self): 871 try: 872 import zstandard # noqa: F401 873 except (ImportError, AttributeError): 874 self.activate_virtualenv() 875 self.virtualenv_manager.install_pip_requirements( 876 os.path.join(self.topsrcdir, "build", "zstandard_requirements.txt") 877 ) 878 879 880class MachCommandBase(MozbuildObject): 881 """Base class for mach command providers that wish to be MozbuildObjects. 882 883 This provides a level of indirection so MozbuildObject can be refactored 884 without having to change everything that inherits from it. 885 """ 886 887 def __init__(self, context, virtualenv_name=None, metrics=None, no_auto_log=False): 888 # Attempt to discover topobjdir through environment detection, as it is 889 # more reliable than mozconfig when cwd is inside an objdir. 890 topsrcdir = context.topdir 891 topobjdir = None 892 detect_virtualenv_mozinfo = True 893 if hasattr(context, "detect_virtualenv_mozinfo"): 894 detect_virtualenv_mozinfo = getattr(context, "detect_virtualenv_mozinfo") 895 try: 896 dummy = MozbuildObject.from_environment( 897 cwd=context.cwd, detect_virtualenv_mozinfo=detect_virtualenv_mozinfo 898 ) 899 topsrcdir = dummy.topsrcdir 900 topobjdir = dummy._topobjdir 901 if topobjdir: 902 # If we're inside a objdir and the found mozconfig resolves to 903 # another objdir, we abort. The reasoning here is that if you 904 # are inside an objdir you probably want to perform actions on 905 # that objdir, not another one. This prevents accidental usage 906 # of the wrong objdir when the current objdir is ambiguous. 907 config_topobjdir = dummy.resolve_mozconfig_topobjdir() 908 909 if config_topobjdir and not samepath(topobjdir, config_topobjdir): 910 raise ObjdirMismatchException(topobjdir, config_topobjdir) 911 except BuildEnvironmentNotFoundException: 912 pass 913 except ObjdirMismatchException as e: 914 print( 915 "Ambiguous object directory detected. We detected that " 916 "both %s and %s could be object directories. This is " 917 "typically caused by having a mozconfig pointing to a " 918 "different object directory from the current working " 919 "directory. To solve this problem, ensure you do not have a " 920 "default mozconfig in searched paths." % (e.objdir1, e.objdir2) 921 ) 922 sys.exit(1) 923 924 except MozconfigLoadException as e: 925 print(e) 926 sys.exit(1) 927 928 MozbuildObject.__init__( 929 self, 930 topsrcdir, 931 context.settings, 932 context.log_manager, 933 topobjdir=topobjdir, 934 virtualenv_name=virtualenv_name, 935 ) 936 937 self._mach_context = context 938 self.metrics = metrics 939 940 # Incur mozconfig processing so we have unified error handling for 941 # errors. Otherwise, the exceptions could bubble back to mach's error 942 # handler. 943 try: 944 self.mozconfig 945 946 except MozconfigFindException as e: 947 print(e) 948 sys.exit(1) 949 950 except MozconfigLoadException as e: 951 print(e) 952 sys.exit(1) 953 954 # Always keep a log of the last command, but don't do that for mach 955 # invokations from scripts (especially not the ones done by the build 956 # system itself). 957 try: 958 fileno = getattr(sys.stdout, "fileno", lambda: None)() 959 except io.UnsupportedOperation: 960 fileno = None 961 if fileno and os.isatty(fileno) and not no_auto_log: 962 self._ensure_state_subdir_exists(".") 963 logfile = self._get_state_filename("last_log.json") 964 try: 965 fd = open(logfile, "wt") 966 self.log_manager.add_json_handler(fd) 967 except Exception as e: 968 self.log( 969 logging.WARNING, 970 "mach", 971 {"error": str(e)}, 972 "Log will not be kept for this command: {error}.", 973 ) 974 975 def _sub_mach(self, argv): 976 return subprocess.call( 977 [sys.executable, os.path.join(self.topsrcdir, "mach")] + argv 978 ) 979 980 981class MachCommandConditions(object): 982 """A series of commonly used condition functions which can be applied to 983 mach commands with providers deriving from MachCommandBase. 984 """ 985 986 @staticmethod 987 def is_firefox(cls): 988 """Must have a Firefox build.""" 989 if hasattr(cls, "substs"): 990 return cls.substs.get("MOZ_BUILD_APP") == "browser" 991 return False 992 993 @staticmethod 994 def is_jsshell(cls): 995 """Must have a jsshell build.""" 996 if hasattr(cls, "substs"): 997 return cls.substs.get("MOZ_BUILD_APP") == "js" 998 return False 999 1000 @staticmethod 1001 def is_thunderbird(cls): 1002 """Must have a Thunderbird build.""" 1003 if hasattr(cls, "substs"): 1004 return cls.substs.get("MOZ_BUILD_APP") == "comm/mail" 1005 return False 1006 1007 @staticmethod 1008 def is_firefox_or_thunderbird(cls): 1009 """Must have a Firefox or Thunderbird build.""" 1010 return MachCommandConditions.is_firefox( 1011 cls 1012 ) or MachCommandConditions.is_thunderbird(cls) 1013 1014 @staticmethod 1015 def is_android(cls): 1016 """Must have an Android build.""" 1017 if hasattr(cls, "substs"): 1018 return cls.substs.get("MOZ_WIDGET_TOOLKIT") == "android" 1019 return False 1020 1021 @staticmethod 1022 def is_not_android(cls): 1023 """Must not have an Android build.""" 1024 if hasattr(cls, "substs"): 1025 return cls.substs.get("MOZ_WIDGET_TOOLKIT") != "android" 1026 return False 1027 1028 @staticmethod 1029 def is_firefox_or_android(cls): 1030 """Must have a Firefox or Android build.""" 1031 return MachCommandConditions.is_firefox( 1032 cls 1033 ) or MachCommandConditions.is_android(cls) 1034 1035 @staticmethod 1036 def has_build(cls): 1037 """Must have a build.""" 1038 return MachCommandConditions.is_firefox_or_android( 1039 cls 1040 ) or MachCommandConditions.is_thunderbird(cls) 1041 1042 @staticmethod 1043 def has_build_or_shell(cls): 1044 """Must have a build or a shell build.""" 1045 return MachCommandConditions.has_build(cls) or MachCommandConditions.is_jsshell( 1046 cls 1047 ) 1048 1049 @staticmethod 1050 def is_hg(cls): 1051 """Must have a mercurial source checkout.""" 1052 try: 1053 return isinstance(cls.repository, HgRepository) 1054 except InvalidRepoPath: 1055 return False 1056 1057 @staticmethod 1058 def is_git(cls): 1059 """Must have a git source checkout.""" 1060 try: 1061 return isinstance(cls.repository, GitRepository) 1062 except InvalidRepoPath: 1063 return False 1064 1065 @staticmethod 1066 def is_artifact_build(cls): 1067 """Must be an artifact build.""" 1068 if hasattr(cls, "substs"): 1069 return getattr(cls, "substs", {}).get("MOZ_ARTIFACT_BUILDS") 1070 return False 1071 1072 @staticmethod 1073 def is_non_artifact_build(cls): 1074 """Must not be an artifact build.""" 1075 if hasattr(cls, "substs"): 1076 return not MachCommandConditions.is_artifact_build(cls) 1077 return False 1078 1079 @staticmethod 1080 def is_buildapp_in(cls, apps): 1081 """Must have a build for one of the given app""" 1082 for app in apps: 1083 attr = getattr(MachCommandConditions, "is_{}".format(app), None) 1084 if attr and attr(cls): 1085 return True 1086 return False 1087 1088 1089class PathArgument(object): 1090 """Parse a filesystem path argument and transform it in various ways.""" 1091 1092 def __init__(self, arg, topsrcdir, topobjdir, cwd=None): 1093 self.arg = arg 1094 self.topsrcdir = topsrcdir 1095 self.topobjdir = topobjdir 1096 self.cwd = os.getcwd() if cwd is None else cwd 1097 1098 def relpath(self): 1099 """Return a path relative to the topsrcdir or topobjdir. 1100 1101 If the argument is a path to a location in one of the base directories 1102 (topsrcdir or topobjdir), then strip off the base directory part and 1103 just return the path within the base directory.""" 1104 1105 abspath = os.path.abspath(os.path.join(self.cwd, self.arg)) 1106 1107 # If that path is within topsrcdir or topobjdir, return an equivalent 1108 # path relative to that base directory. 1109 for base_dir in [self.topobjdir, self.topsrcdir]: 1110 if abspath.startswith(os.path.abspath(base_dir)): 1111 return mozpath.relpath(abspath, base_dir) 1112 1113 return mozpath.normsep(self.arg) 1114 1115 def srcdir_path(self): 1116 return mozpath.join(self.topsrcdir, self.relpath()) 1117 1118 def objdir_path(self): 1119 return mozpath.join(self.topobjdir, self.relpath()) 1120 1121 1122class ExecutionSummary(dict): 1123 """Helper for execution summaries.""" 1124 1125 def __init__(self, summary_format, **data): 1126 self._summary_format = "" 1127 assert "execution_time" in data 1128 self.extend(summary_format, **data) 1129 1130 def extend(self, summary_format, **data): 1131 self._summary_format += summary_format 1132 self.update(data) 1133 1134 def __str__(self): 1135 return self._summary_format.format(**self) 1136 1137 def __getattr__(self, key): 1138 return self[key] 1139