1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5from __future__ import absolute_import, print_function, unicode_literals 6 7import io 8import json 9import logging 10import mozpack.path as mozpath 11import multiprocessing 12import os 13import six 14import subprocess 15import sys 16import errno 17 18from mach.mixin.process import ProcessExecutionMixin 19from mozboot.mozconfig import MozconfigFindException 20from mozfile import which 21from mozversioncontrol import ( 22 get_repository_from_build_config, 23 get_repository_object, 24 GitRepository, 25 HgRepository, 26 InvalidRepoPath, 27 MissingConfigureInfo, 28) 29 30from .backend.configenvironment import ( 31 ConfigEnvironment, 32 ConfigStatusFailure, 33) 34from .configure import ConfigureSandbox 35from .controller.clobber import Clobberer 36from .mozconfig import ( 37 MozconfigLoadException, 38 MozconfigLoader, 39) 40from .pythonutil import find_python3_executable 41from .util import ( 42 memoize, 43 memoized_property, 44) 45 46 47def ancestors(path): 48 """Emit the parent directories of a path.""" 49 while path: 50 yield path 51 newpath = os.path.dirname(path) 52 if newpath == path: 53 break 54 path = newpath 55 56 57def samepath(path1, path2): 58 # Under Python 3 (but NOT Python 2), MozillaBuild exposes the 59 # os.path.samefile function despite it not working, so only use it if we're 60 # not running under Windows. 61 if hasattr(os.path, "samefile") and os.name != "nt": 62 return os.path.samefile(path1, path2) 63 return os.path.normcase(os.path.realpath(path1)) == os.path.normcase( 64 os.path.realpath(path2) 65 ) 66 67 68class BadEnvironmentException(Exception): 69 """Base class for errors raised when the build environment is not sane.""" 70 71 72class BuildEnvironmentNotFoundException(BadEnvironmentException, AttributeError): 73 """Raised when we could not find a build environment.""" 74 75 76class ObjdirMismatchException(BadEnvironmentException): 77 """Raised when the current dir is an objdir and doesn't match the mozconfig.""" 78 79 def __init__(self, objdir1, objdir2): 80 self.objdir1 = objdir1 81 self.objdir2 = objdir2 82 83 def __str__(self): 84 return "Objdir mismatch: %s != %s" % (self.objdir1, self.objdir2) 85 86 87class BinaryNotFoundException(Exception): 88 """Raised when the binary is not found in the expected location.""" 89 90 def __init__(self, path): 91 self.path = path 92 93 def __str__(self): 94 return "Binary expected at {} does not exist.".format(self.path) 95 96 def help(self): 97 return "It looks like your program isn't built. You can run |./mach build| to build it." 98 99 100class MozbuildObject(ProcessExecutionMixin): 101 """Base class providing basic functionality useful to many modules. 102 103 Modules in this package typically require common functionality such as 104 accessing the current config, getting the location of the source directory, 105 running processes, etc. This classes provides that functionality. Other 106 modules can inherit from this class to obtain this functionality easily. 107 """ 108 109 def __init__( 110 self, 111 topsrcdir, 112 settings, 113 log_manager, 114 topobjdir=None, 115 mozconfig=MozconfigLoader.AUTODETECT, 116 virtualenv_name=None, 117 ): 118 """Create a new Mozbuild object instance. 119 120 Instances are bound to a source directory, a ConfigSettings instance, 121 and a LogManager instance. The topobjdir may be passed in as well. If 122 it isn't, it will be calculated from the active mozconfig. 123 """ 124 self.topsrcdir = mozpath.normsep(topsrcdir) 125 self.settings = settings 126 127 self.populate_logger() 128 self.log_manager = log_manager 129 130 self._make = None 131 self._topobjdir = mozpath.normsep(topobjdir) if topobjdir else topobjdir 132 self._mozconfig = mozconfig 133 self._config_environment = None 134 self._virtualenv_name = virtualenv_name or "common" 135 self._virtualenv_manager = None 136 137 @classmethod 138 def from_environment(cls, cwd=None, detect_virtualenv_mozinfo=True, **kwargs): 139 """Create a MozbuildObject by detecting the proper one from the env. 140 141 This examines environment state like the current working directory and 142 creates a MozbuildObject from the found source directory, mozconfig, etc. 143 144 The role of this function is to identify a topsrcdir, topobjdir, and 145 mozconfig file. 146 147 If the current working directory is inside a known objdir, we always 148 use the topsrcdir and mozconfig associated with that objdir. 149 150 If the current working directory is inside a known srcdir, we use that 151 topsrcdir and look for mozconfigs using the default mechanism, which 152 looks inside environment variables. 153 154 If the current Python interpreter is running from a virtualenv inside 155 an objdir, we use that as our objdir. 156 157 If we're not inside a srcdir or objdir, an exception is raised. 158 159 detect_virtualenv_mozinfo determines whether we should look for a 160 mozinfo.json file relative to the virtualenv directory. This was 161 added to facilitate testing. Callers likely shouldn't change the 162 default. 163 """ 164 165 cwd = os.path.realpath(cwd or os.getcwd()) 166 topsrcdir = None 167 topobjdir = None 168 mozconfig = MozconfigLoader.AUTODETECT 169 170 def load_mozinfo(path): 171 info = json.load(io.open(path, "rt", encoding="utf-8")) 172 topsrcdir = info.get("topsrcdir") 173 topobjdir = os.path.dirname(path) 174 mozconfig = info.get("mozconfig") 175 return topsrcdir, topobjdir, mozconfig 176 177 for dir_path in ancestors(cwd): 178 # If we find a mozinfo.json, we are in the objdir. 179 mozinfo_path = os.path.join(dir_path, "mozinfo.json") 180 if os.path.isfile(mozinfo_path): 181 topsrcdir, topobjdir, mozconfig = load_mozinfo(mozinfo_path) 182 break 183 184 # We choose an arbitrary file as an indicator that this is a 185 # srcdir. We go with ourself because why not! 186 our_path = os.path.join( 187 dir_path, "python", "mozbuild", "mozbuild", "base.py" 188 ) 189 if os.path.isfile(our_path): 190 topsrcdir = dir_path 191 break 192 193 # See if we're running from a Python virtualenv that's inside an objdir. 194 mozinfo_path = os.path.join(os.path.dirname(sys.prefix), "../mozinfo.json") 195 if detect_virtualenv_mozinfo and os.path.isfile(mozinfo_path): 196 topsrcdir, topobjdir, mozconfig = load_mozinfo(mozinfo_path) 197 198 if not topsrcdir: 199 topsrcdir = os.path.abspath( 200 os.path.join(os.path.dirname(__file__), "..", "..", "..") 201 ) 202 203 topsrcdir = mozpath.normsep(topsrcdir) 204 if topobjdir: 205 topobjdir = mozpath.normsep(os.path.normpath(topobjdir)) 206 207 if topsrcdir == topobjdir: 208 raise BadEnvironmentException( 209 "The object directory appears " 210 "to be the same as your source directory (%s). This build " 211 "configuration is not supported." % topsrcdir 212 ) 213 214 # If we can't resolve topobjdir, oh well. We'll figure out when we need 215 # one. 216 return cls( 217 topsrcdir, None, None, topobjdir=topobjdir, mozconfig=mozconfig, **kwargs 218 ) 219 220 def resolve_mozconfig_topobjdir(self, default=None): 221 topobjdir = self.mozconfig["topobjdir"] or default 222 if not topobjdir: 223 return None 224 225 if "@CONFIG_GUESS@" in topobjdir: 226 topobjdir = topobjdir.replace("@CONFIG_GUESS@", self.resolve_config_guess()) 227 228 if not os.path.isabs(topobjdir): 229 topobjdir = os.path.abspath(os.path.join(self.topsrcdir, topobjdir)) 230 231 return mozpath.normsep(os.path.normpath(topobjdir)) 232 233 def build_out_of_date(self, output, dep_file): 234 if not os.path.isfile(output): 235 print(" Output reference file not found: %s" % output) 236 return True 237 if not os.path.isfile(dep_file): 238 print(" Dependency file not found: %s" % dep_file) 239 return True 240 241 deps = [] 242 with io.open(dep_file, "r", encoding="utf-8", newline="\n") as fh: 243 deps = fh.read().splitlines() 244 245 mtime = os.path.getmtime(output) 246 for f in deps: 247 try: 248 dep_mtime = os.path.getmtime(f) 249 except OSError as e: 250 if e.errno == errno.ENOENT: 251 print(" Input not found: %s" % f) 252 return True 253 raise 254 if dep_mtime > mtime: 255 print(" %s is out of date with respect to %s" % (output, f)) 256 return True 257 return False 258 259 def backend_out_of_date(self, backend_file): 260 if not os.path.isfile(backend_file): 261 return True 262 263 # Check if any of our output files have been removed since 264 # we last built the backend, re-generate the backend if 265 # so. 266 outputs = [] 267 with io.open(backend_file, "r", encoding="utf-8", newline="\n") as fh: 268 outputs = fh.read().splitlines() 269 for output in outputs: 270 if not os.path.isfile(mozpath.join(self.topobjdir, output)): 271 return True 272 273 dep_file = "%s.in" % backend_file 274 return self.build_out_of_date(backend_file, dep_file) 275 276 @property 277 def topobjdir(self): 278 if self._topobjdir is None: 279 self._topobjdir = self.resolve_mozconfig_topobjdir( 280 default="obj-@CONFIG_GUESS@" 281 ) 282 283 return self._topobjdir 284 285 @property 286 def virtualenv_manager(self): 287 from .virtualenv import VirtualenvManager 288 289 if self._virtualenv_manager is None: 290 self._virtualenv_manager = VirtualenvManager( 291 self.topsrcdir, 292 os.path.join(self.topobjdir, "_virtualenvs", self._virtualenv_name), 293 sys.stdout, 294 os.path.join(self.topsrcdir, "build", "build_virtualenv_packages.txt"), 295 ) 296 297 return self._virtualenv_manager 298 299 @staticmethod 300 @memoize 301 def get_mozconfig_and_target(topsrcdir, path, env_mozconfig): 302 # env_mozconfig is only useful for unittests, which change the value of 303 # the environment variable, which has an impact on autodetection (when 304 # path is MozconfigLoader.AUTODETECT), and memoization wouldn't account 305 # for it without the explicit (unused) argument. 306 out = six.StringIO() 307 env = os.environ 308 if path and path != MozconfigLoader.AUTODETECT: 309 env = dict(env) 310 env["MOZCONFIG"] = path 311 312 # We use python configure to get mozconfig content and the value for 313 # --target (from mozconfig if necessary, guessed otherwise). 314 315 # Modified configure sandbox that replaces '--help' dependencies with 316 # `always`, such that depends functions with a '--help' dependency are 317 # not automatically executed when including files. We don't want all of 318 # those from init.configure to execute, only a subset. 319 class ReducedConfigureSandbox(ConfigureSandbox): 320 def depends_impl(self, *args, **kwargs): 321 args = tuple( 322 a 323 if not isinstance(a, six.string_types) or a != "--help" 324 else self._always.sandboxed 325 for a in args 326 ) 327 return super(ReducedConfigureSandbox, self).depends_impl( 328 *args, **kwargs 329 ) 330 331 # This may be called recursively from configure itself for $reasons, 332 # so avoid logging to the same logger (configure uses "moz.configure") 333 logger = logging.getLogger("moz.configure.reduced") 334 handler = logging.StreamHandler(out) 335 logger.addHandler(handler) 336 # If this were true, logging would still propagate to "moz.configure". 337 logger.propagate = False 338 sandbox = ReducedConfigureSandbox( 339 {}, 340 environ=env, 341 argv=["mach", "--help"], 342 logger=logger, 343 ) 344 base_dir = os.path.join(topsrcdir, "build", "moz.configure") 345 try: 346 sandbox.include_file(os.path.join(base_dir, "init.configure")) 347 # Force mozconfig options injection before getting the target. 348 sandbox._value_for(sandbox["mozconfig_options"]) 349 return ( 350 sandbox._value_for(sandbox["mozconfig"]), 351 sandbox._value_for(sandbox["real_target"]), 352 ) 353 except SystemExit: 354 print(out.getvalue()) 355 raise 356 357 @property 358 def mozconfig_and_target(self): 359 return self.get_mozconfig_and_target( 360 self.topsrcdir, self._mozconfig, os.environ.get("MOZCONFIG") 361 ) 362 363 @property 364 def mozconfig(self): 365 """Returns information about the current mozconfig file. 366 367 This a dict as returned by MozconfigLoader.read_mozconfig() 368 """ 369 return self.mozconfig_and_target[0] 370 371 @property 372 def config_environment(self): 373 """Returns the ConfigEnvironment for the current build configuration. 374 375 This property is only available once configure has executed. 376 377 If configure's output is not available, this will raise. 378 """ 379 if self._config_environment: 380 return self._config_environment 381 382 config_status = os.path.join(self.topobjdir, "config.status") 383 384 if not os.path.exists(config_status) or not os.path.getsize(config_status): 385 raise BuildEnvironmentNotFoundException( 386 "config.status not available. Run configure." 387 ) 388 389 try: 390 self._config_environment = ConfigEnvironment.from_config_status( 391 config_status 392 ) 393 except ConfigStatusFailure as e: 394 six.raise_from( 395 BuildEnvironmentNotFoundException( 396 "config.status is outdated or broken. Run configure." 397 ), 398 e, 399 ) 400 401 return self._config_environment 402 403 @property 404 def defines(self): 405 return self.config_environment.defines 406 407 @property 408 def substs(self): 409 return self.config_environment.substs 410 411 @property 412 def distdir(self): 413 return os.path.join(self.topobjdir, "dist") 414 415 @property 416 def bindir(self): 417 return os.path.join(self.topobjdir, "dist", "bin") 418 419 @property 420 def includedir(self): 421 return os.path.join(self.topobjdir, "dist", "include") 422 423 @property 424 def statedir(self): 425 return os.path.join(self.topobjdir, ".mozbuild") 426 427 @property 428 def platform(self): 429 """Returns current platform and architecture name""" 430 import mozinfo 431 432 platform_name = None 433 bits = str(mozinfo.info["bits"]) 434 if mozinfo.isLinux: 435 platform_name = "linux" + bits 436 elif mozinfo.isWin: 437 platform_name = "win" + bits 438 elif mozinfo.isMac: 439 platform_name = "macosx" + bits 440 441 return platform_name, bits + "bit" 442 443 @memoized_property 444 def repository(self): 445 """Get a `mozversioncontrol.Repository` object for the 446 top source directory.""" 447 # We try to obtain a repo using the configured VCS info first. 448 # If we don't have a configure context, fall back to auto-detection. 449 try: 450 return get_repository_from_build_config(self) 451 except (BuildEnvironmentNotFoundException, MissingConfigureInfo): 452 pass 453 454 return get_repository_object(self.topsrcdir) 455 456 def reload_config_environment(self): 457 """Force config.status to be re-read and return the new value 458 of ``self.config_environment``. 459 """ 460 self._config_environment = None 461 return self.config_environment 462 463 def mozbuild_reader( 464 self, config_mode="build", vcs_revision=None, vcs_check_clean=True 465 ): 466 """Obtain a ``BuildReader`` for evaluating moz.build files. 467 468 Given arguments, returns a ``mozbuild.frontend.reader.BuildReader`` 469 that can be used to evaluate moz.build files for this repo. 470 471 ``config_mode`` is either ``build`` or ``empty``. If ``build``, 472 ``self.config_environment`` is used. This requires a configured build 473 system to work. If ``empty``, an empty config is used. ``empty`` is 474 appropriate for file-based traversal mode where ``Files`` metadata is 475 read. 476 477 If ``vcs_revision`` is defined, it specifies a version control revision 478 to use to obtain files content. The default is to use the filesystem. 479 This mode is only supported with Mercurial repositories. 480 481 If ``vcs_revision`` is not defined and the version control checkout is 482 sparse, this implies ``vcs_revision='.'``. 483 484 If ``vcs_revision`` is ``.`` (denotes the parent of the working 485 directory), we will verify that the working directory is clean unless 486 ``vcs_check_clean`` is False. This prevents confusion due to uncommitted 487 file changes not being reflected in the reader. 488 """ 489 from mozbuild.frontend.reader import ( 490 default_finder, 491 BuildReader, 492 EmptyConfig, 493 ) 494 from mozpack.files import MercurialRevisionFinder 495 496 if config_mode == "build": 497 config = self.config_environment 498 elif config_mode == "empty": 499 config = EmptyConfig(self.topsrcdir) 500 else: 501 raise ValueError("unknown config_mode value: %s" % config_mode) 502 503 try: 504 repo = self.repository 505 except InvalidRepoPath: 506 repo = None 507 508 if repo and not vcs_revision and repo.sparse_checkout_present(): 509 vcs_revision = "." 510 511 if vcs_revision is None: 512 finder = default_finder 513 else: 514 # If we failed to detect the repo prior, check again to raise its 515 # exception. 516 if not repo: 517 self.repository 518 assert False 519 520 if repo.name != "hg": 521 raise Exception("do not support VCS reading mode for %s" % repo.name) 522 523 if vcs_revision == "." and vcs_check_clean: 524 with repo: 525 if not repo.working_directory_clean(): 526 raise Exception( 527 "working directory is not clean; " 528 "refusing to use a VCS-based finder" 529 ) 530 531 finder = MercurialRevisionFinder( 532 self.topsrcdir, rev=vcs_revision, recognize_repo_paths=True 533 ) 534 535 return BuildReader(config, finder=finder) 536 537 @memoized_property 538 def python3(self): 539 """Obtain info about a Python 3 executable. 540 541 Returns a tuple of an executable path and its version (as a tuple). 542 Either both entries will have a value or both will be None. 543 """ 544 # Search configured build info first. Then fall back to system. 545 try: 546 subst = self.substs 547 548 if "PYTHON3" in subst: 549 version = tuple(map(int, subst["PYTHON3_VERSION"].split("."))) 550 return subst["PYTHON3"], version 551 except BuildEnvironmentNotFoundException: 552 pass 553 554 return find_python3_executable() 555 556 def is_clobber_needed(self): 557 if not os.path.exists(self.topobjdir): 558 return False 559 return Clobberer(self.topsrcdir, self.topobjdir).clobber_needed() 560 561 def get_binary_path(self, what="app", validate_exists=True, where="default"): 562 """Obtain the path to a compiled binary for this build configuration. 563 564 The what argument is the program or tool being sought after. See the 565 code implementation for supported values. 566 567 If validate_exists is True (the default), we will ensure the found path 568 exists before returning, raising an exception if it doesn't. 569 570 If where is 'staged-package', we will return the path to the binary in 571 the package staging directory. 572 573 If no arguments are specified, we will return the main binary for the 574 configured XUL application. 575 """ 576 577 if where not in ("default", "staged-package"): 578 raise Exception("Don't know location %s" % where) 579 580 substs = self.substs 581 582 stem = self.distdir 583 if where == "staged-package": 584 stem = os.path.join(stem, substs["MOZ_APP_NAME"]) 585 586 if substs["OS_ARCH"] == "Darwin" and "MOZ_MACBUNDLE_NAME" in substs: 587 stem = os.path.join(stem, substs["MOZ_MACBUNDLE_NAME"], "Contents", "MacOS") 588 elif where == "default": 589 stem = os.path.join(stem, "bin") 590 591 leaf = None 592 593 leaf = (substs["MOZ_APP_NAME"] if what == "app" else what) + substs[ 594 "BIN_SUFFIX" 595 ] 596 path = os.path.join(stem, leaf) 597 598 if validate_exists and not os.path.exists(path): 599 raise BinaryNotFoundException(path) 600 601 return path 602 603 def resolve_config_guess(self): 604 return self.mozconfig_and_target[1].alias 605 606 def notify(self, msg): 607 """Show a desktop notification with the supplied message 608 609 On Linux and Mac, this will show a desktop notification with the message, 610 but on Windows we can only flash the screen. 611 """ 612 if "MOZ_NOSPAM" in os.environ or "MOZ_AUTOMATION" in os.environ: 613 return 614 615 try: 616 if sys.platform.startswith("darwin"): 617 notifier = which("terminal-notifier") 618 if not notifier: 619 raise Exception( 620 "Install terminal-notifier to get " 621 "a notification when the build finishes." 622 ) 623 self.run_process( 624 [ 625 notifier, 626 "-title", 627 "Mozilla Build System", 628 "-group", 629 "mozbuild", 630 "-message", 631 msg, 632 ], 633 ensure_exit_code=False, 634 ) 635 elif sys.platform.startswith("win"): 636 from ctypes import Structure, windll, POINTER, sizeof, WINFUNCTYPE 637 from ctypes.wintypes import DWORD, HANDLE, BOOL, UINT 638 639 class FLASHWINDOW(Structure): 640 _fields_ = [ 641 ("cbSize", UINT), 642 ("hwnd", HANDLE), 643 ("dwFlags", DWORD), 644 ("uCount", UINT), 645 ("dwTimeout", DWORD), 646 ] 647 648 FlashWindowExProto = WINFUNCTYPE(BOOL, POINTER(FLASHWINDOW)) 649 FlashWindowEx = FlashWindowExProto(("FlashWindowEx", windll.user32)) 650 FLASHW_CAPTION = 0x01 651 FLASHW_TRAY = 0x02 652 FLASHW_TIMERNOFG = 0x0C 653 654 # GetConsoleWindows returns NULL if no console is attached. We 655 # can't flash nothing. 656 console = windll.kernel32.GetConsoleWindow() 657 if not console: 658 return 659 660 params = FLASHWINDOW( 661 sizeof(FLASHWINDOW), 662 console, 663 FLASHW_CAPTION | FLASHW_TRAY | FLASHW_TIMERNOFG, 664 3, 665 0, 666 ) 667 FlashWindowEx(params) 668 else: 669 notifier = which("notify-send") 670 if not notifier: 671 raise Exception( 672 "Install notify-send (usually part of " 673 "the libnotify package) to get a notification when " 674 "the build finishes." 675 ) 676 self.run_process( 677 [ 678 notifier, 679 "--app-name=Mozilla Build System", 680 "Mozilla Build System", 681 msg, 682 ], 683 ensure_exit_code=False, 684 ) 685 except Exception as e: 686 self.log( 687 logging.WARNING, 688 "notifier-failed", 689 {"error": str(e)}, 690 "Notification center failed: {error}", 691 ) 692 693 def _ensure_objdir_exists(self): 694 if os.path.isdir(self.statedir): 695 return 696 697 os.makedirs(self.statedir) 698 699 def _ensure_state_subdir_exists(self, subdir): 700 path = os.path.join(self.statedir, subdir) 701 702 if os.path.isdir(path): 703 return 704 705 os.makedirs(path) 706 707 def _get_state_filename(self, filename, subdir=None): 708 path = self.statedir 709 710 if subdir: 711 path = os.path.join(path, subdir) 712 713 return os.path.join(path, filename) 714 715 def _wrap_path_argument(self, arg): 716 return PathArgument(arg, self.topsrcdir, self.topobjdir) 717 718 def _run_make( 719 self, 720 directory=None, 721 filename=None, 722 target=None, 723 log=True, 724 srcdir=False, 725 allow_parallel=True, 726 line_handler=None, 727 append_env=None, 728 explicit_env=None, 729 ignore_errors=False, 730 ensure_exit_code=0, 731 silent=True, 732 print_directory=True, 733 pass_thru=False, 734 num_jobs=0, 735 keep_going=False, 736 ): 737 """Invoke make. 738 739 directory -- Relative directory to look for Makefile in. 740 filename -- Explicit makefile to run. 741 target -- Makefile target(s) to make. Can be a string or iterable of 742 strings. 743 srcdir -- If True, invoke make from the source directory tree. 744 Otherwise, make will be invoked from the object directory. 745 silent -- If True (the default), run make in silent mode. 746 print_directory -- If True (the default), have make print directories 747 while doing traversal. 748 """ 749 self._ensure_objdir_exists() 750 751 args = [self.substs["GMAKE"]] 752 753 if directory: 754 args.extend(["-C", directory.replace(os.sep, "/")]) 755 756 if filename: 757 args.extend(["-f", filename]) 758 759 if num_jobs == 0 and self.mozconfig["make_flags"]: 760 flags = iter(self.mozconfig["make_flags"]) 761 for flag in flags: 762 if flag == "-j": 763 try: 764 flag = flags.next() 765 except StopIteration: 766 break 767 try: 768 num_jobs = int(flag) 769 except ValueError: 770 args.append(flag) 771 elif flag.startswith("-j"): 772 try: 773 num_jobs = int(flag[2:]) 774 except (ValueError, IndexError): 775 break 776 else: 777 args.append(flag) 778 779 if allow_parallel: 780 if num_jobs > 0: 781 args.append("-j%d" % num_jobs) 782 else: 783 args.append("-j%d" % multiprocessing.cpu_count()) 784 elif num_jobs > 0: 785 args.append("MOZ_PARALLEL_BUILD=%d" % num_jobs) 786 elif os.environ.get("MOZ_LOW_PARALLELISM_BUILD"): 787 cpus = multiprocessing.cpu_count() 788 jobs = max(1, int(0.75 * cpus)) 789 print( 790 " Low parallelism requested: using %d jobs for %d cores" % (jobs, cpus) 791 ) 792 args.append("MOZ_PARALLEL_BUILD=%d" % jobs) 793 794 if ignore_errors: 795 args.append("-k") 796 797 if silent: 798 args.append("-s") 799 800 # Print entering/leaving directory messages. Some consumers look at 801 # these to measure progress. 802 if print_directory: 803 args.append("-w") 804 805 if keep_going: 806 args.append("-k") 807 808 if isinstance(target, list): 809 args.extend(target) 810 elif target: 811 args.append(target) 812 813 fn = self._run_command_in_objdir 814 815 if srcdir: 816 fn = self._run_command_in_srcdir 817 818 append_env = dict(append_env or ()) 819 append_env["MACH"] = "1" 820 821 params = { 822 "args": args, 823 "line_handler": line_handler, 824 "append_env": append_env, 825 "explicit_env": explicit_env, 826 "log_level": logging.INFO, 827 "require_unix_environment": False, 828 "ensure_exit_code": ensure_exit_code, 829 "pass_thru": pass_thru, 830 # Make manages its children, so mozprocess doesn't need to bother. 831 # Having mozprocess manage children can also have side-effects when 832 # building on Windows. See bug 796840. 833 "ignore_children": True, 834 } 835 836 if log: 837 params["log_name"] = "make" 838 839 return fn(**params) 840 841 def _run_command_in_srcdir(self, **args): 842 return self.run_process(cwd=self.topsrcdir, **args) 843 844 def _run_command_in_objdir(self, **args): 845 return self.run_process(cwd=self.topobjdir, **args) 846 847 def _is_windows(self): 848 return os.name in ("nt", "ce") 849 850 def _is_osx(self): 851 return "darwin" in str(sys.platform).lower() 852 853 def _spawn(self, cls): 854 """Create a new MozbuildObject-derived class instance from ourselves. 855 856 This is used as a convenience method to create other 857 MozbuildObject-derived class instances. It can only be used on 858 classes that have the same constructor arguments as us. 859 """ 860 861 return cls( 862 self.topsrcdir, self.settings, self.log_manager, topobjdir=self.topobjdir 863 ) 864 865 def activate_virtualenv(self): 866 self.virtualenv_manager.ensure() 867 self.virtualenv_manager.activate() 868 869 def _set_log_level(self, verbose): 870 self.log_manager.terminal_handler.setLevel( 871 logging.INFO if not verbose else logging.DEBUG 872 ) 873 874 def _ensure_zstd(self): 875 try: 876 import zstandard # noqa: F401 877 except (ImportError, AttributeError): 878 self.activate_virtualenv() 879 self.virtualenv_manager.install_pip_requirements( 880 os.path.join(self.topsrcdir, "build", "zstandard_requirements.txt") 881 ) 882 883 884class MachCommandBase(MozbuildObject): 885 """Base class for mach command providers that wish to be MozbuildObjects. 886 887 This provides a level of indirection so MozbuildObject can be refactored 888 without having to change everything that inherits from it. 889 """ 890 891 def __init__(self, context, virtualenv_name=None, metrics=None): 892 # Attempt to discover topobjdir through environment detection, as it is 893 # more reliable than mozconfig when cwd is inside an objdir. 894 topsrcdir = context.topdir 895 topobjdir = None 896 detect_virtualenv_mozinfo = True 897 if hasattr(context, "detect_virtualenv_mozinfo"): 898 detect_virtualenv_mozinfo = getattr(context, "detect_virtualenv_mozinfo") 899 try: 900 dummy = MozbuildObject.from_environment( 901 cwd=context.cwd, detect_virtualenv_mozinfo=detect_virtualenv_mozinfo 902 ) 903 topsrcdir = dummy.topsrcdir 904 topobjdir = dummy._topobjdir 905 if topobjdir: 906 # If we're inside a objdir and the found mozconfig resolves to 907 # another objdir, we abort. The reasoning here is that if you 908 # are inside an objdir you probably want to perform actions on 909 # that objdir, not another one. This prevents accidental usage 910 # of the wrong objdir when the current objdir is ambiguous. 911 config_topobjdir = dummy.resolve_mozconfig_topobjdir() 912 913 if config_topobjdir and not samepath(topobjdir, config_topobjdir): 914 raise ObjdirMismatchException(topobjdir, config_topobjdir) 915 except BuildEnvironmentNotFoundException: 916 pass 917 except ObjdirMismatchException as e: 918 print( 919 "Ambiguous object directory detected. We detected that " 920 "both %s and %s could be object directories. This is " 921 "typically caused by having a mozconfig pointing to a " 922 "different object directory from the current working " 923 "directory. To solve this problem, ensure you do not have a " 924 "default mozconfig in searched paths." % (e.objdir1, e.objdir2) 925 ) 926 sys.exit(1) 927 928 except MozconfigLoadException as e: 929 print(e) 930 sys.exit(1) 931 932 MozbuildObject.__init__( 933 self, 934 topsrcdir, 935 context.settings, 936 context.log_manager, 937 topobjdir=topobjdir, 938 virtualenv_name=virtualenv_name, 939 ) 940 941 self._mach_context = context 942 self.metrics = metrics 943 944 # Incur mozconfig processing so we have unified error handling for 945 # errors. Otherwise, the exceptions could bubble back to mach's error 946 # handler. 947 try: 948 self.mozconfig 949 950 except MozconfigFindException as e: 951 print(e) 952 sys.exit(1) 953 954 except MozconfigLoadException as e: 955 print(e) 956 sys.exit(1) 957 958 # Always keep a log of the last command, but don't do that for mach 959 # invokations from scripts (especially not the ones done by the build 960 # system itself). 961 try: 962 fileno = getattr(sys.stdout, "fileno", lambda: None)() 963 except io.UnsupportedOperation: 964 fileno = None 965 if fileno and os.isatty(fileno) and not getattr(self, "NO_AUTO_LOG", False): 966 self._ensure_state_subdir_exists(".") 967 logfile = self._get_state_filename("last_log.json") 968 try: 969 fd = open(logfile, "wt") 970 self.log_manager.add_json_handler(fd) 971 except Exception as e: 972 self.log( 973 logging.WARNING, 974 "mach", 975 {"error": str(e)}, 976 "Log will not be kept for this command: {error}.", 977 ) 978 979 def _sub_mach(self, argv): 980 return subprocess.call( 981 [sys.executable, os.path.join(self.topsrcdir, "mach")] + argv 982 ) 983 984 985class MachCommandConditions(object): 986 """A series of commonly used condition functions which can be applied to 987 mach commands with providers deriving from MachCommandBase. 988 """ 989 990 @staticmethod 991 def is_firefox(cls): 992 """Must have a Firefox build.""" 993 if hasattr(cls, "substs"): 994 return cls.substs.get("MOZ_BUILD_APP") == "browser" 995 return False 996 997 @staticmethod 998 def is_jsshell(cls): 999 """Must have a jsshell build.""" 1000 if hasattr(cls, "substs"): 1001 return cls.substs.get("MOZ_BUILD_APP") == "js" 1002 return False 1003 1004 @staticmethod 1005 def is_thunderbird(cls): 1006 """Must have a Thunderbird build.""" 1007 if hasattr(cls, "substs"): 1008 return cls.substs.get("MOZ_BUILD_APP") == "comm/mail" 1009 return False 1010 1011 @staticmethod 1012 def is_firefox_or_thunderbird(cls): 1013 """Must have a Firefox or Thunderbird build.""" 1014 return MachCommandConditions.is_firefox( 1015 cls 1016 ) or MachCommandConditions.is_thunderbird(cls) 1017 1018 @staticmethod 1019 def is_android(cls): 1020 """Must have an Android build.""" 1021 if hasattr(cls, "substs"): 1022 return cls.substs.get("MOZ_WIDGET_TOOLKIT") == "android" 1023 return False 1024 1025 @staticmethod 1026 def is_not_android(cls): 1027 """Must not have an Android build.""" 1028 if hasattr(cls, "substs"): 1029 return cls.substs.get("MOZ_WIDGET_TOOLKIT") != "android" 1030 return False 1031 1032 @staticmethod 1033 def is_firefox_or_android(cls): 1034 """Must have a Firefox or Android build.""" 1035 return MachCommandConditions.is_firefox( 1036 cls 1037 ) or MachCommandConditions.is_android(cls) 1038 1039 @staticmethod 1040 def has_build(cls): 1041 """Must have a build.""" 1042 return MachCommandConditions.is_firefox_or_android( 1043 cls 1044 ) or MachCommandConditions.is_thunderbird(cls) 1045 1046 @staticmethod 1047 def has_build_or_shell(cls): 1048 """Must have a build or a shell build.""" 1049 return MachCommandConditions.has_build(cls) or MachCommandConditions.is_jsshell( 1050 cls 1051 ) 1052 1053 @staticmethod 1054 def is_hg(cls): 1055 """Must have a mercurial source checkout.""" 1056 try: 1057 return isinstance(cls.repository, HgRepository) 1058 except InvalidRepoPath: 1059 return False 1060 1061 @staticmethod 1062 def is_git(cls): 1063 """Must have a git source checkout.""" 1064 try: 1065 return isinstance(cls.repository, GitRepository) 1066 except InvalidRepoPath: 1067 return False 1068 1069 @staticmethod 1070 def is_artifact_build(cls): 1071 """Must be an artifact build.""" 1072 if hasattr(cls, "substs"): 1073 return getattr(cls, "substs", {}).get("MOZ_ARTIFACT_BUILDS") 1074 return False 1075 1076 @staticmethod 1077 def is_non_artifact_build(cls): 1078 """Must not be an artifact build.""" 1079 if hasattr(cls, "substs"): 1080 return not MachCommandConditions.is_artifact_build(cls) 1081 return False 1082 1083 @staticmethod 1084 def is_buildapp_in(cls, apps): 1085 """Must have a build for one of the given app""" 1086 for app in apps: 1087 attr = getattr(MachCommandConditions, "is_{}".format(app), None) 1088 if attr and attr(cls): 1089 return True 1090 return False 1091 1092 1093class PathArgument(object): 1094 """Parse a filesystem path argument and transform it in various ways.""" 1095 1096 def __init__(self, arg, topsrcdir, topobjdir, cwd=None): 1097 self.arg = arg 1098 self.topsrcdir = topsrcdir 1099 self.topobjdir = topobjdir 1100 self.cwd = os.getcwd() if cwd is None else cwd 1101 1102 def relpath(self): 1103 """Return a path relative to the topsrcdir or topobjdir. 1104 1105 If the argument is a path to a location in one of the base directories 1106 (topsrcdir or topobjdir), then strip off the base directory part and 1107 just return the path within the base directory.""" 1108 1109 abspath = os.path.abspath(os.path.join(self.cwd, self.arg)) 1110 1111 # If that path is within topsrcdir or topobjdir, return an equivalent 1112 # path relative to that base directory. 1113 for base_dir in [self.topobjdir, self.topsrcdir]: 1114 if abspath.startswith(os.path.abspath(base_dir)): 1115 return mozpath.relpath(abspath, base_dir) 1116 1117 return mozpath.normsep(self.arg) 1118 1119 def srcdir_path(self): 1120 return mozpath.join(self.topsrcdir, self.relpath()) 1121 1122 def objdir_path(self): 1123 return mozpath.join(self.topobjdir, self.relpath()) 1124 1125 1126class ExecutionSummary(dict): 1127 """Helper for execution summaries.""" 1128 1129 def __init__(self, summary_format, **data): 1130 self._summary_format = "" 1131 assert "execution_time" in data 1132 self.extend(summary_format, **data) 1133 1134 def extend(self, summary_format, **data): 1135 self._summary_format += summary_format 1136 self.update(data) 1137 1138 def __str__(self): 1139 return self._summary_format.format(**self) 1140 1141 def __getattr__(self, key): 1142 return self[key] 1143