1# 2# Copyright (C) 2016-2018 Codethink Limited 3# 4# This program is free software; you can redistribute it and/or 5# modify it under the terms of the GNU Lesser General Public 6# License as published by the Free Software Foundation; either 7# version 2 of the License, or (at your option) any later version. 8# 9# This library is distributed in the hope that it will be useful, 10# but WITHOUT ANY WARRANTY; without even the implied warranty of 11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12# Lesser General Public License for more details. 13# 14# You should have received a copy of the GNU Lesser General Public 15# License along with this library. If not, see <http://www.gnu.org/licenses/>. 16# 17# Authors: 18# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> 19 20""" 21Element - Base element class 22============================ 23 24 25.. _core_element_abstract_methods: 26 27Abstract Methods 28---------------- 29For loading and configuration purposes, Elements must implement the 30:ref:`Plugin base class abstract methods <core_plugin_abstract_methods>`. 31 32 33.. _core_element_build_phase: 34 35Build Phase 36~~~~~~~~~~~ 37The following methods are the foundation of the element's *build 38phase*, they must be implemented by all Element classes, unless 39explicitly stated otherwise. 40 41* :func:`Element.configure_sandbox() <buildstream.element.Element.configure_sandbox>` 42 43 Configures the :class:`.Sandbox`. This is called before anything else 44 45* :func:`Element.stage() <buildstream.element.Element.stage>` 46 47 Stage dependencies and :class:`Sources <buildstream.source.Source>` into 48 the sandbox. 49 50* :func:`Element.prepare() <buildstream.element.Element.prepare>` 51 52 Call preparation methods that should only be performed once in the 53 lifetime of a build directory (e.g. autotools' ./configure). 54 55 **Optional**: If left unimplemented, this step will be skipped. 56 57* :func:`Element.assemble() <buildstream.element.Element.assemble>` 58 59 Perform the actual assembly of the element 60 61 62Miscellaneous 63~~~~~~~~~~~~~ 64Miscellaneous abstract methods also exist: 65 66* :func:`Element.generate_script() <buildstream.element.Element.generate_script>` 67 68 For the purpose of ``bst source bundle``, an Element may optionally implement this. 69 70 71Class Reference 72--------------- 73""" 74 75import os 76import re 77import stat 78import copy 79from collections import Mapping, OrderedDict 80from contextlib import contextmanager 81from enum import Enum 82import tempfile 83import time 84import shutil 85 86from . import _yaml 87from ._variables import Variables 88from ._versions import BST_CORE_ARTIFACT_VERSION 89from ._exceptions import BstError, LoadError, LoadErrorReason, ImplError, ErrorDomain 90from .utils import UtilError 91from .types import _UniquePriorityQueue 92from . import Plugin, Consistency 93from . import SandboxFlags 94from . import utils 95from . import _cachekey 96from . import _signals 97from . import _site 98from ._platform import Platform 99from .sandbox._config import SandboxConfig 100 101 102# _KeyStrength(): 103# 104# Strength of cache key 105# 106class _KeyStrength(Enum): 107 108 # Includes strong cache keys of all build dependencies and their 109 # runtime dependencies. 110 STRONG = 1 111 112 # Includes names of direct build dependencies but does not include 113 # cache keys of dependencies. 114 WEAK = 2 115 116 117class Scope(Enum): 118 """Types of scope for a given element""" 119 120 ALL = 1 121 """All elements which the given element depends on, following 122 all elements required for building. Including the element itself. 123 """ 124 125 BUILD = 2 126 """All elements required for building the element, including their 127 respective run dependencies. Not including the given element itself. 128 """ 129 130 RUN = 3 131 """All elements required for running the element. Including the element 132 itself. 133 """ 134 135 136class ElementError(BstError): 137 """This exception should be raised by :class:`.Element` implementations 138 to report errors to the user. 139 140 Args: 141 message (str): The error message to report to the user 142 detail (str): A possibly multiline, more detailed error message 143 reason (str): An optional machine readable reason string, used for test cases 144 temporary (bool): An indicator to whether the error may occur if the operation was run again. (*Since: 1.2*) 145 """ 146 def __init__(self, message, *, detail=None, reason=None, temporary=False): 147 super().__init__(message, detail=detail, domain=ErrorDomain.ELEMENT, reason=reason, temporary=temporary) 148 149 150class Element(Plugin): 151 """Element() 152 153 Base Element class. 154 155 All elements derive from this class, this interface defines how 156 the core will be interacting with Elements. 157 """ 158 __defaults = {} # The defaults from the yaml file and project 159 __defaults_set = False # Flag, in case there are no defaults at all 160 __instantiated_elements = {} # A hash of Element by MetaElement 161 __redundant_source_refs = [] # A list of (source, ref) tuples which were redundantly specified 162 163 BST_ARTIFACT_VERSION = 0 164 """The element plugin's artifact version 165 166 Elements must first set this to 1 if they change their unique key 167 structure in a way that would produce a different key for the 168 same input, or introduce a change in the build output for the 169 same unique key. Further changes of this nature require bumping the 170 artifact version. 171 """ 172 173 BST_STRICT_REBUILD = False 174 """Whether to rebuild this element in non strict mode if 175 any of the dependencies have changed. 176 """ 177 178 BST_FORBID_RDEPENDS = False 179 """Whether to raise exceptions if an element has runtime dependencies. 180 181 *Since: 1.2* 182 """ 183 184 BST_FORBID_BDEPENDS = False 185 """Whether to raise exceptions if an element has build dependencies. 186 187 *Since: 1.2* 188 """ 189 190 BST_FORBID_SOURCES = False 191 """Whether to raise exceptions if an element has sources. 192 193 *Since: 1.2* 194 """ 195 196 def __init__(self, context, project, artifacts, meta, plugin_conf): 197 198 self.__cache_key_dict = None # Dict for cache key calculation 199 self.__cache_key = None # Our cached cache key 200 201 super().__init__(meta.name, context, project, meta.provenance, "element") 202 203 self.__is_junction = meta.kind == "junction" 204 205 if not self.__is_junction: 206 project.ensure_fully_loaded() 207 208 self.normal_name = os.path.splitext(self.name.replace(os.sep, '-'))[0] 209 """A normalized element name 210 211 This is the original element without path separators or 212 the extension, it's used mainly for composing log file names 213 and creating directory names and such. 214 """ 215 216 self.__runtime_dependencies = [] # Direct runtime dependency Elements 217 self.__build_dependencies = [] # Direct build dependency Elements 218 self.__reverse_dependencies = set() # Direct reverse dependency Elements 219 self.__ready_for_runtime = False # Wether the element has all its dependencies ready and has a cache key 220 self.__sources = [] # List of Sources 221 self.__weak_cache_key = None # Our cached weak cache key 222 self.__strict_cache_key = None # Our cached cache key for strict builds 223 self.__artifacts = artifacts # Artifact cache 224 self.__consistency = Consistency.INCONSISTENT # Cached overall consistency state 225 self.__cached = None # Whether we have a cached artifact 226 self.__strong_cached = None # Whether we have a cached artifact 227 self.__assemble_scheduled = False # Element is scheduled to be assembled 228 self.__assemble_done = False # Element is assembled 229 self.__tracking_scheduled = False # Sources are scheduled to be tracked 230 self.__tracking_done = False # Sources have been tracked 231 self.__pull_done = False # Whether pull was attempted 232 self.__splits = None # Resolved regex objects for computing split domains 233 self.__whitelist_regex = None # Resolved regex object to check if file is allowed to overlap 234 self.__staged_sources_directory = None # Location where Element.stage_sources() was called 235 self.__tainted = None # Whether the artifact is tainted and should not be shared 236 self.__required = False # Whether the artifact is required in the current session 237 238 # hash tables of loaded artifact metadata, hashed by key 239 self.__metadata_keys = {} # Strong and weak keys for this key 240 self.__metadata_dependencies = {} # Dictionary of dependency strong keys 241 self.__metadata_workspaced = {} # Boolean of whether it's workspaced 242 self.__metadata_workspaced_dependencies = {} # List of which dependencies are workspaced 243 244 # Ensure we have loaded this class's defaults 245 self.__init_defaults(plugin_conf) 246 247 # Collect the composited variables and resolve them 248 variables = self.__extract_variables(meta) 249 variables['element-name'] = self.name 250 self.__variables = Variables(variables) 251 252 # Collect the composited environment now that we have variables 253 env = self.__extract_environment(meta) 254 self.__environment = env 255 256 # Collect the environment nocache blacklist list 257 nocache = self.__extract_env_nocache(meta) 258 self.__env_nocache = nocache 259 260 # Grab public domain data declared for this instance 261 self.__public = self.__extract_public(meta) 262 self.__dynamic_public = None 263 264 # Collect the composited element configuration and 265 # ask the element to configure itself. 266 self.__config = self.__extract_config(meta) 267 self._configure(self.__config) 268 269 # Extract Sandbox config 270 self.__sandbox_config = self.__extract_sandbox_config(meta) 271 272 # Extract Sandbox config 273 self.__sandbox_config = self.__extract_sandbox_config(meta) 274 275 self.__sandbox_config_supported = True 276 platform = Platform.get_platform() 277 if not platform.check_sandbox_config(self.__sandbox_config): 278 # Local sandbox does not fully support specified sandbox config. 279 # This will taint the artifact, disable pushing. 280 self.__sandbox_config_supported = False 281 282 def __lt__(self, other): 283 return self.name < other.name 284 285 ############################################################# 286 # Abstract Methods # 287 ############################################################# 288 def configure_sandbox(self, sandbox): 289 """Configures the the sandbox for execution 290 291 Args: 292 sandbox (:class:`.Sandbox`): The build sandbox 293 294 Raises: 295 (:class:`.ElementError`): When the element raises an error 296 297 Elements must implement this method to configure the sandbox object 298 for execution. 299 """ 300 raise ImplError("element plugin '{kind}' does not implement configure_sandbox()".format( 301 kind=self.get_kind())) 302 303 def stage(self, sandbox): 304 """Stage inputs into the sandbox directories 305 306 Args: 307 sandbox (:class:`.Sandbox`): The build sandbox 308 309 Raises: 310 (:class:`.ElementError`): When the element raises an error 311 312 Elements must implement this method to populate the sandbox 313 directory with data. This is done either by staging :class:`.Source` 314 objects, by staging the artifacts of the elements this element depends 315 on, or both. 316 """ 317 raise ImplError("element plugin '{kind}' does not implement stage()".format( 318 kind=self.get_kind())) 319 320 def prepare(self, sandbox): 321 """Run one-off preparation commands. 322 323 This is run before assemble(), but is guaranteed to run only 324 the first time if we build incrementally - this makes it 325 possible to run configure-like commands without causing the 326 entire element to rebuild. 327 328 Args: 329 sandbox (:class:`.Sandbox`): The build sandbox 330 331 Raises: 332 (:class:`.ElementError`): When the element raises an error 333 334 By default, this method does nothing, but may be overriden to 335 allow configure-like commands. 336 337 *Since: 1.2* 338 """ 339 pass 340 341 def assemble(self, sandbox): 342 """Assemble the output artifact 343 344 Args: 345 sandbox (:class:`.Sandbox`): The build sandbox 346 347 Returns: 348 (str): An absolute path within the sandbox to collect the artifact from 349 350 Raises: 351 (:class:`.ElementError`): When the element raises an error 352 353 Elements must implement this method to create an output 354 artifact from its sources and dependencies. 355 """ 356 raise ImplError("element plugin '{kind}' does not implement assemble()".format( 357 kind=self.get_kind())) 358 359 def generate_script(self): 360 """Generate a build (sh) script to build this element 361 362 Returns: 363 (str): A string containing the shell commands required to build the element 364 365 BuildStream guarantees the following environment when the 366 generated script is run: 367 368 - All element variables have been exported. 369 - The cwd is `self.get_variable('build_root')/self.normal_name`. 370 - $PREFIX is set to `self.get_variable('install_root')`. 371 - The directory indicated by $PREFIX is an empty directory. 372 373 Files are expected to be installed to $PREFIX. 374 375 If the script fails, it is expected to return with an exit 376 code != 0. 377 """ 378 raise ImplError("element plugin '{kind}' does not implement write_script()".format( 379 kind=self.get_kind())) 380 381 ############################################################# 382 # Public Methods # 383 ############################################################# 384 def sources(self): 385 """A generator function to enumerate the element sources 386 387 Yields: 388 (:class:`.Source`): The sources of this element 389 """ 390 for source in self.__sources: 391 yield source 392 393 def dependencies(self, scope, *, recurse=True, visited=None, recursed=False): 394 """dependencies(scope, *, recurse=True) 395 396 A generator function which yields the dependencies of the given element. 397 398 If `recurse` is specified (the default), the full dependencies will be listed 399 in deterministic staging order, starting with the basemost elements in the 400 given `scope`. Otherwise, if `recurse` is not specified then only the direct 401 dependencies in the given `scope` will be traversed, and the element itself 402 will be omitted. 403 404 Args: 405 scope (:class:`.Scope`): The scope to iterate in 406 recurse (bool): Whether to recurse 407 408 Yields: 409 (:class:`.Element`): The dependencies in `scope`, in deterministic staging order 410 """ 411 if visited is None: 412 visited = {} 413 414 full_name = self._get_full_name() 415 416 scope_set = set((Scope.BUILD, Scope.RUN)) if scope == Scope.ALL else set((scope,)) 417 418 if full_name in visited and scope_set.issubset(visited[full_name]): 419 return 420 421 should_yield = False 422 if full_name not in visited: 423 visited[full_name] = scope_set 424 should_yield = True 425 else: 426 visited[full_name] |= scope_set 427 428 if recurse or not recursed: 429 if scope == Scope.ALL: 430 for dep in self.__build_dependencies: 431 yield from dep.dependencies(Scope.ALL, recurse=recurse, 432 visited=visited, recursed=True) 433 434 for dep in self.__runtime_dependencies: 435 if dep not in self.__build_dependencies: 436 yield from dep.dependencies(Scope.ALL, recurse=recurse, 437 visited=visited, recursed=True) 438 439 elif scope == Scope.BUILD: 440 for dep in self.__build_dependencies: 441 yield from dep.dependencies(Scope.RUN, recurse=recurse, 442 visited=visited, recursed=True) 443 444 elif scope == Scope.RUN: 445 for dep in self.__runtime_dependencies: 446 yield from dep.dependencies(Scope.RUN, recurse=recurse, 447 visited=visited, recursed=True) 448 449 # Yeild self only at the end, after anything needed has been traversed 450 if should_yield and (recurse or recursed) and scope != Scope.BUILD: 451 yield self 452 453 def search(self, scope, name): 454 """Search for a dependency by name 455 456 Args: 457 scope (:class:`.Scope`): The scope to search 458 name (str): The dependency to search for 459 460 Returns: 461 (:class:`.Element`): The dependency element, or None if not found. 462 """ 463 for dep in self.dependencies(scope): 464 if dep.name == name: 465 return dep 466 467 return None 468 469 def node_subst_member(self, node, member_name, default=utils._sentinel): 470 """Fetch the value of a string node member, substituting any variables 471 in the loaded value with the element contextual variables. 472 473 Args: 474 node (dict): A dictionary loaded from YAML 475 member_name (str): The name of the member to fetch 476 default (str): A value to return when *member_name* is not specified in *node* 477 478 Returns: 479 The value of *member_name* in *node*, otherwise *default* 480 481 Raises: 482 :class:`.LoadError`: When *member_name* is not found and no *default* was provided 483 484 This is essentially the same as :func:`~buildstream.plugin.Plugin.node_get_member` 485 except that it assumes the expected type is a string and will also perform variable 486 substitutions. 487 488 **Example:** 489 490 .. code:: python 491 492 # Expect a string 'name' in 'node', substituting any 493 # variables in the returned string 494 name = self.node_subst_member(node, 'name') 495 """ 496 value = self.node_get_member(node, str, member_name, default) 497 try: 498 return self.__variables.subst(value) 499 except LoadError as e: 500 provenance = _yaml.node_get_provenance(node, key=member_name) 501 raise LoadError(e.reason, '{}: {}'.format(provenance, str(e))) from e 502 503 def node_subst_list(self, node, member_name): 504 """Fetch a list from a node member, substituting any variables in the list 505 506 Args: 507 node (dict): A dictionary loaded from YAML 508 member_name (str): The name of the member to fetch (a list) 509 510 Returns: 511 The list in *member_name* 512 513 Raises: 514 :class:`.LoadError` 515 516 This is essentially the same as :func:`~buildstream.plugin.Plugin.node_get_member` 517 except that it assumes the expected type is a list of strings and will also 518 perform variable substitutions. 519 """ 520 value = self.node_get_member(node, list, member_name) 521 ret = [] 522 for index, x in enumerate(value): 523 try: 524 ret.append(self.__variables.subst(x)) 525 except LoadError as e: 526 provenance = _yaml.node_get_provenance(node, key=member_name, indices=[index]) 527 raise LoadError(e.reason, '{}: {}'.format(provenance, str(e))) from e 528 return ret 529 530 def node_subst_list_element(self, node, member_name, indices): 531 """Fetch the value of a list element from a node member, substituting any variables 532 in the loaded value with the element contextual variables. 533 534 Args: 535 node (dict): A dictionary loaded from YAML 536 member_name (str): The name of the member to fetch 537 indices (list of int): List of indices to search, in case of nested lists 538 539 Returns: 540 The value of the list element in *member_name* at the specified *indices* 541 542 Raises: 543 :class:`.LoadError` 544 545 This is essentially the same as :func:`~buildstream.plugin.Plugin.node_get_list_element` 546 except that it assumes the expected type is a string and will also perform variable 547 substitutions. 548 549 **Example:** 550 551 .. code:: python 552 553 # Fetch the list itself 554 strings = self.node_get_member(node, list, 'strings') 555 556 # Iterate over the list indices 557 for i in range(len(strings)): 558 559 # Fetch the strings in this list, substituting content 560 # with our element's variables if needed 561 string = self.node_subst_list_element( 562 node, 'strings', [ i ]) 563 """ 564 value = self.node_get_list_element(node, str, member_name, indices) 565 try: 566 return self.__variables.subst(value) 567 except LoadError as e: 568 provenance = _yaml.node_get_provenance(node, key=member_name, indices=indices) 569 raise LoadError(e.reason, '{}: {}'.format(provenance, str(e))) from e 570 571 def compute_manifest(self, *, include=None, exclude=None, orphans=True): 572 """Compute and return this element's selective manifest 573 574 The manifest consists on the list of file paths in the 575 artifact. The files in the manifest are selected according to 576 `include`, `exclude` and `orphans` parameters. If `include` is 577 not specified then all files spoken for by any domain are 578 included unless explicitly excluded with an `exclude` domain. 579 580 Args: 581 include (list): An optional list of domains to include files from 582 exclude (list): An optional list of domains to exclude files from 583 orphans (bool): Whether to include files not spoken for by split domains 584 585 Yields: 586 (str): The paths of the files in manifest 587 """ 588 self.__assert_cached() 589 return self.__compute_splits(include, exclude, orphans) 590 591 def stage_artifact(self, sandbox, *, path=None, include=None, exclude=None, orphans=True, update_mtimes=None): 592 """Stage this element's output artifact in the sandbox 593 594 This will stage the files from the artifact to the sandbox at specified location. 595 The files are selected for staging according to the `include`, `exclude` and `orphans` 596 parameters; if `include` is not specified then all files spoken for by any domain 597 are included unless explicitly excluded with an `exclude` domain. 598 599 Args: 600 sandbox (:class:`.Sandbox`): The build sandbox 601 path (str): An optional sandbox relative path 602 include (list): An optional list of domains to include files from 603 exclude (list): An optional list of domains to exclude files from 604 orphans (bool): Whether to include files not spoken for by split domains 605 update_mtimes (list): An optional list of files whose mtimes to set to the current time. 606 607 Raises: 608 (:class:`.ElementError`): If the element has not yet produced an artifact. 609 610 Returns: 611 (:class:`~.utils.FileListResult`): The result describing what happened while staging 612 613 .. note:: 614 615 Directories in `dest` are replaced with files from `src`, 616 unless the existing directory in `dest` is not empty in 617 which case the path will be reported in the return value. 618 619 **Example:** 620 621 .. code:: python 622 623 # Stage the dependencies for a build of 'self' 624 for dep in self.dependencies(Scope.BUILD): 625 dep.stage_artifact(sandbox) 626 """ 627 628 if not self._cached(): 629 detail = "No artifacts have been cached yet for that element\n" + \ 630 "Try building the element first with `bst build`\n" 631 raise ElementError("No artifacts to stage", 632 detail=detail, reason="uncached-checkout-attempt") 633 634 if update_mtimes is None: 635 update_mtimes = [] 636 637 # Time to use the artifact, check once more that it's there 638 self.__assert_cached() 639 640 with self.timed_activity("Staging {}/{}".format(self.name, self._get_brief_display_key())): 641 # Get the extracted artifact 642 artifact_base, _ = self.__extract() 643 artifact = os.path.join(artifact_base, 'files') 644 645 # Hard link it into the staging area 646 # 647 basedir = sandbox.get_directory() 648 stagedir = basedir \ 649 if path is None \ 650 else os.path.join(basedir, path.lstrip(os.sep)) 651 652 files = list(self.__compute_splits(include, exclude, orphans)) 653 654 # We must not hardlink files whose mtimes we want to update 655 if update_mtimes: 656 link_files = [f for f in files if f not in update_mtimes] 657 copy_files = [f for f in files if f in update_mtimes] 658 else: 659 link_files = files 660 copy_files = [] 661 662 link_result = utils.link_files(artifact, stagedir, files=link_files, 663 report_written=True) 664 copy_result = utils.copy_files(artifact, stagedir, files=copy_files, 665 report_written=True) 666 667 cur_time = time.time() 668 669 for f in copy_result.files_written: 670 os.utime(os.path.join(stagedir, f), times=(cur_time, cur_time)) 671 672 return link_result.combine(copy_result) 673 674 def stage_dependency_artifacts(self, sandbox, scope, *, path=None, 675 include=None, exclude=None, orphans=True): 676 """Stage element dependencies in scope 677 678 This is primarily a convenience wrapper around 679 :func:`Element.stage_artifact() <buildstream.element.Element.stage_artifact>` 680 which takes care of staging all the dependencies in `scope` and issueing the 681 appropriate warnings. 682 683 Args: 684 sandbox (:class:`.Sandbox`): The build sandbox 685 scope (:class:`.Scope`): The scope to stage dependencies in 686 path (str): An optional sandbox relative path 687 include (list): An optional list of domains to include files from 688 exclude (list): An optional list of domains to exclude files from 689 orphans (bool): Whether to include files not spoken for by split domains 690 691 Raises: 692 (:class:`.ElementError`): If any of the dependencies in `scope` have not 693 yet produced artifacts, or if forbidden overlaps 694 occur. 695 """ 696 ignored = {} 697 overlaps = OrderedDict() 698 files_written = {} 699 old_dep_keys = {} 700 workspace = self._get_workspace() 701 project = self._get_project() 702 context = self._get_context() 703 704 if self.__can_build_incrementally() and workspace.last_successful: 705 706 # Try to perform an incremental build if the last successful 707 # build is still in the artifact cache 708 # 709 if self.__artifacts.contains(self, workspace.last_successful): 710 old_dep_keys = self.__get_artifact_metadata_dependencies(workspace.last_successful) 711 else: 712 # Last successful build is no longer in the artifact cache, 713 # so let's reset it and perform a full build now. 714 workspace.prepared = False 715 workspace.last_successful = None 716 717 self.info("Resetting workspace state, last successful build is no longer in the cache") 718 719 # In case we are staging in the main process 720 if utils._is_main_process(): 721 context.get_workspaces().save_config() 722 723 for dep in self.dependencies(scope): 724 # If we are workspaced, and we therefore perform an 725 # incremental build, we must ensure that we update the mtimes 726 # of any files created by our dependencies since the last 727 # successful build. 728 to_update = None 729 if workspace and old_dep_keys: 730 dep.__assert_cached() 731 732 if dep.name in old_dep_keys: 733 key_new = dep._get_cache_key() 734 key_old = old_dep_keys[dep.name] 735 736 # We only need to worry about modified and added 737 # files, since removed files will be picked up by 738 # build systems anyway. 739 to_update, _, added = self.__artifacts.diff(dep, key_old, key_new, subdir='files') 740 workspace.add_running_files(dep.name, to_update + added) 741 to_update.extend(workspace.running_files[dep.name]) 742 743 # In case we are running `bst shell`, this happens in the 744 # main process and we need to update the workspace config 745 if utils._is_main_process(): 746 context.get_workspaces().save_config() 747 748 result = dep.stage_artifact(sandbox, 749 path=path, 750 include=include, 751 exclude=exclude, 752 orphans=orphans, 753 update_mtimes=to_update) 754 if result.overwritten: 755 for overwrite in result.overwritten: 756 # Completely new overwrite 757 if overwrite not in overlaps: 758 # Find the overwritten element by checking where we've 759 # written the element before 760 for elm, contents in files_written.items(): 761 if overwrite in contents: 762 overlaps[overwrite] = [elm, dep.name] 763 else: 764 overlaps[overwrite].append(dep.name) 765 files_written[dep.name] = result.files_written 766 767 if result.ignored: 768 ignored[dep.name] = result.ignored 769 770 if overlaps: 771 overlap_error = overlap_warning = False 772 error_detail = warning_detail = "Staged files overwrite existing files in staging area:\n" 773 for f, elements in overlaps.items(): 774 overlap_error_elements = [] 775 overlap_warning_elements = [] 776 # The bottom item overlaps nothing 777 overlapping_elements = elements[1:] 778 for elm in overlapping_elements: 779 element = self.search(scope, elm) 780 if not element.__file_is_whitelisted(f): 781 if project.fail_on_overlap: 782 overlap_error_elements.append(elm) 783 overlap_error = True 784 else: 785 overlap_warning_elements.append(elm) 786 overlap_warning = True 787 788 warning_detail += _overlap_error_detail(f, overlap_warning_elements, elements) 789 error_detail += _overlap_error_detail(f, overlap_error_elements, elements) 790 791 if overlap_warning: 792 self.warn("Non-whitelisted overlaps detected", detail=warning_detail) 793 if overlap_error: 794 raise ElementError("Non-whitelisted overlaps detected and fail-on-overlaps is set", 795 detail=error_detail, reason="overlap-error") 796 797 if ignored: 798 detail = "Not staging files which would replace non-empty directories:\n" 799 for key, value in ignored.items(): 800 detail += "\nFrom {}:\n".format(key) 801 detail += " " + " ".join(["/" + f + "\n" for f in value]) 802 self.warn("Ignored files", detail=detail) 803 804 def integrate(self, sandbox): 805 """Integrate currently staged filesystem against this artifact. 806 807 Args: 808 sandbox (:class:`.Sandbox`): The build sandbox 809 810 This modifies the sysroot staged inside the sandbox so that 811 the sysroot is *integrated*. Only an *integrated* sandbox 812 may be trusted for running the software therein, as the integration 813 commands will create and update important system cache files 814 required for running the installed software (such as the ld.so.cache). 815 """ 816 bstdata = self.get_public_data('bst') 817 environment = self.get_environment() 818 819 if bstdata is not None: 820 commands = self.node_get_member(bstdata, list, 'integration-commands', []) 821 for i in range(len(commands)): 822 cmd = self.node_subst_list_element(bstdata, 'integration-commands', [i]) 823 self.status("Running integration command", detail=cmd) 824 exitcode = sandbox.run(['sh', '-e', '-c', cmd], 0, env=environment, cwd='/') 825 if exitcode != 0: 826 raise ElementError("Command '{}' failed with exitcode {}".format(cmd, exitcode)) 827 828 def stage_sources(self, sandbox, directory): 829 """Stage this element's sources to a directory in the sandbox 830 831 Args: 832 sandbox (:class:`.Sandbox`): The build sandbox 833 directory (str): An absolute path within the sandbox to stage the sources at 834 """ 835 836 # Hold on to the location where a plugin decided to stage sources, 837 # this will be used to reconstruct the failed sysroot properly 838 # after a failed build. 839 # 840 assert self.__staged_sources_directory is None 841 self.__staged_sources_directory = directory 842 843 self._stage_sources_in_sandbox(sandbox, directory) 844 845 def get_public_data(self, domain): 846 """Fetch public data on this element 847 848 Args: 849 domain (str): A public domain name to fetch data for 850 851 Returns: 852 (dict): The public data dictionary for the given domain 853 854 .. note:: 855 856 This can only be called the abstract methods which are 857 called as a part of the :ref:`build phase <core_element_build_phase>` 858 and never before. 859 """ 860 if self.__dynamic_public is None: 861 self.__load_public_data() 862 863 data = self.__dynamic_public.get(domain) 864 if data is not None: 865 data = _yaml.node_copy(data) 866 867 return data 868 869 def set_public_data(self, domain, data): 870 """Set public data on this element 871 872 Args: 873 domain (str): A public domain name to fetch data for 874 data (dict): The public data dictionary for the given domain 875 876 This allows an element to dynamically mutate public data of 877 elements or add new domains as the result of success completion 878 of the :func:`Element.assemble() <buildstream.element.Element.assemble>` 879 method. 880 """ 881 if self.__dynamic_public is None: 882 self.__load_public_data() 883 884 if data is not None: 885 data = _yaml.node_copy(data) 886 887 self.__dynamic_public[domain] = data 888 889 def get_environment(self): 890 """Fetch the environment suitable for running in the sandbox 891 892 Returns: 893 (dict): A dictionary of string key/values suitable for passing 894 to :func:`Sandbox.run() <buildstream.sandbox.Sandbox.run>` 895 """ 896 return _yaml.node_sanitize(self.__environment) 897 898 def get_variable(self, varname): 899 """Fetch the value of a variable resolved for this element. 900 901 Args: 902 varname (str): The name of the variable to fetch 903 904 Returns: 905 (str): The resolved value for *varname*, or None if no 906 variable was declared with the given name. 907 """ 908 if varname in self.__variables.variables: 909 return self.__variables.variables[varname] 910 911 return None 912 913 ############################################################# 914 # Private Methods used in BuildStream # 915 ############################################################# 916 917 # _new_from_meta(): 918 # 919 # Recursively instantiate a new Element instance, it's sources 920 # and it's dependencies from a meta element. 921 # 922 # Args: 923 # artifacts (ArtifactCache): The artifact cache 924 # meta (MetaElement): The meta element 925 # 926 # Returns: 927 # (Element): A newly created Element instance 928 # 929 @classmethod 930 def _new_from_meta(cls, meta, artifacts): 931 932 if not meta.first_pass: 933 meta.project.ensure_fully_loaded() 934 935 if meta in cls.__instantiated_elements: 936 return cls.__instantiated_elements[meta] 937 938 element = meta.project.create_element(artifacts, meta, first_pass=meta.first_pass) 939 cls.__instantiated_elements[meta] = element 940 941 # Instantiate sources 942 for meta_source in meta.sources: 943 meta_source.first_pass = meta.kind == "junction" 944 source = meta.project.create_source(meta_source, 945 first_pass=meta.first_pass) 946 redundant_ref = source._load_ref() 947 element.__sources.append(source) 948 949 # Collect redundant refs which occurred at load time 950 if redundant_ref is not None: 951 cls.__redundant_source_refs.append((source, redundant_ref)) 952 953 # Instantiate dependencies 954 for meta_dep in meta.dependencies: 955 dependency = Element._new_from_meta(meta_dep, artifacts) 956 element.__runtime_dependencies.append(dependency) 957 dependency.__reverse_dependencies.add(element) 958 959 for meta_dep in meta.build_dependencies: 960 dependency = Element._new_from_meta(meta_dep, artifacts) 961 element.__build_dependencies.append(dependency) 962 dependency.__reverse_dependencies.add(element) 963 964 return element 965 966 # _get_redundant_source_refs() 967 # 968 # Fetches a list of (Source, ref) tuples of all the Sources 969 # which were loaded with a ref specified in the element declaration 970 # for projects which use project.refs ref-storage. 971 # 972 # This is used to produce a warning 973 @classmethod 974 def _get_redundant_source_refs(cls): 975 return cls.__redundant_source_refs 976 977 # _reset_load_state() 978 # 979 # This is called by Pipeline.cleanup() and is used to 980 # reset the loader state between multiple sessions. 981 # 982 @classmethod 983 def _reset_load_state(cls): 984 cls.__instantiated_elements = {} 985 cls.__redundant_source_refs = [] 986 987 # _get_consistency() 988 # 989 # Returns cached consistency state 990 # 991 def _get_consistency(self): 992 return self.__consistency 993 994 # _cached(): 995 # 996 # Returns: 997 # (bool): Whether this element is already present in 998 # the artifact cache 999 # 1000 def _cached(self): 1001 return self.__cached 1002 1003 # _buildable(): 1004 # 1005 # Returns: 1006 # (bool): Whether this element can currently be built 1007 # 1008 def _buildable(self): 1009 if self._get_consistency() != Consistency.CACHED: 1010 return False 1011 1012 for dependency in self.dependencies(Scope.BUILD): 1013 # In non-strict mode an element's strong cache key may not be available yet 1014 # even though an artifact is available in the local cache. This can happen 1015 # if the pull job is still pending as the remote cache may have an artifact 1016 # that matches the strict cache key, which is preferred over a locally 1017 # cached artifact with a weak cache key match. 1018 if not dependency._cached() or not dependency._get_cache_key(strength=_KeyStrength.STRONG): 1019 return False 1020 1021 if not self.__assemble_scheduled: 1022 return False 1023 1024 return True 1025 1026 # _get_cache_key(): 1027 # 1028 # Returns the cache key 1029 # 1030 # Args: 1031 # strength (_KeyStrength): Either STRONG or WEAK key strength 1032 # 1033 # Returns: 1034 # (str): A hex digest cache key for this Element, or None 1035 # 1036 # None is returned if information for the cache key is missing. 1037 # 1038 def _get_cache_key(self, strength=_KeyStrength.STRONG): 1039 if strength == _KeyStrength.STRONG: 1040 return self.__cache_key 1041 else: 1042 return self.__weak_cache_key 1043 1044 # _can_query_cache(): 1045 # 1046 # Returns whether the cache key required for cache queries is available. 1047 # 1048 # Returns: 1049 # (bool): True if cache can be queried 1050 # 1051 def _can_query_cache(self): 1052 # If build has already been scheduled, we know that the element is 1053 # not cached and thus can allow cache query even if the strict cache key 1054 # is not available yet. 1055 # This special case is required for workspaced elements to prevent 1056 # them from getting blocked in the pull queue. 1057 if self.__assemble_scheduled: 1058 return True 1059 1060 # cache cannot be queried until strict cache key is available 1061 return self.__strict_cache_key is not None 1062 1063 # _update_state() 1064 # 1065 # Keep track of element state. Calculate cache keys if possible and 1066 # check whether artifacts are cached. 1067 # 1068 # This must be called whenever the state of an element may have changed. 1069 # 1070 def _update_state(self): 1071 context = self._get_context() 1072 1073 # Compute and determine consistency of sources 1074 self.__update_source_state() 1075 1076 if self._get_consistency() == Consistency.INCONSISTENT: 1077 # Tracking may still be pending 1078 return 1079 1080 if self._get_workspace() and self.__assemble_scheduled: 1081 # If we have an active workspace and are going to build, then 1082 # discard current cache key values as their correct values can only 1083 # be calculated once the build is complete 1084 self.__cache_key_dict = None 1085 self.__cache_key = None 1086 self.__weak_cache_key = None 1087 self.__strict_cache_key = None 1088 self.__strong_cached = None 1089 return 1090 1091 if self.__weak_cache_key is None: 1092 # Calculate weak cache key 1093 # Weak cache key includes names of direct build dependencies 1094 # but does not include keys of dependencies. 1095 if self.BST_STRICT_REBUILD: 1096 dependencies = [ 1097 e._get_cache_key(strength=_KeyStrength.WEAK) 1098 for e in self.dependencies(Scope.BUILD) 1099 ] 1100 else: 1101 dependencies = [ 1102 e.name for e in self.dependencies(Scope.BUILD, recurse=False) 1103 ] 1104 1105 self.__weak_cache_key = self.__calculate_cache_key(dependencies) 1106 1107 if self.__weak_cache_key is None: 1108 # Weak cache key could not be calculated yet 1109 return 1110 1111 if not context.get_strict(): 1112 # Full cache query in non-strict mode requires both the weak and 1113 # strict cache keys. However, we need to determine as early as 1114 # possible whether a build is pending to discard unstable cache keys 1115 # for workspaced elements. For this cache check the weak cache keys 1116 # are sufficient. However, don't update the `cached` attributes 1117 # until the full cache query below. 1118 cached = self.__artifacts.contains(self, self.__weak_cache_key) 1119 if (not self.__assemble_scheduled and not self.__assemble_done and 1120 not cached and not self._pull_pending()): 1121 # For uncached workspaced elements, assemble is required 1122 # even if we only need the cache key 1123 if self._is_required() or self._get_workspace(): 1124 self._schedule_assemble() 1125 return 1126 1127 if self.__strict_cache_key is None: 1128 dependencies = [ 1129 e.__strict_cache_key for e in self.dependencies(Scope.BUILD) 1130 ] 1131 self.__strict_cache_key = self.__calculate_cache_key(dependencies) 1132 1133 if self.__strict_cache_key is None: 1134 # Strict cache key could not be calculated yet 1135 return 1136 1137 # Query caches now that the weak and strict cache keys are available 1138 key_for_cache_lookup = self.__strict_cache_key if context.get_strict() else self.__weak_cache_key 1139 if not self.__cached: 1140 self.__cached = self.__artifacts.contains(self, key_for_cache_lookup) 1141 if not self.__strong_cached: 1142 self.__strong_cached = self.__artifacts.contains(self, self.__strict_cache_key) 1143 1144 if (not self.__assemble_scheduled and not self.__assemble_done and 1145 not self.__cached and not self._pull_pending()): 1146 # Workspaced sources are considered unstable if a build is pending 1147 # as the build will modify the contents of the workspace. 1148 # Determine as early as possible if a build is pending to discard 1149 # unstable cache keys. 1150 1151 # For uncached workspaced elements, assemble is required 1152 # even if we only need the cache key 1153 if self._is_required() or self._get_workspace(): 1154 self._schedule_assemble() 1155 return 1156 1157 if self.__cache_key is None: 1158 # Calculate strong cache key 1159 if context.get_strict(): 1160 self.__cache_key = self.__strict_cache_key 1161 elif self._pull_pending(): 1162 # Effective strong cache key is unknown until after the pull 1163 pass 1164 elif self._cached(): 1165 # Load the strong cache key from the artifact 1166 strong_key, _ = self.__get_artifact_metadata_keys() 1167 self.__cache_key = strong_key 1168 elif self.__assemble_scheduled or self.__assemble_done: 1169 # Artifact will or has been built, not downloaded 1170 dependencies = [ 1171 e._get_cache_key() for e in self.dependencies(Scope.BUILD) 1172 ] 1173 self.__cache_key = self.__calculate_cache_key(dependencies) 1174 1175 if self.__cache_key is None: 1176 # Strong cache key could not be calculated yet 1177 return 1178 1179 if not self.__ready_for_runtime and self.__cache_key is not None: 1180 self.__ready_for_runtime = all( 1181 dep.__ready_for_runtime for dep in self.__runtime_dependencies) 1182 1183 # _get_display_key(): 1184 # 1185 # Returns cache keys for display purposes 1186 # 1187 # Returns: 1188 # (str): A full hex digest cache key for this Element 1189 # (str): An abbreviated hex digest cache key for this Element 1190 # (bool): True if key should be shown as dim, False otherwise 1191 # 1192 # Question marks are returned if information for the cache key is missing. 1193 # 1194 def _get_display_key(self): 1195 context = self._get_context() 1196 dim_key = True 1197 1198 cache_key = self._get_cache_key() 1199 1200 if not cache_key: 1201 cache_key = "{:?<64}".format('') 1202 elif self._get_cache_key() == self.__strict_cache_key: 1203 # Strong cache key used in this session matches cache key 1204 # that would be used in strict build mode 1205 dim_key = False 1206 1207 length = min(len(cache_key), context.log_key_length) 1208 return (cache_key, cache_key[0:length], dim_key) 1209 1210 # _get_brief_display_key() 1211 # 1212 # Returns an abbreviated cache key for display purposes 1213 # 1214 # Returns: 1215 # (str): An abbreviated hex digest cache key for this Element 1216 # 1217 # Question marks are returned if information for the cache key is missing. 1218 # 1219 def _get_brief_display_key(self): 1220 _, display_key, _ = self._get_display_key() 1221 return display_key 1222 1223 # _preflight(): 1224 # 1225 # A wrapper for calling the abstract preflight() method on 1226 # the element and it's sources. 1227 # 1228 def _preflight(self): 1229 1230 if self.BST_FORBID_RDEPENDS and self.BST_FORBID_BDEPENDS: 1231 if any(self.dependencies(Scope.RUN, recurse=False)) or any(self.dependencies(Scope.BUILD, recurse=False)): 1232 raise ElementError("{}: Dependencies are forbidden for '{}' elements" 1233 .format(self, self.get_kind()), reason="element-forbidden-depends") 1234 1235 if self.BST_FORBID_RDEPENDS: 1236 if any(self.dependencies(Scope.RUN, recurse=False)): 1237 raise ElementError("{}: Runtime dependencies are forbidden for '{}' elements" 1238 .format(self, self.get_kind()), reason="element-forbidden-rdepends") 1239 1240 if self.BST_FORBID_BDEPENDS: 1241 if any(self.dependencies(Scope.BUILD, recurse=False)): 1242 raise ElementError("{}: Build dependencies are forbidden for '{}' elements" 1243 .format(self, self.get_kind()), reason="element-forbidden-bdepends") 1244 1245 if self.BST_FORBID_SOURCES: 1246 if any(self.sources()): 1247 raise ElementError("{}: Sources are forbidden for '{}' elements" 1248 .format(self, self.get_kind()), reason="element-forbidden-sources") 1249 1250 try: 1251 self.preflight() 1252 except BstError as e: 1253 # Prepend provenance to the error 1254 raise ElementError("{}: {}".format(self, e), reason=e.reason) from e 1255 1256 # Preflight the sources 1257 for source in self.sources(): 1258 source._preflight() 1259 1260 # _schedule_tracking(): 1261 # 1262 # Force an element state to be inconsistent. Any sources appear to be 1263 # inconsistent. 1264 # 1265 # This is used across the pipeline in sessions where the 1266 # elements in question are going to be tracked, causing the 1267 # pipeline to rebuild safely by ensuring cache key recalculation 1268 # and reinterrogation of element state after tracking of elements 1269 # succeeds. 1270 # 1271 def _schedule_tracking(self): 1272 self.__tracking_scheduled = True 1273 self._update_state() 1274 1275 # _tracking_done(): 1276 # 1277 # This is called in the main process after the element has been tracked 1278 # 1279 def _tracking_done(self): 1280 assert self.__tracking_scheduled 1281 1282 self.__tracking_scheduled = False 1283 self.__tracking_done = True 1284 1285 self.__update_state_recursively() 1286 1287 # _track(): 1288 # 1289 # Calls track() on the Element sources 1290 # 1291 # Raises: 1292 # SourceError: If one of the element sources has an error 1293 # 1294 # Returns: 1295 # (list): A list of Source object ids and their new references 1296 # 1297 def _track(self): 1298 refs = [] 1299 for source in self.__sources: 1300 old_ref = source.get_ref() 1301 new_ref = source._track() 1302 refs.append((source._unique_id, new_ref)) 1303 1304 # Complimentary warning that the new ref will be unused. 1305 if old_ref != new_ref and self._get_workspace(): 1306 detail = "This source has an open workspace.\n" \ 1307 + "To start using the new reference, please close the existing workspace." 1308 source.warn("Updated reference will be ignored as source has open workspace", detail=detail) 1309 1310 return refs 1311 1312 # _prepare_sandbox(): 1313 # 1314 # This stages things for either _shell() (below) or also 1315 # is used to stage things by the `bst checkout` codepath 1316 # 1317 @contextmanager 1318 def _prepare_sandbox(self, scope, directory, deps='run', integrate=True): 1319 with self.__sandbox(directory, config=self.__sandbox_config) as sandbox: 1320 1321 # Configure always comes first, and we need it. 1322 self.configure_sandbox(sandbox) 1323 1324 # Stage something if we need it 1325 if not directory: 1326 if scope == Scope.BUILD: 1327 self.stage(sandbox) 1328 elif scope == Scope.RUN: 1329 if deps == 'run': 1330 dependency_scope = Scope.RUN 1331 else: 1332 dependency_scope = None 1333 1334 # Stage deps in the sandbox root 1335 with self.timed_activity("Staging dependencies", silent_nested=True): 1336 self.stage_dependency_artifacts(sandbox, dependency_scope) 1337 1338 # Run any integration commands provided by the dependencies 1339 # once they are all staged and ready 1340 if integrate: 1341 with self.timed_activity("Integrating sandbox"): 1342 for dep in self.dependencies(dependency_scope): 1343 dep.integrate(sandbox) 1344 1345 yield sandbox 1346 1347 # _stage_sources_in_sandbox(): 1348 # 1349 # Stage this element's sources to a directory inside sandbox 1350 # 1351 # Args: 1352 # sandbox (:class:`.Sandbox`): The build sandbox 1353 # directory (str): An absolute path to stage the sources at 1354 # mount_workspaces (bool): mount workspaces if True, copy otherwise 1355 # 1356 def _stage_sources_in_sandbox(self, sandbox, directory, mount_workspaces=True): 1357 1358 # Only artifact caches that implement diff() are allowed to 1359 # perform incremental builds. 1360 if mount_workspaces and self.__can_build_incrementally(): 1361 workspace = self._get_workspace() 1362 sandbox.mark_directory(directory) 1363 sandbox._set_mount_source(directory, workspace.get_absolute_path()) 1364 1365 # Stage all sources that need to be copied 1366 sandbox_root = sandbox.get_directory() 1367 host_directory = os.path.join(sandbox_root, directory.lstrip(os.sep)) 1368 self._stage_sources_at(host_directory, mount_workspaces=mount_workspaces) 1369 1370 # _stage_sources_at(): 1371 # 1372 # Stage this element's sources to a directory 1373 # 1374 # Args: 1375 # directory (str): An absolute path to stage the sources at 1376 # mount_workspaces (bool): mount workspaces if True, copy otherwise 1377 # 1378 def _stage_sources_at(self, directory, mount_workspaces=True): 1379 with self.timed_activity("Staging sources", silent_nested=True): 1380 1381 if os.path.isdir(directory) and os.listdir(directory): 1382 raise ElementError("Staging directory '{}' is not empty".format(directory)) 1383 1384 workspace = self._get_workspace() 1385 if workspace: 1386 # If mount_workspaces is set and we're doing incremental builds, 1387 # the workspace is already mounted into the sandbox. 1388 if not (mount_workspaces and self.__can_build_incrementally()): 1389 with self.timed_activity("Staging local files at {}".format(workspace.path)): 1390 workspace.stage(directory) 1391 else: 1392 # No workspace, stage directly 1393 for source in self.sources(): 1394 source._stage(directory) 1395 1396 # Ensure deterministic mtime of sources at build time 1397 utils._set_deterministic_mtime(directory) 1398 # Ensure deterministic owners of sources at build time 1399 utils._set_deterministic_user(directory) 1400 1401 # _set_required(): 1402 # 1403 # Mark this element and its runtime dependencies as required. 1404 # This unblocks pull/fetch/build. 1405 # 1406 def _set_required(self): 1407 if self.__required: 1408 # Already done 1409 return 1410 1411 self.__required = True 1412 1413 # Request artifacts of runtime dependencies 1414 for dep in self.dependencies(Scope.RUN, recurse=False): 1415 dep._set_required() 1416 1417 self._update_state() 1418 1419 # _is_required(): 1420 # 1421 # Returns whether this element has been marked as required. 1422 # 1423 def _is_required(self): 1424 return self.__required 1425 1426 # _schedule_assemble(): 1427 # 1428 # This is called in the main process before the element is assembled 1429 # in a subprocess. 1430 # 1431 def _schedule_assemble(self): 1432 assert not self.__assemble_scheduled 1433 self.__assemble_scheduled = True 1434 1435 # Requests artifacts of build dependencies 1436 for dep in self.dependencies(Scope.BUILD, recurse=False): 1437 dep._set_required() 1438 1439 self._set_required() 1440 1441 # Invalidate workspace key as the build modifies the workspace directory 1442 workspace = self._get_workspace() 1443 if workspace: 1444 workspace.invalidate_key() 1445 1446 self._update_state() 1447 1448 # _assemble_done(): 1449 # 1450 # This is called in the main process after the element has been assembled 1451 # and in the a subprocess after assembly completes. 1452 # 1453 # This will result in updating the element state. 1454 # 1455 def _assemble_done(self): 1456 assert self.__assemble_scheduled 1457 1458 self.__assemble_scheduled = False 1459 self.__assemble_done = True 1460 1461 self.__update_state_recursively() 1462 1463 if self._get_workspace() and self._cached(): 1464 # 1465 # Note that this block can only happen in the 1466 # main process, since `self._cached()` cannot 1467 # be true when assembly is completed in the task. 1468 # 1469 # For this reason, it is safe to update and 1470 # save the workspaces configuration 1471 # 1472 key = self._get_cache_key() 1473 workspace = self._get_workspace() 1474 workspace.last_successful = key 1475 workspace.clear_running_files() 1476 self._get_context().get_workspaces().save_config() 1477 1478 # This element will have already been marked as 1479 # required, but we bump the atime again, in case 1480 # we did not know the cache key until now. 1481 # 1482 # FIXME: This is not exactly correct, we should be 1483 # doing this at the time which we have discovered 1484 # a new cache key, this just happens to be the 1485 # last place where that can happen. 1486 # 1487 # Ultimately, we should be refactoring 1488 # Element._update_state() such that we know 1489 # when a cache key is actually discovered. 1490 # 1491 self.__artifacts.mark_required_elements([self]) 1492 1493 # _assemble(): 1494 # 1495 # Internal method for running the entire build phase. 1496 # 1497 # This will: 1498 # - Prepare a sandbox for the build 1499 # - Call the public abstract methods for the build phase 1500 # - Cache the resulting artifact 1501 # 1502 # Returns: 1503 # (int): The size of the newly cached artifact 1504 # 1505 def _assemble(self): 1506 1507 # Assert call ordering 1508 assert not self._cached() 1509 1510 context = self._get_context() 1511 with self._output_file() as output_file: 1512 1513 if not self.__sandbox_config_supported: 1514 self.warn("Sandbox configuration is not supported by the platform.", 1515 detail="Falling back to UID {} GID {}. Artifact will not be pushed." 1516 .format(self.__sandbox_config.build_uid, self.__sandbox_config.build_gid)) 1517 1518 # Explicitly clean it up, keep the build dir around if exceptions are raised 1519 os.makedirs(context.builddir, exist_ok=True) 1520 rootdir = tempfile.mkdtemp(prefix="{}-".format(self.normal_name), dir=context.builddir) 1521 1522 # Cleanup the build directory on explicit SIGTERM 1523 def cleanup_rootdir(): 1524 utils._force_rmtree(rootdir) 1525 1526 with _signals.terminator(cleanup_rootdir), \ 1527 self.__sandbox(rootdir, output_file, output_file, self.__sandbox_config) as sandbox: # nopep8 1528 1529 sandbox_root = sandbox.get_directory() 1530 1531 # By default, the dynamic public data is the same as the static public data. 1532 # The plugin's assemble() method may modify this, though. 1533 self.__dynamic_public = _yaml.node_copy(self.__public) 1534 1535 # Call the abstract plugin methods 1536 try: 1537 # Step 1 - Configure 1538 self.configure_sandbox(sandbox) 1539 # Step 2 - Stage 1540 self.stage(sandbox) 1541 # Step 3 - Prepare 1542 self.__prepare(sandbox) 1543 # Step 4 - Assemble 1544 collect = self.assemble(sandbox) 1545 except BstError as e: 1546 # If an error occurred assembling an element in a sandbox, 1547 # then tack on the sandbox directory to the error 1548 e.sandbox = rootdir 1549 1550 # If there is a workspace open on this element, it will have 1551 # been mounted for sandbox invocations instead of being staged. 1552 # 1553 # In order to preserve the correct failure state, we need to 1554 # copy over the workspace files into the appropriate directory 1555 # in the sandbox. 1556 # 1557 workspace = self._get_workspace() 1558 if workspace and self.__staged_sources_directory: 1559 sandbox_root = sandbox.get_directory() 1560 sandbox_path = os.path.join(sandbox_root, 1561 self.__staged_sources_directory.lstrip(os.sep)) 1562 try: 1563 utils.copy_files(workspace.path, sandbox_path) 1564 except UtilError as e: 1565 self.warn("Failed to preserve workspace state for failed build sysroot: {}" 1566 .format(e)) 1567 1568 raise 1569 1570 collectdir = os.path.join(sandbox_root, collect.lstrip(os.sep)) 1571 if not os.path.exists(collectdir): 1572 raise ElementError( 1573 "Directory '{}' was not found inside the sandbox, " 1574 "unable to collect artifact contents" 1575 .format(collect)) 1576 1577 # At this point, we expect an exception was raised leading to 1578 # an error message, or we have good output to collect. 1579 1580 # Create artifact directory structure 1581 assembledir = os.path.join(rootdir, 'artifact') 1582 filesdir = os.path.join(assembledir, 'files') 1583 logsdir = os.path.join(assembledir, 'logs') 1584 metadir = os.path.join(assembledir, 'meta') 1585 os.mkdir(assembledir) 1586 os.mkdir(filesdir) 1587 os.mkdir(logsdir) 1588 os.mkdir(metadir) 1589 1590 # Hard link files from collect dir to files directory 1591 utils.link_files(collectdir, filesdir) 1592 1593 # Copy build log 1594 log_filename = context.get_log_filename() 1595 if log_filename: 1596 shutil.copyfile(log_filename, os.path.join(logsdir, 'build.log')) 1597 1598 # Store public data 1599 _yaml.dump(_yaml.node_sanitize(self.__dynamic_public), os.path.join(metadir, 'public.yaml')) 1600 1601 # ensure we have cache keys 1602 self._assemble_done() 1603 1604 # Store keys.yaml 1605 _yaml.dump(_yaml.node_sanitize({ 1606 'strong': self._get_cache_key(), 1607 'weak': self._get_cache_key(_KeyStrength.WEAK), 1608 }), os.path.join(metadir, 'keys.yaml')) 1609 1610 # Store dependencies.yaml 1611 _yaml.dump(_yaml.node_sanitize({ 1612 e.name: e._get_cache_key() for e in self.dependencies(Scope.BUILD) 1613 }), os.path.join(metadir, 'dependencies.yaml')) 1614 1615 # Store workspaced.yaml 1616 _yaml.dump(_yaml.node_sanitize({ 1617 'workspaced': True if self._get_workspace() else False 1618 }), os.path.join(metadir, 'workspaced.yaml')) 1619 1620 # Store workspaced-dependencies.yaml 1621 _yaml.dump(_yaml.node_sanitize({ 1622 'workspaced-dependencies': [ 1623 e.name for e in self.dependencies(Scope.BUILD) 1624 if e._get_workspace() 1625 ] 1626 }), os.path.join(metadir, 'workspaced-dependencies.yaml')) 1627 1628 with self.timed_activity("Caching artifact"): 1629 artifact_size = utils._get_dir_size(assembledir) 1630 self.__artifacts.commit(self, assembledir, self.__get_cache_keys_for_commit()) 1631 1632 # Finally cleanup the build dir 1633 cleanup_rootdir() 1634 1635 return artifact_size 1636 1637 # _fetch_done() 1638 # 1639 # Indicates that fetching the sources for this element has been done. 1640 # 1641 def _fetch_done(self): 1642 # We are not updating the state recursively here since fetching can 1643 # never end up in updating them. 1644 self._update_state() 1645 1646 # _pull_pending() 1647 # 1648 # Check whether the artifact will be pulled. 1649 # 1650 # Returns: 1651 # (bool): Whether a pull operation is pending 1652 # 1653 def _pull_pending(self): 1654 if self._get_workspace(): 1655 # Workspace builds are never pushed to artifact servers 1656 return False 1657 1658 if self.__strong_cached: 1659 # Artifact already in local cache 1660 return False 1661 1662 # Pull is pending if artifact remote server available 1663 # and pull has not been attempted yet 1664 return self.__artifacts.has_fetch_remotes(element=self) and not self.__pull_done 1665 1666 # _pull_done() 1667 # 1668 # Indicate that pull was attempted. 1669 # 1670 # This needs to be called in the main process after a pull 1671 # succeeds or fails so that we properly update the main 1672 # process data model 1673 # 1674 # This will result in updating the element state. 1675 # 1676 def _pull_done(self): 1677 self.__pull_done = True 1678 1679 self.__update_state_recursively() 1680 1681 def _pull_strong(self, *, progress=None): 1682 weak_key = self._get_cache_key(strength=_KeyStrength.WEAK) 1683 1684 key = self.__strict_cache_key 1685 if not self.__artifacts.pull(self, key, progress=progress): 1686 return False 1687 1688 # update weak ref by pointing it to this newly fetched artifact 1689 self.__artifacts.link_key(self, key, weak_key) 1690 1691 return True 1692 1693 def _pull_weak(self, *, progress=None): 1694 weak_key = self._get_cache_key(strength=_KeyStrength.WEAK) 1695 1696 if not self.__artifacts.pull(self, weak_key, progress=progress): 1697 return False 1698 1699 # extract strong cache key from this newly fetched artifact 1700 self._pull_done() 1701 1702 # create tag for strong cache key 1703 key = self._get_cache_key(strength=_KeyStrength.STRONG) 1704 self.__artifacts.link_key(self, weak_key, key) 1705 1706 return True 1707 1708 # _pull(): 1709 # 1710 # Pull artifact from remote artifact repository into local artifact cache. 1711 # 1712 # Returns: True if the artifact has been downloaded, False otherwise 1713 # 1714 def _pull(self): 1715 context = self._get_context() 1716 1717 def progress(percent, message): 1718 self.status(message) 1719 1720 # Attempt to pull artifact without knowing whether it's available 1721 pulled = self._pull_strong(progress=progress) 1722 1723 if not pulled and not self._cached() and not context.get_strict(): 1724 pulled = self._pull_weak(progress=progress) 1725 1726 if not pulled: 1727 return False 1728 1729 # Notify successfull download 1730 return True 1731 1732 # _skip_push(): 1733 # 1734 # Determine whether we should create a push job for this element. 1735 # 1736 # Returns: 1737 # (bool): True if this element does not need a push job to be created 1738 # 1739 def _skip_push(self): 1740 if not self.__artifacts.has_push_remotes(element=self): 1741 # No push remotes for this element's project 1742 return True 1743 1744 if not self._cached(): 1745 return True 1746 1747 # Do not push tained artifact 1748 if self.__get_tainted(): 1749 return True 1750 1751 return False 1752 1753 # _push(): 1754 # 1755 # Push locally cached artifact to remote artifact repository. 1756 # 1757 # Returns: 1758 # (bool): True if the remote was updated, False if it already existed 1759 # and no updated was required 1760 # 1761 def _push(self): 1762 self.__assert_cached() 1763 1764 if self.__get_tainted(): 1765 self.warn("Not pushing tainted artifact.") 1766 return False 1767 1768 # Push all keys used for local commit 1769 pushed = self.__artifacts.push(self, self.__get_cache_keys_for_commit()) 1770 if not pushed: 1771 return False 1772 1773 # Notify successful upload 1774 return True 1775 1776 # _shell(): 1777 # 1778 # Connects the terminal with a shell running in a staged 1779 # environment 1780 # 1781 # Args: 1782 # scope (Scope): Either BUILD or RUN scopes are valid, or None 1783 # directory (str): A directory to an existing sandbox, or None 1784 # mounts (list): A list of (str, str) tuples, representing host/target paths to mount 1785 # isolate (bool): Whether to isolate the environment like we do in builds 1786 # prompt (str): A suitable prompt string for PS1 1787 # command (list): An argv to launch in the sandbox 1788 # 1789 # Returns: Exit code 1790 # 1791 # If directory is not specified, one will be staged using scope 1792 def _shell(self, scope=None, directory=None, *, mounts=None, isolate=False, prompt=None, command=None): 1793 1794 with self._prepare_sandbox(scope, directory) as sandbox: 1795 environment = self.get_environment() 1796 environment = copy.copy(environment) 1797 flags = SandboxFlags.INTERACTIVE | SandboxFlags.ROOT_READ_ONLY 1798 1799 # Fetch the main toplevel project, in case this is a junctioned 1800 # subproject, we want to use the rules defined by the main one. 1801 context = self._get_context() 1802 project = context.get_toplevel_project() 1803 shell_command, shell_environment, shell_host_files = project.get_shell_config() 1804 1805 if prompt is not None: 1806 environment['PS1'] = prompt 1807 1808 # Special configurations for non-isolated sandboxes 1809 if not isolate: 1810 1811 # Open the network, and reuse calling uid/gid 1812 # 1813 flags |= SandboxFlags.NETWORK_ENABLED | SandboxFlags.INHERIT_UID 1814 1815 # Apply project defined environment vars to set for a shell 1816 for key, value in _yaml.node_items(shell_environment): 1817 environment[key] = value 1818 1819 # Setup any requested bind mounts 1820 if mounts is None: 1821 mounts = [] 1822 1823 for mount in shell_host_files + mounts: 1824 if not os.path.exists(mount.host_path): 1825 if not mount.optional: 1826 self.warn("Not mounting non-existing host file: {}".format(mount.host_path)) 1827 else: 1828 sandbox.mark_directory(mount.path) 1829 sandbox._set_mount_source(mount.path, mount.host_path) 1830 1831 if command: 1832 argv = [arg for arg in command] 1833 else: 1834 argv = shell_command 1835 1836 self.status("Running command", detail=" ".join(argv)) 1837 1838 # Run shells with network enabled and readonly root. 1839 return sandbox.run(argv, flags, env=environment) 1840 1841 # _open_workspace(): 1842 # 1843 # "Open" a workspace for this element 1844 # 1845 # This requires that a workspace already be created in 1846 # the workspaces metadata first. 1847 # 1848 def _open_workspace(self): 1849 context = self._get_context() 1850 workspace = self._get_workspace() 1851 assert workspace is not None 1852 1853 # First lets get a temp dir in our build directory 1854 # and stage there, then link the files over to the desired 1855 # path. 1856 # 1857 # We do this so that force opening workspaces which overwrites 1858 # files in the target directory actually works without any 1859 # additional support from Source implementations. 1860 # 1861 os.makedirs(context.builddir, exist_ok=True) 1862 with utils._tempdir(dir=context.builddir, prefix='workspace-{}' 1863 .format(self.normal_name)) as temp: 1864 for source in self.sources(): 1865 source._init_workspace(temp) 1866 1867 # Now hardlink the files into the workspace target. 1868 utils.link_files(temp, workspace.path) 1869 1870 # _get_workspace(): 1871 # 1872 # Returns: 1873 # (Workspace|None): A workspace associated with this element 1874 # 1875 def _get_workspace(self): 1876 workspaces = self._get_context().get_workspaces() 1877 return workspaces.get_workspace(self._get_full_name()) 1878 1879 # _write_script(): 1880 # 1881 # Writes a script to the given directory. 1882 def _write_script(self, directory): 1883 with open(_site.build_module_template, "r") as f: 1884 script_template = f.read() 1885 1886 variable_string = "" 1887 for var, val in self.get_environment().items(): 1888 variable_string += "{0}={1} ".format(var, val) 1889 1890 script = script_template.format( 1891 name=self.normal_name, 1892 build_root=self.get_variable('build-root'), 1893 install_root=self.get_variable('install-root'), 1894 variables=variable_string, 1895 commands=self.generate_script() 1896 ) 1897 1898 os.makedirs(directory, exist_ok=True) 1899 script_path = os.path.join(directory, "build-" + self.normal_name) 1900 1901 with self.timed_activity("Writing build script", silent_nested=True): 1902 with utils.save_file_atomic(script_path, "w") as script_file: 1903 script_file.write(script) 1904 1905 os.chmod(script_path, stat.S_IEXEC | stat.S_IREAD) 1906 1907 # _subst_string() 1908 # 1909 # Substitue a string, this is an internal function related 1910 # to how junctions are loaded and needs to be more generic 1911 # than the public node_subst_member() 1912 # 1913 # Args: 1914 # value (str): A string value 1915 # 1916 # Returns: 1917 # (str): The string after substitutions have occurred 1918 # 1919 def _subst_string(self, value): 1920 return self.__variables.subst(value) 1921 1922 # Returns the element whose sources this element is ultimately derived from. 1923 # 1924 # This is intended for being used to redirect commands that operate on an 1925 # element to the element whose sources it is ultimately derived from. 1926 # 1927 # For example, element A is a build element depending on source foo, 1928 # element B is a filter element that depends on element A. The source 1929 # element of B is A, since B depends on A, and A has sources. 1930 # 1931 def _get_source_element(self): 1932 return self 1933 1934 ############################################################# 1935 # Private Local Methods # 1936 ############################################################# 1937 1938 # __update_source_state() 1939 # 1940 # Updates source consistency state 1941 # 1942 def __update_source_state(self): 1943 1944 # Cannot resolve source state until tracked 1945 if self.__tracking_scheduled: 1946 return 1947 1948 self.__consistency = Consistency.CACHED 1949 workspace = self._get_workspace() 1950 1951 # Special case for workspaces 1952 if workspace: 1953 1954 # A workspace is considered inconsistent in the case 1955 # that it's directory went missing 1956 # 1957 fullpath = workspace.get_absolute_path() 1958 if not os.path.exists(fullpath): 1959 self.__consistency = Consistency.INCONSISTENT 1960 else: 1961 1962 # Determine overall consistency of the element 1963 for source in self.__sources: 1964 source._update_state() 1965 source_consistency = source._get_consistency() 1966 self.__consistency = min(self.__consistency, source_consistency) 1967 1968 # __calculate_cache_key(): 1969 # 1970 # Calculates the cache key 1971 # 1972 # Returns: 1973 # (str): A hex digest cache key for this Element, or None 1974 # 1975 # None is returned if information for the cache key is missing. 1976 # 1977 def __calculate_cache_key(self, dependencies): 1978 # No cache keys for dependencies which have no cache keys 1979 if None in dependencies: 1980 return None 1981 1982 # Generate dict that is used as base for all cache keys 1983 if self.__cache_key_dict is None: 1984 # Filter out nocache variables from the element's environment 1985 cache_env = { 1986 key: value 1987 for key, value in self.node_items(self.__environment) 1988 if key not in self.__env_nocache 1989 } 1990 1991 context = self._get_context() 1992 project = self._get_project() 1993 workspace = self._get_workspace() 1994 1995 self.__cache_key_dict = { 1996 'artifact-version': "{}.{}".format(BST_CORE_ARTIFACT_VERSION, 1997 self.BST_ARTIFACT_VERSION), 1998 'context': context.get_cache_key(), 1999 'project': project.get_cache_key(), 2000 'element': self.get_unique_key(), 2001 'execution-environment': self.__sandbox_config.get_unique_key(), 2002 'environment': cache_env, 2003 'sources': [s._get_unique_key(workspace is None) for s in self.__sources], 2004 'workspace': '' if workspace is None else workspace.get_key(self._get_project()), 2005 'public': self.__public, 2006 'cache': type(self.__artifacts).__name__ 2007 } 2008 2009 # fail-on-overlap setting cannot affect elements without dependencies 2010 if project.fail_on_overlap and dependencies: 2011 self.__cache_key_dict['fail-on-overlap'] = True 2012 2013 cache_key_dict = self.__cache_key_dict.copy() 2014 cache_key_dict['dependencies'] = dependencies 2015 2016 return _cachekey.generate_key(cache_key_dict) 2017 2018 # __can_build_incrementally() 2019 # 2020 # Check if the element can be built incrementally, this 2021 # is used to decide how to stage things 2022 # 2023 # Returns: 2024 # (bool): Whether this element can be built incrementally 2025 # 2026 def __can_build_incrementally(self): 2027 return bool(self._get_workspace()) 2028 2029 # __prepare(): 2030 # 2031 # Internal method for calling public abstract prepare() method. 2032 # 2033 def __prepare(self, sandbox): 2034 workspace = self._get_workspace() 2035 2036 # We need to ensure that the prepare() method is only called 2037 # once in workspaces, because the changes will persist across 2038 # incremental builds - not desirable, for example, in the case 2039 # of autotools' `./configure`. 2040 if not (workspace and workspace.prepared): 2041 self.prepare(sandbox) 2042 2043 if workspace: 2044 workspace.prepared = True 2045 2046 # __assert_cached() 2047 # 2048 # Raises an error if the artifact is not cached. 2049 # 2050 def __assert_cached(self): 2051 assert self._cached(), "{}: Missing artifact {}".format(self, self._get_brief_display_key()) 2052 2053 # __get_tainted(): 2054 # 2055 # Checkes whether this artifact should be pushed to an artifact cache. 2056 # 2057 # Args: 2058 # recalculate (bool) - Whether to force recalculation 2059 # 2060 # Returns: 2061 # (bool) False if this artifact should be excluded from pushing. 2062 # 2063 # Note: 2064 # This method should only be called after the element's 2065 # artifact is present in the local artifact cache. 2066 # 2067 def __get_tainted(self, recalculate=False): 2068 if recalculate or self.__tainted is None: 2069 2070 # Whether this artifact has a workspace 2071 workspaced = self.__get_artifact_metadata_workspaced() 2072 2073 # Whether this artifact's dependencies have workspaces 2074 workspaced_dependencies = self.__get_artifact_metadata_workspaced_dependencies() 2075 2076 # Other conditions should be or-ed 2077 self.__tainted = (workspaced or workspaced_dependencies or 2078 not self.__sandbox_config_supported) 2079 2080 return self.__tainted 2081 2082 # __sandbox(): 2083 # 2084 # A context manager to prepare a Sandbox object at the specified directory, 2085 # if the directory is None, then a directory will be chosen automatically 2086 # in the configured build directory. 2087 # 2088 # Args: 2089 # directory (str): The local directory where the sandbox will live, or None 2090 # stdout (fileobject): The stream for stdout for the sandbox 2091 # stderr (fileobject): The stream for stderr for the sandbox 2092 # config (SandboxConfig): The SandboxConfig object 2093 # 2094 # Yields: 2095 # (Sandbox): A usable sandbox 2096 # 2097 @contextmanager 2098 def __sandbox(self, directory, stdout=None, stderr=None, config=None): 2099 context = self._get_context() 2100 project = self._get_project() 2101 platform = Platform.get_platform() 2102 2103 if directory is not None and os.path.exists(directory): 2104 sandbox = platform.create_sandbox(context, project, 2105 directory, 2106 stdout=stdout, 2107 stderr=stderr, 2108 config=config) 2109 yield sandbox 2110 2111 else: 2112 os.makedirs(context.builddir, exist_ok=True) 2113 rootdir = tempfile.mkdtemp(prefix="{}-".format(self.normal_name), dir=context.builddir) 2114 2115 # Recursive contextmanager... 2116 with self.__sandbox(rootdir, stdout=stdout, stderr=stderr, config=config) as sandbox: 2117 yield sandbox 2118 2119 # Cleanup the build dir 2120 utils._force_rmtree(rootdir) 2121 2122 def __compose_default_splits(self, defaults): 2123 project = self._get_project() 2124 2125 element_public = _yaml.node_get(defaults, Mapping, 'public', default_value={}) 2126 element_bst = _yaml.node_get(element_public, Mapping, 'bst', default_value={}) 2127 element_splits = _yaml.node_get(element_bst, Mapping, 'split-rules', default_value={}) 2128 2129 if self.__is_junction: 2130 splits = _yaml.node_chain_copy(element_splits) 2131 else: 2132 assert project._splits is not None 2133 2134 splits = _yaml.node_chain_copy(project._splits) 2135 # Extend project wide split rules with any split rules defined by the element 2136 _yaml.composite(splits, element_splits) 2137 2138 element_bst['split-rules'] = splits 2139 element_public['bst'] = element_bst 2140 defaults['public'] = element_public 2141 2142 def __init_defaults(self, plugin_conf): 2143 2144 # Defaults are loaded once per class and then reused 2145 # 2146 if not self.__defaults_set: 2147 2148 # Load the plugin's accompanying .yaml file if one was provided 2149 defaults = {} 2150 try: 2151 defaults = _yaml.load(plugin_conf, os.path.basename(plugin_conf)) 2152 except LoadError as e: 2153 if e.reason != LoadErrorReason.MISSING_FILE: 2154 raise e 2155 2156 # Special case; compose any element-wide split-rules declarations 2157 self.__compose_default_splits(defaults) 2158 2159 # Override the element's defaults with element specific 2160 # overrides from the project.conf 2161 project = self._get_project() 2162 if self.__is_junction: 2163 elements = project.first_pass_config.element_overrides 2164 else: 2165 elements = project.element_overrides 2166 2167 overrides = elements.get(self.get_kind()) 2168 if overrides: 2169 _yaml.composite(defaults, overrides) 2170 2171 # Set the data class wide 2172 type(self).__defaults = defaults 2173 type(self).__defaults_set = True 2174 2175 # This will resolve the final environment to be used when 2176 # creating sandboxes for this element 2177 # 2178 def __extract_environment(self, meta): 2179 default_env = _yaml.node_get(self.__defaults, Mapping, 'environment', default_value={}) 2180 2181 if self.__is_junction: 2182 environment = {} 2183 else: 2184 project = self._get_project() 2185 environment = _yaml.node_chain_copy(project.base_environment) 2186 2187 _yaml.composite(environment, default_env) 2188 _yaml.composite(environment, meta.environment) 2189 _yaml.node_final_assertions(environment) 2190 2191 # Resolve variables in environment value strings 2192 final_env = {} 2193 for key, _ in self.node_items(environment): 2194 final_env[key] = self.node_subst_member(environment, key) 2195 2196 return final_env 2197 2198 def __extract_env_nocache(self, meta): 2199 if self.__is_junction: 2200 project_nocache = [] 2201 else: 2202 project = self._get_project() 2203 project.ensure_fully_loaded() 2204 project_nocache = project.base_env_nocache 2205 2206 default_nocache = _yaml.node_get(self.__defaults, list, 'environment-nocache', default_value=[]) 2207 element_nocache = meta.env_nocache 2208 2209 # Accumulate values from the element default, the project and the element 2210 # itself to form a complete list of nocache env vars. 2211 env_nocache = set(project_nocache + default_nocache + element_nocache) 2212 2213 # Convert back to list now we know they're unique 2214 return list(env_nocache) 2215 2216 # This will resolve the final variables to be used when 2217 # substituting command strings to be run in the sandbox 2218 # 2219 def __extract_variables(self, meta): 2220 default_vars = _yaml.node_get(self.__defaults, Mapping, 'variables', 2221 default_value={}) 2222 2223 project = self._get_project() 2224 if self.__is_junction: 2225 variables = _yaml.node_chain_copy(project.first_pass_config.base_variables) 2226 else: 2227 project.ensure_fully_loaded() 2228 variables = _yaml.node_chain_copy(project.base_variables) 2229 2230 _yaml.composite(variables, default_vars) 2231 _yaml.composite(variables, meta.variables) 2232 _yaml.node_final_assertions(variables) 2233 2234 for var in ('project-name', 'element-name', 'max-jobs'): 2235 provenance = _yaml.node_get_provenance(variables, var) 2236 if provenance and provenance.filename != '': 2237 raise LoadError(LoadErrorReason.PROTECTED_VARIABLE_REDEFINED, 2238 "{}: invalid redefinition of protected variable '{}'" 2239 .format(provenance, var)) 2240 2241 return variables 2242 2243 # This will resolve the final configuration to be handed 2244 # off to element.configure() 2245 # 2246 def __extract_config(self, meta): 2247 2248 # The default config is already composited with the project overrides 2249 config = _yaml.node_get(self.__defaults, Mapping, 'config', default_value={}) 2250 config = _yaml.node_chain_copy(config) 2251 2252 _yaml.composite(config, meta.config) 2253 _yaml.node_final_assertions(config) 2254 2255 return config 2256 2257 # Sandbox-specific configuration data, to be passed to the sandbox's constructor. 2258 # 2259 def __extract_sandbox_config(self, meta): 2260 if self.__is_junction: 2261 sandbox_config = {'build-uid': 0, 2262 'build-gid': 0} 2263 else: 2264 project = self._get_project() 2265 project.ensure_fully_loaded() 2266 sandbox_config = _yaml.node_chain_copy(project._sandbox) 2267 2268 # The default config is already composited with the project overrides 2269 sandbox_defaults = _yaml.node_get(self.__defaults, Mapping, 'sandbox', default_value={}) 2270 sandbox_defaults = _yaml.node_chain_copy(sandbox_defaults) 2271 2272 _yaml.composite(sandbox_config, sandbox_defaults) 2273 _yaml.composite(sandbox_config, meta.sandbox) 2274 _yaml.node_final_assertions(sandbox_config) 2275 2276 # Sandbox config, unlike others, has fixed members so we should validate them 2277 _yaml.node_validate(sandbox_config, ['build-uid', 'build-gid']) 2278 2279 return SandboxConfig(self.node_get_member(sandbox_config, int, 'build-uid'), 2280 self.node_get_member(sandbox_config, int, 'build-gid')) 2281 2282 # This makes a special exception for the split rules, which 2283 # elements may extend but whos defaults are defined in the project. 2284 # 2285 def __extract_public(self, meta): 2286 base_public = _yaml.node_get(self.__defaults, Mapping, 'public', default_value={}) 2287 base_public = _yaml.node_chain_copy(base_public) 2288 2289 base_bst = _yaml.node_get(base_public, Mapping, 'bst', default_value={}) 2290 base_splits = _yaml.node_get(base_bst, Mapping, 'split-rules', default_value={}) 2291 2292 element_public = _yaml.node_chain_copy(meta.public) 2293 element_bst = _yaml.node_get(element_public, Mapping, 'bst', default_value={}) 2294 element_splits = _yaml.node_get(element_bst, Mapping, 'split-rules', default_value={}) 2295 2296 # Allow elements to extend the default splits defined in their project or 2297 # element specific defaults 2298 _yaml.composite(base_splits, element_splits) 2299 2300 element_bst['split-rules'] = base_splits 2301 element_public['bst'] = element_bst 2302 2303 _yaml.node_final_assertions(element_public) 2304 2305 # Also, resolve any variables in the public split rules directly 2306 for domain, splits in self.node_items(base_splits): 2307 base_splits[domain] = [ 2308 self.__variables.subst(split.strip()) 2309 for split in splits 2310 ] 2311 2312 return element_public 2313 2314 def __init_splits(self): 2315 bstdata = self.get_public_data('bst') 2316 splits = bstdata.get('split-rules') 2317 self.__splits = { 2318 domain: re.compile('^(?:' + '|'.join([utils._glob2re(r) for r in rules]) + ')$') 2319 for domain, rules in self.node_items(splits) 2320 } 2321 2322 def __compute_splits(self, include=None, exclude=None, orphans=True): 2323 artifact_base, _ = self.__extract() 2324 basedir = os.path.join(artifact_base, 'files') 2325 2326 # No splitting requested, just report complete artifact 2327 if orphans and not (include or exclude): 2328 for filename in utils.list_relative_paths(basedir): 2329 yield filename 2330 return 2331 2332 if not self.__splits: 2333 self.__init_splits() 2334 2335 element_domains = list(self.__splits.keys()) 2336 if not include: 2337 include = element_domains 2338 if not exclude: 2339 exclude = [] 2340 2341 # Ignore domains that dont apply to this element 2342 # 2343 include = [domain for domain in include if domain in element_domains] 2344 exclude = [domain for domain in exclude if domain in element_domains] 2345 2346 # FIXME: Instead of listing the paths in an extracted artifact, 2347 # we should be using a manifest loaded from the artifact 2348 # metadata. 2349 # 2350 element_files = [ 2351 os.path.join(os.sep, filename) 2352 for filename in utils.list_relative_paths(basedir) 2353 ] 2354 2355 for filename in element_files: 2356 include_file = False 2357 exclude_file = False 2358 claimed_file = False 2359 2360 for domain in element_domains: 2361 if self.__splits[domain].match(filename): 2362 claimed_file = True 2363 if domain in include: 2364 include_file = True 2365 if domain in exclude: 2366 exclude_file = True 2367 2368 if orphans and not claimed_file: 2369 include_file = True 2370 2371 if include_file and not exclude_file: 2372 yield filename.lstrip(os.sep) 2373 2374 def __file_is_whitelisted(self, pattern): 2375 # Considered storing the whitelist regex for re-use, but public data 2376 # can be altered mid-build. 2377 # Public data is not guaranteed to stay the same for the duration of 2378 # the build, but I can think of no reason to change it mid-build. 2379 # If this ever changes, things will go wrong unexpectedly. 2380 if not self.__whitelist_regex: 2381 bstdata = self.get_public_data('bst') 2382 whitelist = _yaml.node_get(bstdata, list, 'overlap-whitelist', default_value=[]) 2383 whitelist_expressions = [utils._glob2re(self.__variables.subst(exp.strip())) for exp in whitelist] 2384 expression = ('^(?:' + '|'.join(whitelist_expressions) + ')$') 2385 self.__whitelist_regex = re.compile(expression) 2386 return self.__whitelist_regex.match(pattern) 2387 2388 # __extract(): 2389 # 2390 # Extract an artifact and return the directory 2391 # 2392 # Args: 2393 # key (str): The key for the artifact to extract, 2394 # or None for the default key 2395 # 2396 # Returns: 2397 # (str): The path to the extracted artifact 2398 # (str): The chosen key 2399 # 2400 def __extract(self, key=None): 2401 2402 if key is None: 2403 context = self._get_context() 2404 key = self.__strict_cache_key 2405 2406 # Use weak cache key, if artifact is missing for strong cache key 2407 # and the context allows use of weak cache keys 2408 if not context.get_strict() and not self.__artifacts.contains(self, key): 2409 key = self._get_cache_key(strength=_KeyStrength.WEAK) 2410 2411 return (self.__artifacts.extract(self, key), key) 2412 2413 # __get_artifact_metadata_keys(): 2414 # 2415 # Retrieve the strong and weak keys from the given artifact. 2416 # 2417 # Args: 2418 # key (str): The artifact key, or None for the default key 2419 # 2420 # Returns: 2421 # (str): The strong key 2422 # (str): The weak key 2423 # 2424 def __get_artifact_metadata_keys(self, key=None): 2425 2426 # Now extract it and possibly derive the key 2427 artifact_base, key = self.__extract(key) 2428 2429 # Now try the cache, once we're sure about the key 2430 if key in self.__metadata_keys: 2431 return (self.__metadata_keys[key]['strong'], 2432 self.__metadata_keys[key]['weak']) 2433 2434 # Parse the expensive yaml now and cache the result 2435 meta_file = os.path.join(artifact_base, 'meta', 'keys.yaml') 2436 meta = _yaml.load(meta_file) 2437 strong_key = meta['strong'] 2438 weak_key = meta['weak'] 2439 2440 assert key == strong_key or key == weak_key 2441 2442 self.__metadata_keys[strong_key] = meta 2443 self.__metadata_keys[weak_key] = meta 2444 return (strong_key, weak_key) 2445 2446 # __get_artifact_metadata_dependencies(): 2447 # 2448 # Retrieve the hash of dependency strong keys from the given artifact. 2449 # 2450 # Args: 2451 # key (str): The artifact key, or None for the default key 2452 # 2453 # Returns: 2454 # (dict): A dictionary of element names and their strong keys 2455 # 2456 def __get_artifact_metadata_dependencies(self, key=None): 2457 2458 # Extract it and possibly derive the key 2459 artifact_base, key = self.__extract(key) 2460 2461 # Now try the cache, once we're sure about the key 2462 if key in self.__metadata_dependencies: 2463 return self.__metadata_dependencies[key] 2464 2465 # Parse the expensive yaml now and cache the result 2466 meta_file = os.path.join(artifact_base, 'meta', 'dependencies.yaml') 2467 meta = _yaml.load(meta_file) 2468 2469 # Cache it under both strong and weak keys 2470 strong_key, weak_key = self.__get_artifact_metadata_keys(key) 2471 self.__metadata_dependencies[strong_key] = meta 2472 self.__metadata_dependencies[weak_key] = meta 2473 return meta 2474 2475 # __get_artifact_metadata_workspaced(): 2476 # 2477 # Retrieve the hash of dependency strong keys from the given artifact. 2478 # 2479 # Args: 2480 # key (str): The artifact key, or None for the default key 2481 # 2482 # Returns: 2483 # (bool): Whether the given artifact was workspaced 2484 # 2485 def __get_artifact_metadata_workspaced(self, key=None): 2486 2487 # Extract it and possibly derive the key 2488 artifact_base, key = self.__extract(key) 2489 2490 # Now try the cache, once we're sure about the key 2491 if key in self.__metadata_workspaced: 2492 return self.__metadata_workspaced[key] 2493 2494 # Parse the expensive yaml now and cache the result 2495 meta_file = os.path.join(artifact_base, 'meta', 'workspaced.yaml') 2496 meta = _yaml.load(meta_file) 2497 workspaced = meta['workspaced'] 2498 2499 # Cache it under both strong and weak keys 2500 strong_key, weak_key = self.__get_artifact_metadata_keys(key) 2501 self.__metadata_workspaced[strong_key] = workspaced 2502 self.__metadata_workspaced[weak_key] = workspaced 2503 return workspaced 2504 2505 # __get_artifact_metadata_workspaced_dependencies(): 2506 # 2507 # Retrieve the hash of dependency strong keys from the given artifact. 2508 # 2509 # Args: 2510 # key (str): The artifact key, or None for the default key 2511 # 2512 # Returns: 2513 # (list): List of which dependencies are workspaced 2514 # 2515 def __get_artifact_metadata_workspaced_dependencies(self, key=None): 2516 2517 # Extract it and possibly derive the key 2518 artifact_base, key = self.__extract(key) 2519 2520 # Now try the cache, once we're sure about the key 2521 if key in self.__metadata_workspaced_dependencies: 2522 return self.__metadata_workspaced_dependencies[key] 2523 2524 # Parse the expensive yaml now and cache the result 2525 meta_file = os.path.join(artifact_base, 'meta', 'workspaced-dependencies.yaml') 2526 meta = _yaml.load(meta_file) 2527 workspaced = meta['workspaced-dependencies'] 2528 2529 # Cache it under both strong and weak keys 2530 strong_key, weak_key = self.__get_artifact_metadata_keys(key) 2531 self.__metadata_workspaced_dependencies[strong_key] = workspaced 2532 self.__metadata_workspaced_dependencies[weak_key] = workspaced 2533 return workspaced 2534 2535 # __load_public_data(): 2536 # 2537 # Loads the public data from the cached artifact 2538 # 2539 def __load_public_data(self): 2540 self.__assert_cached() 2541 assert self.__dynamic_public is None 2542 2543 # Load the public data from the artifact 2544 artifact_base, _ = self.__extract() 2545 metadir = os.path.join(artifact_base, 'meta') 2546 self.__dynamic_public = _yaml.load(os.path.join(metadir, 'public.yaml')) 2547 2548 def __get_cache_keys_for_commit(self): 2549 keys = [] 2550 2551 # tag with strong cache key based on dependency versions used for the build 2552 keys.append(self._get_cache_key(strength=_KeyStrength.STRONG)) 2553 2554 # also store under weak cache key 2555 keys.append(self._get_cache_key(strength=_KeyStrength.WEAK)) 2556 2557 return utils._deduplicate(keys) 2558 2559 # __update_state_recursively() 2560 # 2561 # Update the state of all reverse dependencies, recursively. 2562 # 2563 def __update_state_recursively(self): 2564 queue = _UniquePriorityQueue() 2565 queue.push(self._unique_id, self) 2566 2567 while queue: 2568 element = queue.pop() 2569 2570 old_ready_for_runtime = element.__ready_for_runtime 2571 old_strict_cache_key = element.__strict_cache_key 2572 element._update_state() 2573 2574 if element.__ready_for_runtime != old_ready_for_runtime or \ 2575 element.__strict_cache_key != old_strict_cache_key: 2576 for rdep in element.__reverse_dependencies: 2577 queue.push(rdep._unique_id, rdep) 2578 2579 2580def _overlap_error_detail(f, forbidden_overlap_elements, elements): 2581 if forbidden_overlap_elements: 2582 return ("/{}: {} {} not permitted to overlap other elements, order {} \n" 2583 .format(f, " and ".join(forbidden_overlap_elements), 2584 "is" if len(forbidden_overlap_elements) == 1 else "are", 2585 " above ".join(reversed(elements)))) 2586 else: 2587 return "" 2588