1# 2# Copyright (C) 2016-2018 Codethink Limited 3# 4# This program is free software; you can redistribute it and/or 5# modify it under the terms of the GNU Lesser General Public 6# License as published by the Free Software Foundation; either 7# version 2 of the License, or (at your option) any later version. 8# 9# This library is distributed in the hope that it will be useful, 10# but WITHOUT ANY WARRANTY; without even the implied warranty of 11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12# Lesser General Public License for more details. 13# 14# You should have received a copy of the GNU Lesser General Public 15# License along with this library. If not, see <http://www.gnu.org/licenses/>. 16# 17# Authors: 18# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> 19 20import os 21import datetime 22from collections import deque, Mapping 23from contextlib import contextmanager 24from . import utils 25from . import _cachekey 26from . import _signals 27from . import _site 28from . import _yaml 29from ._exceptions import LoadError, LoadErrorReason, BstError 30from ._message import Message, MessageType 31from ._profile import Topics, profile_start, profile_end 32from ._artifactcache import ArtifactCache, ArtifactCacheUsage 33from ._artifactcache.cascache import CASCache 34from ._workspaces import Workspaces 35from .plugin import Plugin 36 37 38# Context() 39# 40# The Context object holds all of the user preferences 41# and context for a given invocation of BuildStream. 42# 43# This is a collection of data from configuration files and command 44# line arguments and consists of information such as where to store 45# logs and artifacts, where to perform builds and cache downloaded sources, 46# verbosity levels and basically anything pertaining to the context 47# in which BuildStream was invoked. 48# 49class Context(): 50 51 def __init__(self): 52 53 # Filename indicating which configuration file was used, or None for the defaults 54 self.config_origin = None 55 56 # The directory where various sources are stored 57 self.sourcedir = None 58 59 # The directory where build sandboxes will be created 60 self.builddir = None 61 62 # The local binary artifact cache directory 63 self.artifactdir = None 64 65 # The locations from which to push and pull prebuilt artifacts 66 self.artifact_cache_specs = [] 67 68 # The directory to store build logs 69 self.logdir = None 70 71 # The abbreviated cache key length to display in the UI 72 self.log_key_length = 0 73 74 # Whether debug mode is enabled 75 self.log_debug = False 76 77 # Whether verbose mode is enabled 78 self.log_verbose = False 79 80 # Maximum number of lines to print from build logs 81 self.log_error_lines = 0 82 83 # Maximum number of lines to print in the master log for a detailed message 84 self.log_message_lines = 0 85 86 # Format string for printing the pipeline at startup time 87 self.log_element_format = None 88 89 # Format string for printing message lines in the master log 90 self.log_message_format = None 91 92 # Maximum number of fetch or refresh tasks 93 self.sched_fetchers = 4 94 95 # Maximum number of build tasks 96 self.sched_builders = 4 97 98 # Maximum number of push tasks 99 self.sched_pushers = 4 100 101 # Maximum number of retries for network tasks 102 self.sched_network_retries = 2 103 104 # What to do when a build fails in non interactive mode 105 self.sched_error_action = 'continue' 106 107 # Whether elements must be rebuilt when their dependencies have changed 108 self._strict_build_plan = None 109 110 # Make sure the XDG vars are set in the environment before loading anything 111 self._init_xdg() 112 113 # Private variables 114 self._cache_key = None 115 self._message_handler = None 116 self._message_depth = deque() 117 self._artifactcache = None 118 self._projects = [] 119 self._project_overrides = {} 120 self._workspaces = None 121 self._log_handle = None 122 self._log_filename = None 123 self.config_cache_quota = 'infinity' 124 self.artifactdir_volume = None 125 126 # load() 127 # 128 # Loads the configuration files 129 # 130 # Args: 131 # config (filename): The user specified configuration file, if any 132 # 133 # Raises: 134 # LoadError 135 # 136 # This will first load the BuildStream default configuration and then 137 # override that configuration with the configuration file indicated 138 # by *config*, if any was specified. 139 # 140 def load(self, config=None): 141 profile_start(Topics.LOAD_CONTEXT, 'load') 142 143 # If a specific config file is not specified, default to trying 144 # a $XDG_CONFIG_HOME/buildstream.conf file 145 # 146 if not config: 147 default_config = os.path.join(os.environ['XDG_CONFIG_HOME'], 148 'buildstream.conf') 149 if os.path.exists(default_config): 150 config = default_config 151 152 # Load default config 153 # 154 defaults = _yaml.load(_site.default_user_config) 155 156 if config: 157 self.config_origin = os.path.abspath(config) 158 user_config = _yaml.load(config) 159 _yaml.composite(defaults, user_config) 160 161 _yaml.node_validate(defaults, [ 162 'sourcedir', 'builddir', 'artifactdir', 'logdir', 163 'scheduler', 'artifacts', 'logging', 'projects', 164 'cache' 165 ]) 166 167 for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir']: 168 # Allow the ~ tilde expansion and any environment variables in 169 # path specification in the config files. 170 # 171 path = _yaml.node_get(defaults, str, directory) 172 path = os.path.expanduser(path) 173 path = os.path.expandvars(path) 174 path = os.path.normpath(path) 175 setattr(self, directory, path) 176 177 # Load quota configuration 178 # We need to find the first existing directory in the path of 179 # our artifactdir - the artifactdir may not have been created 180 # yet. 181 cache = _yaml.node_get(defaults, Mapping, 'cache') 182 _yaml.node_validate(cache, ['quota']) 183 184 self.config_cache_quota = _yaml.node_get(cache, str, 'quota', default_value='infinity') 185 186 # Load artifact share configuration 187 self.artifact_cache_specs = ArtifactCache.specs_from_config_node(defaults) 188 189 # Load logging config 190 logging = _yaml.node_get(defaults, Mapping, 'logging') 191 _yaml.node_validate(logging, [ 192 'key-length', 'verbose', 193 'error-lines', 'message-lines', 194 'debug', 'element-format', 'message-format' 195 ]) 196 self.log_key_length = _yaml.node_get(logging, int, 'key-length') 197 self.log_debug = _yaml.node_get(logging, bool, 'debug') 198 self.log_verbose = _yaml.node_get(logging, bool, 'verbose') 199 self.log_error_lines = _yaml.node_get(logging, int, 'error-lines') 200 self.log_message_lines = _yaml.node_get(logging, int, 'message-lines') 201 self.log_element_format = _yaml.node_get(logging, str, 'element-format') 202 self.log_message_format = _yaml.node_get(logging, str, 'message-format') 203 204 # Load scheduler config 205 scheduler = _yaml.node_get(defaults, Mapping, 'scheduler') 206 _yaml.node_validate(scheduler, [ 207 'on-error', 'fetchers', 'builders', 208 'pushers', 'network-retries' 209 ]) 210 self.sched_error_action = _yaml.node_get(scheduler, str, 'on-error') 211 self.sched_fetchers = _yaml.node_get(scheduler, int, 'fetchers') 212 self.sched_builders = _yaml.node_get(scheduler, int, 'builders') 213 self.sched_pushers = _yaml.node_get(scheduler, int, 'pushers') 214 self.sched_network_retries = _yaml.node_get(scheduler, int, 'network-retries') 215 216 # Load per-projects overrides 217 self._project_overrides = _yaml.node_get(defaults, Mapping, 'projects', default_value={}) 218 219 # Shallow validation of overrides, parts of buildstream which rely 220 # on the overrides are expected to validate elsewhere. 221 for _, overrides in _yaml.node_items(self._project_overrides): 222 _yaml.node_validate(overrides, ['artifacts', 'options', 'strict', 'default-mirror']) 223 224 profile_end(Topics.LOAD_CONTEXT, 'load') 225 226 valid_actions = ['continue', 'quit'] 227 if self.sched_error_action not in valid_actions: 228 provenance = _yaml.node_get_provenance(scheduler, 'on-error') 229 raise LoadError(LoadErrorReason.INVALID_DATA, 230 "{}: on-error should be one of: {}".format( 231 provenance, ", ".join(valid_actions))) 232 233 @property 234 def artifactcache(self): 235 if not self._artifactcache: 236 self._artifactcache = CASCache(self) 237 238 return self._artifactcache 239 240 # get_artifact_cache_usage() 241 # 242 # Fetches the current usage of the artifact cache 243 # 244 # Returns: 245 # (ArtifactCacheUsage): The current status 246 # 247 def get_artifact_cache_usage(self): 248 return ArtifactCacheUsage(self.artifactcache) 249 250 # add_project(): 251 # 252 # Add a project to the context. 253 # 254 # Args: 255 # project (Project): The project to add 256 # 257 def add_project(self, project): 258 if not self._projects: 259 self._workspaces = Workspaces(project) 260 self._projects.append(project) 261 262 # get_projects(): 263 # 264 # Return the list of projects in the context. 265 # 266 # Returns: 267 # (list): The list of projects 268 # 269 def get_projects(self): 270 return self._projects 271 272 # get_toplevel_project(): 273 # 274 # Return the toplevel project, the one which BuildStream was 275 # invoked with as opposed to a junctioned subproject. 276 # 277 # Returns: 278 # (list): The list of projects 279 # 280 def get_toplevel_project(self): 281 return self._projects[0] 282 283 def get_workspaces(self): 284 return self._workspaces 285 286 # get_overrides(): 287 # 288 # Fetch the override dictionary for the active project. This returns 289 # a node loaded from YAML and as such, values loaded from the returned 290 # node should be loaded using the _yaml.node_get() family of functions. 291 # 292 # Args: 293 # project_name (str): The project name 294 # 295 # Returns: 296 # (Mapping): The overrides dictionary for the specified project 297 # 298 def get_overrides(self, project_name): 299 return _yaml.node_get(self._project_overrides, Mapping, project_name, default_value={}) 300 301 # get_strict(): 302 # 303 # Fetch whether we are strict or not 304 # 305 # Returns: 306 # (bool): Whether or not to use strict build plan 307 # 308 def get_strict(self): 309 310 # If it was set by the CLI, it overrides any config 311 if self._strict_build_plan is not None: 312 return self._strict_build_plan 313 314 toplevel = self.get_toplevel_project() 315 overrides = self.get_overrides(toplevel.name) 316 return _yaml.node_get(overrides, bool, 'strict', default_value=True) 317 318 # get_cache_key(): 319 # 320 # Returns the cache key, calculating it if necessary 321 # 322 # Returns: 323 # (str): A hex digest cache key for the Context 324 # 325 def get_cache_key(self): 326 if self._cache_key is None: 327 328 # Anything that alters the build goes into the unique key 329 self._cache_key = _cachekey.generate_key({}) 330 331 return self._cache_key 332 333 # set_message_handler() 334 # 335 # Sets the handler for any status messages propagated through 336 # the context. 337 # 338 # The message handler should have the same signature as 339 # the message() method 340 def set_message_handler(self, handler): 341 self._message_handler = handler 342 343 # silent_messages(): 344 # 345 # Returns: 346 # (bool): Whether messages are currently being silenced 347 # 348 def silent_messages(self): 349 for silent in self._message_depth: 350 if silent: 351 return True 352 return False 353 354 # message(): 355 # 356 # Proxies a message back to the caller, this is the central 357 # point through which all messages pass. 358 # 359 # Args: 360 # message: A Message object 361 # 362 def message(self, message): 363 364 # Tag message only once 365 if message.depth is None: 366 message.depth = len(list(self._message_depth)) 367 368 # If we are recording messages, dump a copy into the open log file. 369 self._record_message(message) 370 371 # Send it off to the log handler (can be the frontend, 372 # or it can be the child task which will propagate 373 # to the frontend) 374 assert self._message_handler 375 376 self._message_handler(message, context=self) 377 return 378 379 # silence() 380 # 381 # A context manager to silence messages, this behaves in 382 # the same way as the `silent_nested` argument of the 383 # Context._timed_activity() context manager: especially 384 # important messages will not be silenced. 385 # 386 @contextmanager 387 def silence(self): 388 self._push_message_depth(True) 389 try: 390 yield 391 finally: 392 self._pop_message_depth() 393 394 # timed_activity() 395 # 396 # Context manager for performing timed activities and logging those 397 # 398 # Args: 399 # context (Context): The invocation context object 400 # activity_name (str): The name of the activity 401 # detail (str): An optional detailed message, can be multiline output 402 # silent_nested (bool): If specified, nested messages will be silenced 403 # 404 @contextmanager 405 def timed_activity(self, activity_name, *, unique_id=None, detail=None, silent_nested=False): 406 407 starttime = datetime.datetime.now() 408 stopped_time = None 409 410 def stop_time(): 411 nonlocal stopped_time 412 stopped_time = datetime.datetime.now() 413 414 def resume_time(): 415 nonlocal stopped_time 416 nonlocal starttime 417 sleep_time = datetime.datetime.now() - stopped_time 418 starttime += sleep_time 419 420 with _signals.suspendable(stop_time, resume_time): 421 try: 422 # Push activity depth for status messages 423 message = Message(unique_id, MessageType.START, activity_name, detail=detail) 424 self.message(message) 425 self._push_message_depth(silent_nested) 426 yield 427 428 except BstError: 429 # Note the failure in status messages and reraise, the scheduler 430 # expects an error when there is an error. 431 elapsed = datetime.datetime.now() - starttime 432 message = Message(unique_id, MessageType.FAIL, activity_name, elapsed=elapsed) 433 self._pop_message_depth() 434 self.message(message) 435 raise 436 437 elapsed = datetime.datetime.now() - starttime 438 message = Message(unique_id, MessageType.SUCCESS, activity_name, elapsed=elapsed) 439 self._pop_message_depth() 440 self.message(message) 441 442 # recorded_messages() 443 # 444 # Records all messages in a log file while the context manager 445 # is active. 446 # 447 # In addition to automatically writing all messages to the 448 # specified logging file, an open file handle for process stdout 449 # and stderr will be available via the Context.get_log_handle() API, 450 # and the full logfile path will be available via the 451 # Context.get_log_filename() API. 452 # 453 # Args: 454 # filename (str): A logging directory relative filename, 455 # the pid and .log extension will be automatically 456 # appended 457 # 458 # Yields: 459 # (str): The fully qualified log filename 460 # 461 @contextmanager 462 def recorded_messages(self, filename): 463 464 # We dont allow recursing in this context manager, and 465 # we also do not allow it in the main process. 466 assert self._log_handle is None 467 assert self._log_filename is None 468 assert not utils._is_main_process() 469 470 # Create the fully qualified logfile in the log directory, 471 # appending the pid and .log extension at the end. 472 self._log_filename = os.path.join(self.logdir, 473 '{}.{}.log'.format(filename, os.getpid())) 474 475 # Ensure the directory exists first 476 directory = os.path.dirname(self._log_filename) 477 os.makedirs(directory, exist_ok=True) 478 479 with open(self._log_filename, 'a') as logfile: 480 481 # Write one last line to the log and flush it to disk 482 def flush_log(): 483 484 # If the process currently had something happening in the I/O stack 485 # then trying to reenter the I/O stack will fire a runtime error. 486 # 487 # So just try to flush as well as we can at SIGTERM time 488 try: 489 logfile.write('\n\nForcefully terminated\n') 490 logfile.flush() 491 except RuntimeError: 492 os.fsync(logfile.fileno()) 493 494 self._log_handle = logfile 495 with _signals.terminator(flush_log): 496 yield self._log_filename 497 498 self._log_handle = None 499 self._log_filename = None 500 501 # get_log_handle() 502 # 503 # Fetches the active log handle, this will return the active 504 # log file handle when the Context.recorded_messages() context 505 # manager is active 506 # 507 # Returns: 508 # (file): The active logging file handle, or None 509 # 510 def get_log_handle(self): 511 return self._log_handle 512 513 # get_log_filename() 514 # 515 # Fetches the active log filename, this will return the active 516 # log filename when the Context.recorded_messages() context 517 # manager is active 518 # 519 # Returns: 520 # (str): The active logging filename, or None 521 # 522 def get_log_filename(self): 523 return self._log_filename 524 525 # _record_message() 526 # 527 # Records the message if recording is enabled 528 # 529 # Args: 530 # message (Message): The message to record 531 # 532 def _record_message(self, message): 533 534 if self._log_handle is None: 535 return 536 537 INDENT = " " 538 EMPTYTIME = "--:--:--" 539 template = "[{timecode: <8}] {type: <7}" 540 541 # If this message is associated with a plugin, print what 542 # we know about the plugin. 543 plugin_name = "" 544 if message.unique_id: 545 template += " {plugin}" 546 plugin = Plugin._lookup(message.unique_id) 547 plugin_name = plugin.name 548 549 template += ": {message}" 550 551 detail = '' 552 if message.detail is not None: 553 template += "\n\n{detail}" 554 detail = message.detail.rstrip('\n') 555 detail = INDENT + INDENT.join(detail.splitlines(True)) 556 557 timecode = EMPTYTIME 558 if message.message_type in (MessageType.SUCCESS, MessageType.FAIL): 559 hours, remainder = divmod(int(message.elapsed.total_seconds()), 60**2) 560 minutes, seconds = divmod(remainder, 60) 561 timecode = "{0:02d}:{1:02d}:{2:02d}".format(hours, minutes, seconds) 562 563 text = template.format(timecode=timecode, 564 plugin=plugin_name, 565 type=message.message_type.upper(), 566 message=message.message, 567 detail=detail) 568 569 # Write to the open log file 570 self._log_handle.write('{}\n'.format(text)) 571 self._log_handle.flush() 572 573 # _push_message_depth() / _pop_message_depth() 574 # 575 # For status messages, send the depth of timed 576 # activities inside a given task through the message 577 # 578 def _push_message_depth(self, silent_nested): 579 self._message_depth.appendleft(silent_nested) 580 581 def _pop_message_depth(self): 582 assert self._message_depth 583 self._message_depth.popleft() 584 585 # Force the resolved XDG variables into the environment, 586 # this is so that they can be used directly to specify 587 # preferred locations of things from user configuration 588 # files. 589 def _init_xdg(self): 590 if not os.environ.get('XDG_CACHE_HOME'): 591 os.environ['XDG_CACHE_HOME'] = os.path.expanduser('~/.cache') 592 if not os.environ.get('XDG_CONFIG_HOME'): 593 os.environ['XDG_CONFIG_HOME'] = os.path.expanduser('~/.config') 594 if not os.environ.get('XDG_DATA_HOME'): 595 os.environ['XDG_DATA_HOME'] = os.path.expanduser('~/.local/share') 596