1#!/usr/bin/env python 2# ***** BEGIN LICENSE BLOCK ***** 3# This Source Code Form is subject to the terms of the Mozilla Public 4# License, v. 2.0. If a copy of the MPL was not distributed with this file, 5# You can obtain one at http://mozilla.org/MPL/2.0/. 6# ***** END LICENSE BLOCK ***** 7"""Generic config parsing and dumping, the way I remember it from scripts 8gone by. 9 10The config should be built from script-level defaults, overlaid by 11config-file defaults, overlaid by command line options. 12 13 (For buildbot-analogues that would be factory-level defaults, 14 builder-level defaults, and build request/scheduler settings.) 15 16The config should then be locked (set to read-only, to prevent runtime 17alterations). Afterwards we should dump the config to a file that is 18uploaded with the build, and can be used to debug or replicate the build 19at a later time. 20 21TODO: 22 23* check_required_settings or something -- run at init, assert that 24 these settings are set. 25""" 26 27from __future__ import absolute_import, print_function 28 29import os 30import socket 31import sys 32import time 33from copy import deepcopy 34from optparse import Option, OptionGroup, OptionParser 35 36from mozharness.base.log import CRITICAL, DEBUG, ERROR, FATAL, INFO, WARNING 37 38try: 39 from urllib2 import URLError, urlopen 40except ImportError: 41 from urllib.request import urlopen 42 from urllib.error import URLError 43 44 45try: 46 import simplejson as json 47except ImportError: 48 import json 49 50 51# optparse {{{1 52class ExtendedOptionParser(OptionParser): 53 """OptionParser, but with ExtendOption as the option_class.""" 54 55 def __init__(self, **kwargs): 56 kwargs["option_class"] = ExtendOption 57 OptionParser.__init__(self, **kwargs) 58 59 60class ExtendOption(Option): 61 """from http://docs.python.org/library/optparse.html?highlight=optparse#adding-new-actions""" 62 63 ACTIONS = Option.ACTIONS + ("extend",) 64 STORE_ACTIONS = Option.STORE_ACTIONS + ("extend",) 65 TYPED_ACTIONS = Option.TYPED_ACTIONS + ("extend",) 66 ALWAYS_TYPED_ACTIONS = Option.ALWAYS_TYPED_ACTIONS + ("extend",) 67 68 def take_action(self, action, dest, opt, value, values, parser): 69 if action == "extend": 70 lvalue = value.split(",") 71 values.ensure_value(dest, []).extend(lvalue) 72 else: 73 Option.take_action(self, action, dest, opt, value, values, parser) 74 75 76def make_immutable(item): 77 if isinstance(item, list) or isinstance(item, tuple): 78 result = LockedTuple(item) 79 elif isinstance(item, dict): 80 result = ReadOnlyDict(item) 81 result.lock() 82 else: 83 result = item 84 return result 85 86 87class LockedTuple(tuple): 88 def __new__(cls, items): 89 return tuple.__new__(cls, (make_immutable(x) for x in items)) 90 91 def __deepcopy__(self, memo): 92 return [deepcopy(elem, memo) for elem in self] 93 94 95# ReadOnlyDict {{{1 96class ReadOnlyDict(dict): 97 def __init__(self, dictionary): 98 self._lock = False 99 self.update(dictionary.copy()) 100 101 def _check_lock(self): 102 assert not self._lock, "ReadOnlyDict is locked!" 103 104 def lock(self): 105 for (k, v) in list(self.items()): 106 self[k] = make_immutable(v) 107 self._lock = True 108 109 def __setitem__(self, *args): 110 self._check_lock() 111 return dict.__setitem__(self, *args) 112 113 def __delitem__(self, *args): 114 self._check_lock() 115 return dict.__delitem__(self, *args) 116 117 def clear(self, *args): 118 self._check_lock() 119 return dict.clear(self, *args) 120 121 def pop(self, *args): 122 self._check_lock() 123 return dict.pop(self, *args) 124 125 def popitem(self, *args): 126 self._check_lock() 127 return dict.popitem(self, *args) 128 129 def setdefault(self, *args): 130 self._check_lock() 131 return dict.setdefault(self, *args) 132 133 def update(self, *args): 134 self._check_lock() 135 dict.update(self, *args) 136 137 def __deepcopy__(self, memo): 138 cls = self.__class__ 139 result = cls.__new__(cls) 140 memo[id(self)] = result 141 for k, v in list(self.__dict__.items()): 142 setattr(result, k, deepcopy(v, memo)) 143 result._lock = False 144 for k, v in list(self.items()): 145 result[k] = deepcopy(v, memo) 146 return result 147 148 149DEFAULT_CONFIG_PATH = os.path.join( 150 os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 151 "configs", 152) 153 154 155# parse_config_file {{{1 156def parse_config_file( 157 file_name, quiet=False, search_path=None, config_dict_name="config" 158): 159 """Read a config file and return a dictionary.""" 160 file_path = None 161 if os.path.exists(file_name): 162 file_path = file_name 163 else: 164 if not search_path: 165 search_path = [".", DEFAULT_CONFIG_PATH] 166 for path in search_path: 167 if os.path.exists(os.path.join(path, file_name)): 168 file_path = os.path.join(path, file_name) 169 break 170 else: 171 raise IOError("Can't find %s in %s!" % (file_name, search_path)) 172 if file_name.endswith(".py"): 173 global_dict = {} 174 local_dict = {} 175 exec( 176 compile(open(file_path, "rb").read(), file_path, "exec"), 177 global_dict, 178 local_dict, 179 ) 180 config = local_dict[config_dict_name] 181 elif file_name.endswith(".json"): 182 fh = open(file_path) 183 config = {} 184 json_config = json.load(fh) 185 config = dict(json_config) 186 fh.close() 187 else: 188 raise RuntimeError( 189 "Unknown config file type %s! (config files must end in .json or .py)" 190 % file_name 191 ) 192 # TODO return file_path 193 return config 194 195 196def download_config_file(url, file_name): 197 n = 0 198 attempts = 5 199 sleeptime = 60 200 max_sleeptime = 5 * 60 201 while True: 202 if n >= attempts: 203 print( 204 "Failed to download from url %s after %d attempts, quiting..." 205 % (url, attempts) 206 ) 207 raise SystemError(-1) 208 try: 209 contents = urlopen(url, timeout=30).read() 210 break 211 except URLError as e: 212 print("Error downloading from url %s: %s" % (url, str(e))) 213 except socket.timeout as e: 214 print("Time out accessing %s: %s" % (url, str(e))) 215 except socket.error as e: 216 print("Socket error when accessing %s: %s" % (url, str(e))) 217 print("Sleeping %d seconds before retrying" % sleeptime) 218 time.sleep(sleeptime) 219 sleeptime = sleeptime * 2 220 if sleeptime > max_sleeptime: 221 sleeptime = max_sleeptime 222 n += 1 223 224 try: 225 f = open(file_name, "w") 226 f.write(contents) 227 f.close() 228 except IOError as e: 229 print("Error writing downloaded contents to file %s: %s" % (file_name, str(e))) 230 raise SystemError(-1) 231 232 233# BaseConfig {{{1 234class BaseConfig(object): 235 """Basic config setting/getting.""" 236 237 def __init__( 238 self, 239 config=None, 240 initial_config_file=None, 241 config_options=None, 242 all_actions=None, 243 default_actions=None, 244 volatile_config=None, 245 option_args=None, 246 require_config_file=False, 247 append_env_variables_from_configs=False, 248 usage="usage: %prog [options]", 249 ): 250 self._config = {} 251 self.all_cfg_files_and_dicts = [] 252 self.actions = [] 253 self.config_lock = False 254 self.require_config_file = require_config_file 255 # It allows to append env variables from multiple config files 256 self.append_env_variables_from_configs = append_env_variables_from_configs 257 258 if all_actions: 259 self.all_actions = all_actions[:] 260 else: 261 self.all_actions = ["clobber", "build"] 262 if default_actions: 263 self.default_actions = default_actions[:] 264 else: 265 self.default_actions = self.all_actions[:] 266 if volatile_config is None: 267 self.volatile_config = { 268 "actions": None, 269 "add_actions": None, 270 "no_actions": None, 271 } 272 else: 273 self.volatile_config = deepcopy(volatile_config) 274 275 if config: 276 self.set_config(config) 277 if initial_config_file: 278 initial_config = parse_config_file(initial_config_file) 279 self.all_cfg_files_and_dicts.append((initial_config_file, initial_config)) 280 self.set_config(initial_config) 281 # Since initial_config_file is only set when running unit tests, 282 # if no option_args have been specified, then the parser will 283 # parse sys.argv which in this case would be the command line 284 # options specified to run the tests, e.g. nosetests -v. Clearly, 285 # the options passed to nosetests (such as -v) should not be 286 # interpreted by mozharness as mozharness options, so we specify 287 # a dummy command line with no options, so that the parser does 288 # not add anything from the test invocation command line 289 # arguments to the mozharness options. 290 if option_args is None: 291 option_args = [ 292 "dummy_mozharness_script_with_no_command_line_options.py" 293 ] 294 if config_options is None: 295 config_options = [] 296 self._create_config_parser(config_options, usage) 297 # we allow manually passing of option args for things like nosetests 298 self.parse_args(args=option_args) 299 300 def get_read_only_config(self): 301 return ReadOnlyDict(self._config) 302 303 def _create_config_parser(self, config_options, usage): 304 self.config_parser = ExtendedOptionParser(usage=usage) 305 self.config_parser.add_option( 306 "--work-dir", 307 action="store", 308 dest="work_dir", 309 type="string", 310 default="build", 311 help="Specify the work_dir (subdir of base_work_dir)", 312 ) 313 self.config_parser.add_option( 314 "--base-work-dir", 315 action="store", 316 dest="base_work_dir", 317 type="string", 318 default=os.getcwd(), 319 help="Specify the absolute path of the parent of the working directory", 320 ) 321 self.config_parser.add_option( 322 "--extra-config-path", 323 action="extend", 324 dest="config_paths", 325 type="string", 326 help="Specify additional paths to search for config files.", 327 ) 328 self.config_parser.add_option( 329 "-c", 330 "--config-file", 331 "--cfg", 332 action="extend", 333 dest="config_files", 334 default=[], 335 type="string", 336 help="Specify a config file; can be repeated", 337 ) 338 self.config_parser.add_option( 339 "-C", 340 "--opt-config-file", 341 "--opt-cfg", 342 action="extend", 343 dest="opt_config_files", 344 type="string", 345 default=[], 346 help="Specify an optional config file, like --config-file but with no " 347 "error if the file is missing; can be repeated", 348 ) 349 self.config_parser.add_option( 350 "--dump-config", 351 action="store_true", 352 dest="dump_config", 353 help="List and dump the config generated from this run to " "a JSON file.", 354 ) 355 self.config_parser.add_option( 356 "--dump-config-hierarchy", 357 action="store_true", 358 dest="dump_config_hierarchy", 359 help="Like --dump-config but will list and dump which config " 360 "files were used making up the config and specify their own " 361 "keys/values that were not overwritten by another cfg -- " 362 "held the highest hierarchy.", 363 ) 364 self.config_parser.add_option( 365 "--append-env-variables-from-configs", 366 action="store_true", 367 dest="append_env_variables_from_configs", 368 help="Merge environment variables from config files.", 369 ) 370 371 # Logging 372 log_option_group = OptionGroup(self.config_parser, "Logging") 373 log_option_group.add_option( 374 "--log-level", 375 action="store", 376 type="choice", 377 dest="log_level", 378 default=INFO, 379 choices=[DEBUG, INFO, WARNING, ERROR, CRITICAL, FATAL], 380 help="Set log level (debug|info|warning|error|critical|fatal)", 381 ) 382 log_option_group.add_option( 383 "-q", 384 "--quiet", 385 action="store_false", 386 dest="log_to_console", 387 default=True, 388 help="Don't log to the console", 389 ) 390 log_option_group.add_option( 391 "--append-to-log", 392 action="store_true", 393 dest="append_to_log", 394 default=False, 395 help="Append to the log", 396 ) 397 log_option_group.add_option( 398 "--multi-log", 399 action="store_const", 400 const="multi", 401 dest="log_type", 402 help="Log using MultiFileLogger", 403 ) 404 log_option_group.add_option( 405 "--simple-log", 406 action="store_const", 407 const="simple", 408 dest="log_type", 409 help="Log using SimpleFileLogger", 410 ) 411 self.config_parser.add_option_group(log_option_group) 412 413 # Actions 414 action_option_group = OptionGroup( 415 self.config_parser, 416 "Actions", 417 "Use these options to list or enable/disable actions.", 418 ) 419 action_option_group.add_option( 420 "--list-actions", 421 action="store_true", 422 dest="list_actions", 423 help="List all available actions, then exit", 424 ) 425 action_option_group.add_option( 426 "--add-action", 427 action="extend", 428 dest="add_actions", 429 metavar="ACTIONS", 430 help="Add action %s to the list of actions" % self.all_actions, 431 ) 432 action_option_group.add_option( 433 "--no-action", 434 action="extend", 435 dest="no_actions", 436 metavar="ACTIONS", 437 help="Don't perform action", 438 ) 439 for action in self.all_actions: 440 action_option_group.add_option( 441 "--%s" % action, 442 action="append_const", 443 dest="actions", 444 const=action, 445 help="Add %s to the limited list of actions" % action, 446 ) 447 action_option_group.add_option( 448 "--no-%s" % action, 449 action="append_const", 450 dest="no_actions", 451 const=action, 452 help="Remove %s from the list of actions to perform" % action, 453 ) 454 self.config_parser.add_option_group(action_option_group) 455 # Child-specified options 456 # TODO error checking for overlapping options 457 if config_options: 458 for option in config_options: 459 self.config_parser.add_option(*option[0], **option[1]) 460 461 # Initial-config-specified options 462 config_options = self._config.get("config_options", None) 463 if config_options: 464 for option in config_options: 465 self.config_parser.add_option(*option[0], **option[1]) 466 467 def set_config(self, config, overwrite=False): 468 """This is probably doable some other way.""" 469 if self._config and not overwrite: 470 self._config.update(config) 471 else: 472 self._config = config 473 return self._config 474 475 def get_actions(self): 476 return self.actions 477 478 def verify_actions(self, action_list, quiet=False): 479 for action in action_list: 480 if action not in self.all_actions: 481 if not quiet: 482 print("Invalid action %s not in %s!" % (action, self.all_actions)) 483 raise SystemExit(-1) 484 return action_list 485 486 def verify_actions_order(self, action_list): 487 try: 488 indexes = [self.all_actions.index(elt) for elt in action_list] 489 sorted_indexes = sorted(indexes) 490 for i in range(len(indexes)): 491 if indexes[i] != sorted_indexes[i]: 492 print( 493 ("Action %s comes in different order in %s\n" + "than in %s") 494 % (action_list[i], action_list, self.all_actions) 495 ) 496 raise SystemExit(-1) 497 except ValueError as e: 498 print("Invalid action found: " + str(e)) 499 raise SystemExit(-1) 500 501 def list_actions(self): 502 print("Actions available:") 503 for a in self.all_actions: 504 print(" " + ("*" if a in self.default_actions else " "), a) 505 raise SystemExit(0) 506 507 def get_cfgs_from_files(self, all_config_files, options): 508 """Returns the configuration derived from the list of configuration 509 files. The result is represented as a list of `(filename, 510 config_dict)` tuples; they will be combined with keys in later 511 dictionaries taking precedence over earlier. 512 513 `all_config_files` is all files specified with `--config-file` and 514 `--opt-config-file`; `options` is the argparse options object giving 515 access to any other command-line options. 516 517 This function is also responsible for downloading any configuration 518 files specified by URL. It uses ``parse_config_file`` in this module 519 to parse individual files. 520 521 This method can be overridden in a subclass to add extra logic to the 522 way that self.config is made up. See 523 `mozharness.mozilla.building.buildbase.BuildingConfig` for an example. 524 """ 525 config_paths = options.config_paths or ["."] 526 all_cfg_files_and_dicts = [] 527 for cf in all_config_files: 528 try: 529 if "://" in cf: # config file is an url 530 file_name = os.path.basename(cf) 531 file_path = os.path.join(os.getcwd(), file_name) 532 download_config_file(cf, file_path) 533 all_cfg_files_and_dicts.append( 534 (file_path, parse_config_file(file_path, search_path=["."])) 535 ) 536 else: 537 all_cfg_files_and_dicts.append( 538 ( 539 cf, 540 parse_config_file( 541 cf, search_path=config_paths + [DEFAULT_CONFIG_PATH] 542 ), 543 ) 544 ) 545 except Exception: 546 if cf in options.opt_config_files: 547 print("WARNING: optional config file not found %s" % cf) 548 else: 549 raise 550 551 if "EXTRA_MOZHARNESS_CONFIG" in os.environ: 552 env_config = json.loads(os.environ["EXTRA_MOZHARNESS_CONFIG"]) 553 all_cfg_files_and_dicts.append(("[EXTRA_MOZHARENSS_CONFIG]", env_config)) 554 555 return all_cfg_files_and_dicts 556 557 def parse_args(self, args=None): 558 """Parse command line arguments in a generic way. 559 Return the parser object after adding the basic options, so 560 child objects can manipulate it. 561 """ 562 self.command_line = " ".join(sys.argv) 563 if args is None: 564 args = sys.argv[1:] 565 (options, args) = self.config_parser.parse_args(args) 566 567 defaults = self.config_parser.defaults.copy() 568 569 if not options.config_files: 570 if self.require_config_file: 571 if options.list_actions: 572 self.list_actions() 573 print("Required config file not set! (use --config-file option)") 574 raise SystemExit(-1) 575 576 # this is what get_cfgs_from_files returns. It will represent each 577 # config file name and its assoctiated dict 578 # eg ('builds/branch_specifics.py', {'foo': 'bar'}) 579 # let's store this to self for things like --interpret-config-files 580 self.all_cfg_files_and_dicts.extend( 581 self.get_cfgs_from_files( 582 # append opt_config to allow them to overwrite previous configs 583 options.config_files + options.opt_config_files, 584 options=options, 585 ) 586 ) 587 config = {} 588 if ( 589 self.append_env_variables_from_configs 590 or options.append_env_variables_from_configs 591 ): 592 # We only append values from various configs for the 'env' entry 593 # For everything else we follow the standard behaviour 594 for i, (c_file, c_dict) in enumerate(self.all_cfg_files_and_dicts): 595 for v in list(c_dict.keys()): 596 if v == "env" and v in config: 597 config[v].update(c_dict[v]) 598 else: 599 config[v] = c_dict[v] 600 else: 601 for i, (c_file, c_dict) in enumerate(self.all_cfg_files_and_dicts): 602 config.update(c_dict) 603 # assign or update self._config depending on if it exists or not 604 # NOTE self._config will be passed to ReadOnlyConfig's init -- a 605 # dict subclass with immutable locking capabilities -- and serve 606 # as the keys/values that make up that instance. Ultimately, 607 # this becomes self.config during BaseScript's init 608 self.set_config(config) 609 610 for key in list(defaults.keys()): 611 value = getattr(options, key) 612 if value is None: 613 continue 614 # Don't override config_file defaults with config_parser defaults 615 if key in defaults and value == defaults[key] and key in self._config: 616 continue 617 self._config[key] = value 618 619 # The idea behind the volatile_config is we don't want to save this 620 # info over multiple runs. This defaults to the action-specific 621 # config options, but can be anything. 622 for key in list(self.volatile_config.keys()): 623 if self._config.get(key) is not None: 624 self.volatile_config[key] = self._config[key] 625 del self._config[key] 626 627 self.update_actions() 628 if options.list_actions: 629 self.list_actions() 630 631 # Keep? This is for saving the volatile config in the dump_config 632 self._config["volatile_config"] = self.volatile_config 633 634 self.options = options 635 self.args = args 636 return (self.options, self.args) 637 638 def update_actions(self): 639 """Update actions after reading in config. 640 641 Seems a little complex, but the logic goes: 642 643 First, if default_actions is specified in the config, set our 644 default actions even if the script specifies other default actions. 645 646 Without any other action-specific options, run with default actions. 647 648 If we specify --ACTION or --only-ACTION once or multiple times, 649 we want to override the default_actions list with the one(s) we list. 650 651 Otherwise, if we specify --add-action ACTION, we want to add an 652 action to the list. 653 654 Finally, if we specify --no-ACTION, remove that from the list of 655 actions to perform. 656 """ 657 if self._config.get("default_actions"): 658 default_actions = self.verify_actions(self._config["default_actions"]) 659 self.default_actions = default_actions 660 self.verify_actions_order(self.default_actions) 661 self.actions = self.default_actions[:] 662 if self.volatile_config["actions"]: 663 actions = self.verify_actions(self.volatile_config["actions"]) 664 self.actions = actions 665 elif self.volatile_config["add_actions"]: 666 actions = self.verify_actions(self.volatile_config["add_actions"]) 667 self.actions.extend(actions) 668 if self.volatile_config["no_actions"]: 669 actions = self.verify_actions(self.volatile_config["no_actions"]) 670 for action in actions: 671 if action in self.actions: 672 self.actions.remove(action) 673 674 675# __main__ {{{1 676if __name__ == "__main__": 677 pass 678