1#!/usr/bin/env python 2# ***** BEGIN LICENSE BLOCK ***** 3# This Source Code Form is subject to the terms of the Mozilla Public 4# License, v. 2.0. If a copy of the MPL was not distributed with this file, 5# You can obtain one at http://mozilla.org/MPL/2.0/. 6# ***** END LICENSE BLOCK ***** 7"""Generic config parsing and dumping, the way I remember it from scripts 8gone by. 9 10The config should be built from script-level defaults, overlaid by 11config-file defaults, overlaid by command line options. 12 13 (For buildbot-analogues that would be factory-level defaults, 14 builder-level defaults, and build request/scheduler settings.) 15 16The config should then be locked (set to read-only, to prevent runtime 17alterations). Afterwards we should dump the config to a file that is 18uploaded with the build, and can be used to debug or replicate the build 19at a later time. 20 21TODO: 22 23* check_required_settings or something -- run at init, assert that 24 these settings are set. 25""" 26 27from __future__ import print_function 28 29import os 30import socket 31import sys 32import time 33from copy import deepcopy 34from optparse import Option, OptionGroup, OptionParser 35 36from mozharness.base.log import CRITICAL, DEBUG, ERROR, FATAL, INFO, WARNING 37 38try: 39 from urllib2 import URLError, urlopen 40except ImportError: 41 from urllib.request import urlopen 42 from urllib.error import URLError 43 44 45try: 46 import simplejson as json 47except ImportError: 48 import json 49 50 51# optparse {{{1 52class ExtendedOptionParser(OptionParser): 53 """OptionParser, but with ExtendOption as the option_class. 54 """ 55 56 def __init__(self, **kwargs): 57 kwargs['option_class'] = ExtendOption 58 OptionParser.__init__(self, **kwargs) 59 60 61class ExtendOption(Option): 62 """from http://docs.python.org/library/optparse.html?highlight=optparse#adding-new-actions""" 63 ACTIONS = Option.ACTIONS + ("extend",) 64 STORE_ACTIONS = Option.STORE_ACTIONS + ("extend",) 65 TYPED_ACTIONS = Option.TYPED_ACTIONS + ("extend",) 66 ALWAYS_TYPED_ACTIONS = Option.ALWAYS_TYPED_ACTIONS + ("extend",) 67 68 def take_action(self, action, dest, opt, value, values, parser): 69 if action == "extend": 70 lvalue = value.split(",") 71 values.ensure_value(dest, []).extend(lvalue) 72 else: 73 Option.take_action( 74 self, action, dest, opt, value, values, parser) 75 76 77def make_immutable(item): 78 if isinstance(item, list) or isinstance(item, tuple): 79 result = LockedTuple(item) 80 elif isinstance(item, dict): 81 result = ReadOnlyDict(item) 82 result.lock() 83 else: 84 result = item 85 return result 86 87 88class LockedTuple(tuple): 89 def __new__(cls, items): 90 return tuple.__new__(cls, (make_immutable(x) for x in items)) 91 92 def __deepcopy__(self, memo): 93 return [deepcopy(elem, memo) for elem in self] 94 95 96# ReadOnlyDict {{{1 97class ReadOnlyDict(dict): 98 def __init__(self, dictionary): 99 self._lock = False 100 self.update(dictionary.copy()) 101 102 def _check_lock(self): 103 assert not self._lock, "ReadOnlyDict is locked!" 104 105 def lock(self): 106 for (k, v) in list(self.items()): 107 self[k] = make_immutable(v) 108 self._lock = True 109 110 def __setitem__(self, *args): 111 self._check_lock() 112 return dict.__setitem__(self, *args) 113 114 def __delitem__(self, *args): 115 self._check_lock() 116 return dict.__delitem__(self, *args) 117 118 def clear(self, *args): 119 self._check_lock() 120 return dict.clear(self, *args) 121 122 def pop(self, *args): 123 self._check_lock() 124 return dict.pop(self, *args) 125 126 def popitem(self, *args): 127 self._check_lock() 128 return dict.popitem(self, *args) 129 130 def setdefault(self, *args): 131 self._check_lock() 132 return dict.setdefault(self, *args) 133 134 def update(self, *args): 135 self._check_lock() 136 dict.update(self, *args) 137 138 def __deepcopy__(self, memo): 139 cls = self.__class__ 140 result = cls.__new__(cls) 141 memo[id(self)] = result 142 for k, v in list(self.__dict__.items()): 143 setattr(result, k, deepcopy(v, memo)) 144 result._lock = False 145 for k, v in list(self.items()): 146 result[k] = deepcopy(v, memo) 147 return result 148 149 150DEFAULT_CONFIG_PATH = os.path.join( 151 os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 152 "configs", 153) 154 155 156# parse_config_file {{{1 157def parse_config_file(file_name, quiet=False, search_path=None, 158 config_dict_name="config"): 159 """Read a config file and return a dictionary. 160 """ 161 file_path = None 162 if os.path.exists(file_name): 163 file_path = file_name 164 else: 165 if not search_path: 166 search_path = ['.', DEFAULT_CONFIG_PATH] 167 for path in search_path: 168 if os.path.exists(os.path.join(path, file_name)): 169 file_path = os.path.join(path, file_name) 170 break 171 else: 172 raise IOError("Can't find %s in %s!" % (file_name, search_path)) 173 if file_name.endswith('.py'): 174 global_dict = {} 175 local_dict = {} 176 exec( 177 compile( 178 open( 179 file_path, 180 "rb").read(), 181 file_path, 182 'exec'), 183 global_dict, 184 local_dict) 185 config = local_dict[config_dict_name] 186 elif file_name.endswith('.json'): 187 fh = open(file_path) 188 config = {} 189 json_config = json.load(fh) 190 config = dict(json_config) 191 fh.close() 192 else: 193 raise RuntimeError( 194 "Unknown config file type %s! (config files must end in .json or .py)" % 195 file_name) 196 # TODO return file_path 197 return config 198 199 200def download_config_file(url, file_name): 201 n = 0 202 attempts = 5 203 sleeptime = 60 204 max_sleeptime = 5 * 60 205 while True: 206 if n >= attempts: 207 print( 208 "Failed to download from url %s after %d attempts, quiting..." % 209 (url, attempts)) 210 raise SystemError(-1) 211 try: 212 contents = urlopen(url, timeout=30).read() 213 break 214 except URLError as e: 215 print("Error downloading from url %s: %s" % (url, str(e))) 216 except socket.timeout as e: 217 print("Time out accessing %s: %s" % (url, str(e))) 218 except socket.error as e: 219 print("Socket error when accessing %s: %s" % (url, str(e))) 220 print("Sleeping %d seconds before retrying" % sleeptime) 221 time.sleep(sleeptime) 222 sleeptime = sleeptime * 2 223 if sleeptime > max_sleeptime: 224 sleeptime = max_sleeptime 225 n += 1 226 227 try: 228 f = open(file_name, 'w') 229 f.write(contents) 230 f.close() 231 except IOError as e: 232 print( 233 "Error writing downloaded contents to file %s: %s" % 234 (file_name, str(e))) 235 raise SystemError(-1) 236 237 238# BaseConfig {{{1 239class BaseConfig(object): 240 """Basic config setting/getting. 241 """ 242 243 def __init__( 244 self, 245 config=None, 246 initial_config_file=None, 247 config_options=None, 248 all_actions=None, 249 default_actions=None, 250 volatile_config=None, 251 option_args=None, 252 require_config_file=False, 253 append_env_variables_from_configs=False, 254 usage="usage: %prog [options]"): 255 self._config = {} 256 self.all_cfg_files_and_dicts = [] 257 self.actions = [] 258 self.config_lock = False 259 self.require_config_file = require_config_file 260 # It allows to append env variables from multiple config files 261 self.append_env_variables_from_configs = append_env_variables_from_configs 262 263 if all_actions: 264 self.all_actions = all_actions[:] 265 else: 266 self.all_actions = ['clobber', 'build'] 267 if default_actions: 268 self.default_actions = default_actions[:] 269 else: 270 self.default_actions = self.all_actions[:] 271 if volatile_config is None: 272 self.volatile_config = { 273 'actions': None, 274 'add_actions': None, 275 'no_actions': None, 276 } 277 else: 278 self.volatile_config = deepcopy(volatile_config) 279 280 if config: 281 self.set_config(config) 282 if initial_config_file: 283 initial_config = parse_config_file(initial_config_file) 284 self.all_cfg_files_and_dicts.append( 285 (initial_config_file, initial_config) 286 ) 287 self.set_config(initial_config) 288 # Since initial_config_file is only set when running unit tests, 289 # if no option_args have been specified, then the parser will 290 # parse sys.argv which in this case would be the command line 291 # options specified to run the tests, e.g. nosetests -v. Clearly, 292 # the options passed to nosetests (such as -v) should not be 293 # interpreted by mozharness as mozharness options, so we specify 294 # a dummy command line with no options, so that the parser does 295 # not add anything from the test invocation command line 296 # arguments to the mozharness options. 297 if option_args is None: 298 option_args = [ 299 'dummy_mozharness_script_with_no_command_line_options.py'] 300 if config_options is None: 301 config_options = [] 302 self._create_config_parser(config_options, usage) 303 # we allow manually passing of option args for things like nosetests 304 self.parse_args(args=option_args) 305 306 def get_read_only_config(self): 307 return ReadOnlyDict(self._config) 308 309 def _create_config_parser(self, config_options, usage): 310 self.config_parser = ExtendedOptionParser(usage=usage) 311 self.config_parser.add_option( 312 "--work-dir", action="store", dest="work_dir", 313 type="string", default="build", 314 help="Specify the work_dir (subdir of base_work_dir)" 315 ) 316 self.config_parser.add_option( 317 "--base-work-dir", 318 action="store", 319 dest="base_work_dir", 320 type="string", 321 default=os.getcwd(), 322 help="Specify the absolute path of the parent of the working directory") 323 self.config_parser.add_option( 324 "--extra-config-path", 325 action='extend', 326 dest="config_paths", 327 type="string", 328 help="Specify additional paths to search for config files.", 329 ) 330 self.config_parser.add_option( 331 "-c", "--config-file", "--cfg", action="extend", 332 dest="config_files", default=[], type="string", 333 help="Specify a config file; can be repeated", 334 ) 335 self.config_parser.add_option( 336 "-C", 337 "--opt-config-file", 338 "--opt-cfg", 339 action="extend", 340 dest="opt_config_files", 341 type="string", 342 default=[], 343 help="Specify an optional config file, like --config-file but with no " 344 "error if the file is missing; can be repeated") 345 self.config_parser.add_option( 346 "--dump-config", action="store_true", 347 dest="dump_config", 348 help="List and dump the config generated from this run to " 349 "a JSON file." 350 ) 351 self.config_parser.add_option( 352 "--dump-config-hierarchy", action="store_true", 353 dest="dump_config_hierarchy", 354 help="Like --dump-config but will list and dump which config " 355 "files were used making up the config and specify their own " 356 "keys/values that were not overwritten by another cfg -- " 357 "held the highest hierarchy." 358 ) 359 self.config_parser.add_option( 360 "--append-env-variables-from-configs", action="store_true", 361 dest="append_env_variables_from_configs", 362 help="Merge environment variables from config files." 363 ) 364 365 # Logging 366 log_option_group = OptionGroup(self.config_parser, "Logging") 367 log_option_group.add_option( 368 "--log-level", action="store", 369 type="choice", dest="log_level", default=INFO, 370 choices=[DEBUG, INFO, WARNING, ERROR, CRITICAL, FATAL], 371 help="Set log level (debug|info|warning|error|critical|fatal)" 372 ) 373 log_option_group.add_option( 374 "-q", "--quiet", action="store_false", dest="log_to_console", 375 default=True, help="Don't log to the console" 376 ) 377 log_option_group.add_option( 378 "--append-to-log", action="store_true", 379 dest="append_to_log", default=False, 380 help="Append to the log" 381 ) 382 log_option_group.add_option( 383 "--multi-log", action="store_const", const="multi", 384 dest="log_type", help="Log using MultiFileLogger" 385 ) 386 log_option_group.add_option( 387 "--simple-log", action="store_const", const="simple", 388 dest="log_type", help="Log using SimpleFileLogger" 389 ) 390 self.config_parser.add_option_group(log_option_group) 391 392 # Actions 393 action_option_group = OptionGroup( 394 self.config_parser, "Actions", 395 "Use these options to list or enable/disable actions." 396 ) 397 action_option_group.add_option( 398 "--list-actions", action="store_true", 399 dest="list_actions", 400 help="List all available actions, then exit" 401 ) 402 action_option_group.add_option( 403 "--add-action", action="extend", 404 dest="add_actions", metavar="ACTIONS", 405 help="Add action %s to the list of actions" % self.all_actions 406 ) 407 action_option_group.add_option( 408 "--no-action", action="extend", 409 dest="no_actions", metavar="ACTIONS", 410 help="Don't perform action" 411 ) 412 for action in self.all_actions: 413 action_option_group.add_option( 414 "--%s" % action, action="append_const", 415 dest="actions", const=action, 416 help="Add %s to the limited list of actions" % action 417 ) 418 action_option_group.add_option( 419 "--no-%s" % action, action="append_const", 420 dest="no_actions", const=action, 421 help="Remove %s from the list of actions to perform" % action 422 ) 423 self.config_parser.add_option_group(action_option_group) 424 # Child-specified options 425 # TODO error checking for overlapping options 426 if config_options: 427 for option in config_options: 428 self.config_parser.add_option(*option[0], **option[1]) 429 430 # Initial-config-specified options 431 config_options = self._config.get('config_options', None) 432 if config_options: 433 for option in config_options: 434 self.config_parser.add_option(*option[0], **option[1]) 435 436 def set_config(self, config, overwrite=False): 437 """This is probably doable some other way.""" 438 if self._config and not overwrite: 439 self._config.update(config) 440 else: 441 self._config = config 442 return self._config 443 444 def get_actions(self): 445 return self.actions 446 447 def verify_actions(self, action_list, quiet=False): 448 for action in action_list: 449 if action not in self.all_actions: 450 if not quiet: 451 print("Invalid action %s not in %s!" % (action, 452 self.all_actions)) 453 raise SystemExit(-1) 454 return action_list 455 456 def verify_actions_order(self, action_list): 457 try: 458 indexes = [self.all_actions.index(elt) for elt in action_list] 459 sorted_indexes = sorted(indexes) 460 for i in range(len(indexes)): 461 if indexes[i] != sorted_indexes[i]: 462 print(("Action %s comes in different order in %s\n" + 463 "than in %s") % 464 (action_list[i], action_list, self.all_actions)) 465 raise SystemExit(-1) 466 except ValueError as e: 467 print("Invalid action found: " + str(e)) 468 raise SystemExit(-1) 469 470 def list_actions(self): 471 print("Actions available:") 472 for a in self.all_actions: 473 print(" " + ("*" if a in self.default_actions else " "), a) 474 raise SystemExit(0) 475 476 def get_cfgs_from_files(self, all_config_files, options): 477 """Returns the configuration derived from the list of configuration 478 files. The result is represented as a list of `(filename, 479 config_dict)` tuples; they will be combined with keys in later 480 dictionaries taking precedence over earlier. 481 482 `all_config_files` is all files specified with `--config-file` and 483 `--opt-config-file`; `options` is the argparse options object giving 484 access to any other command-line options. 485 486 This function is also responsible for downloading any configuration 487 files specified by URL. It uses ``parse_config_file`` in this module 488 to parse individual files. 489 490 This method can be overridden in a subclass to add extra logic to the 491 way that self.config is made up. See 492 `mozharness.mozilla.building.buildbase.BuildingConfig` for an example. 493 """ 494 config_paths = options.config_paths or ['.'] 495 all_cfg_files_and_dicts = [] 496 for cf in all_config_files: 497 try: 498 if '://' in cf: # config file is an url 499 file_name = os.path.basename(cf) 500 file_path = os.path.join(os.getcwd(), file_name) 501 download_config_file(cf, file_path) 502 all_cfg_files_and_dicts.append( 503 (file_path, parse_config_file(file_path, search_path=["."])) 504 ) 505 else: 506 all_cfg_files_and_dicts.append( 507 (cf, parse_config_file( 508 cf, 509 search_path=config_paths + [DEFAULT_CONFIG_PATH] 510 )) 511 ) 512 except Exception: 513 if cf in options.opt_config_files: 514 print( 515 "WARNING: optional config file not found %s" % cf 516 ) 517 else: 518 raise 519 520 if 'EXTRA_MOZHARNESS_CONFIG' in os.environ: 521 env_config = json.loads(os.environ['EXTRA_MOZHARNESS_CONFIG']) 522 all_cfg_files_and_dicts.append( 523 ("[EXTRA_MOZHARENSS_CONFIG]", env_config)) 524 525 return all_cfg_files_and_dicts 526 527 def parse_args(self, args=None): 528 """Parse command line arguments in a generic way. 529 Return the parser object after adding the basic options, so 530 child objects can manipulate it. 531 """ 532 self.command_line = ' '.join(sys.argv) 533 if args is None: 534 args = sys.argv[1:] 535 (options, args) = self.config_parser.parse_args(args) 536 537 defaults = self.config_parser.defaults.copy() 538 539 if not options.config_files: 540 if self.require_config_file: 541 if options.list_actions: 542 self.list_actions() 543 print("Required config file not set! (use --config-file option)") 544 raise SystemExit(-1) 545 546 # this is what get_cfgs_from_files returns. It will represent each 547 # config file name and its assoctiated dict 548 # eg ('builds/branch_specifics.py', {'foo': 'bar'}) 549 # let's store this to self for things like --interpret-config-files 550 self.all_cfg_files_and_dicts.extend(self.get_cfgs_from_files( 551 # append opt_config to allow them to overwrite previous configs 552 options.config_files + options.opt_config_files, options=options 553 )) 554 config = {} 555 if (self.append_env_variables_from_configs 556 or options.append_env_variables_from_configs): 557 # We only append values from various configs for the 'env' entry 558 # For everything else we follow the standard behaviour 559 for i, (c_file, c_dict) in enumerate(self.all_cfg_files_and_dicts): 560 for v in list(c_dict.keys()): 561 if v == 'env' and v in config: 562 config[v].update(c_dict[v]) 563 else: 564 config[v] = c_dict[v] 565 else: 566 for i, (c_file, c_dict) in enumerate(self.all_cfg_files_and_dicts): 567 config.update(c_dict) 568 # assign or update self._config depending on if it exists or not 569 # NOTE self._config will be passed to ReadOnlyConfig's init -- a 570 # dict subclass with immutable locking capabilities -- and serve 571 # as the keys/values that make up that instance. Ultimately, 572 # this becomes self.config during BaseScript's init 573 self.set_config(config) 574 575 for key in list(defaults.keys()): 576 value = getattr(options, key) 577 if value is None: 578 continue 579 # Don't override config_file defaults with config_parser defaults 580 if key in defaults and value == defaults[key] and key in self._config: 581 continue 582 self._config[key] = value 583 584 # The idea behind the volatile_config is we don't want to save this 585 # info over multiple runs. This defaults to the action-specific 586 # config options, but can be anything. 587 for key in list(self.volatile_config.keys()): 588 if self._config.get(key) is not None: 589 self.volatile_config[key] = self._config[key] 590 del(self._config[key]) 591 592 self.update_actions() 593 if options.list_actions: 594 self.list_actions() 595 596 # Keep? This is for saving the volatile config in the dump_config 597 self._config['volatile_config'] = self.volatile_config 598 599 self.options = options 600 self.args = args 601 return (self.options, self.args) 602 603 def update_actions(self): 604 """ Update actions after reading in config. 605 606 Seems a little complex, but the logic goes: 607 608 First, if default_actions is specified in the config, set our 609 default actions even if the script specifies other default actions. 610 611 Without any other action-specific options, run with default actions. 612 613 If we specify --ACTION or --only-ACTION once or multiple times, 614 we want to override the default_actions list with the one(s) we list. 615 616 Otherwise, if we specify --add-action ACTION, we want to add an 617 action to the list. 618 619 Finally, if we specify --no-ACTION, remove that from the list of 620 actions to perform. 621 """ 622 if self._config.get('default_actions'): 623 default_actions = self.verify_actions( 624 self._config['default_actions']) 625 self.default_actions = default_actions 626 self.verify_actions_order(self.default_actions) 627 self.actions = self.default_actions[:] 628 if self.volatile_config['actions']: 629 actions = self.verify_actions(self.volatile_config['actions']) 630 self.actions = actions 631 elif self.volatile_config['add_actions']: 632 actions = self.verify_actions(self.volatile_config['add_actions']) 633 self.actions.extend(actions) 634 if self.volatile_config['no_actions']: 635 actions = self.verify_actions(self.volatile_config['no_actions']) 636 for action in actions: 637 if action in self.actions: 638 self.actions.remove(action) 639 640 641# __main__ {{{1 642if __name__ == '__main__': 643 pass 644