1#!/usr/bin/env python 2# ***** BEGIN LICENSE BLOCK ***** 3# This Source Code Form is subject to the terms of the Mozilla Public 4# License, v. 2.0. If a copy of the MPL was not distributed with this file, 5# You can obtain one at http://mozilla.org/MPL/2.0/. 6# ***** END LICENSE BLOCK ***** 7"""Generic config parsing and dumping, the way I remember it from scripts 8gone by. 9 10The config should be built from script-level defaults, overlaid by 11config-file defaults, overlaid by command line options. 12 13 (For buildbot-analogues that would be factory-level defaults, 14 builder-level defaults, and build request/scheduler settings.) 15 16The config should then be locked (set to read-only, to prevent runtime 17alterations). Afterwards we should dump the config to a file that is 18uploaded with the build, and can be used to debug or replicate the build 19at a later time. 20 21TODO: 22 23* check_required_settings or something -- run at init, assert that 24 these settings are set. 25""" 26 27from copy import deepcopy 28from optparse import OptionParser, Option, OptionGroup 29import os 30import sys 31import urllib2 32import socket 33import time 34try: 35 import simplejson as json 36except ImportError: 37 import json 38 39from mozharness.base.log import DEBUG, INFO, WARNING, ERROR, CRITICAL, FATAL 40 41 42# optparse {{{1 43class ExtendedOptionParser(OptionParser): 44 """OptionParser, but with ExtendOption as the option_class. 45 """ 46 def __init__(self, **kwargs): 47 kwargs['option_class'] = ExtendOption 48 OptionParser.__init__(self, **kwargs) 49 50 51class ExtendOption(Option): 52 """from http://docs.python.org/library/optparse.html?highlight=optparse#adding-new-actions""" 53 ACTIONS = Option.ACTIONS + ("extend",) 54 STORE_ACTIONS = Option.STORE_ACTIONS + ("extend",) 55 TYPED_ACTIONS = Option.TYPED_ACTIONS + ("extend",) 56 ALWAYS_TYPED_ACTIONS = Option.ALWAYS_TYPED_ACTIONS + ("extend",) 57 58 def take_action(self, action, dest, opt, value, values, parser): 59 if action == "extend": 60 lvalue = value.split(",") 61 values.ensure_value(dest, []).extend(lvalue) 62 else: 63 Option.take_action( 64 self, action, dest, opt, value, values, parser) 65 66 67def make_immutable(item): 68 if isinstance(item, list) or isinstance(item, tuple): 69 result = LockedTuple(item) 70 elif isinstance(item, dict): 71 result = ReadOnlyDict(item) 72 result.lock() 73 else: 74 result = item 75 return result 76 77 78class LockedTuple(tuple): 79 def __new__(cls, items): 80 return tuple.__new__(cls, (make_immutable(x) for x in items)) 81 82 def __deepcopy__(self, memo): 83 return [deepcopy(elem, memo) for elem in self] 84 85 86# ReadOnlyDict {{{1 87class ReadOnlyDict(dict): 88 def __init__(self, dictionary): 89 self._lock = False 90 self.update(dictionary.copy()) 91 92 def _check_lock(self): 93 assert not self._lock, "ReadOnlyDict is locked!" 94 95 def lock(self): 96 for (k, v) in self.items(): 97 self[k] = make_immutable(v) 98 self._lock = True 99 100 def __setitem__(self, *args): 101 self._check_lock() 102 return dict.__setitem__(self, *args) 103 104 def __delitem__(self, *args): 105 self._check_lock() 106 return dict.__delitem__(self, *args) 107 108 def clear(self, *args): 109 self._check_lock() 110 return dict.clear(self, *args) 111 112 def pop(self, *args): 113 self._check_lock() 114 return dict.pop(self, *args) 115 116 def popitem(self, *args): 117 self._check_lock() 118 return dict.popitem(self, *args) 119 120 def setdefault(self, *args): 121 self._check_lock() 122 return dict.setdefault(self, *args) 123 124 def update(self, *args): 125 self._check_lock() 126 dict.update(self, *args) 127 128 def __deepcopy__(self, memo): 129 cls = self.__class__ 130 result = cls.__new__(cls) 131 memo[id(self)] = result 132 for k, v in self.__dict__.items(): 133 setattr(result, k, deepcopy(v, memo)) 134 result._lock = False 135 for k, v in self.items(): 136 result[k] = deepcopy(v, memo) 137 return result 138 139 140DEFAULT_CONFIG_PATH = os.path.join( 141 os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 142 "configs", 143) 144 145 146# parse_config_file {{{1 147def parse_config_file(file_name, quiet=False, search_path=None, 148 config_dict_name="config"): 149 """Read a config file and return a dictionary. 150 """ 151 file_path = None 152 if os.path.exists(file_name): 153 file_path = file_name 154 else: 155 if not search_path: 156 search_path = ['.', DEFAULT_CONFIG_PATH] 157 for path in search_path: 158 if os.path.exists(os.path.join(path, file_name)): 159 file_path = os.path.join(path, file_name) 160 break 161 else: 162 raise IOError("Can't find %s in %s!" % (file_name, search_path)) 163 if file_name.endswith('.py'): 164 global_dict = {} 165 local_dict = {} 166 execfile(file_path, global_dict, local_dict) 167 config = local_dict[config_dict_name] 168 elif file_name.endswith('.json'): 169 fh = open(file_path) 170 config = {} 171 json_config = json.load(fh) 172 config = dict(json_config) 173 fh.close() 174 else: 175 raise RuntimeError( 176 "Unknown config file type %s! (config files must end in .json or .py)" % file_name) 177 # TODO return file_path 178 return config 179 180 181def download_config_file(url, file_name): 182 n = 0 183 attempts = 5 184 sleeptime = 60 185 max_sleeptime = 5 * 60 186 while True: 187 if n >= attempts: 188 print "Failed to download from url %s after %d attempts, quiting..." % (url, attempts) 189 raise SystemError(-1) 190 try: 191 contents = urllib2.urlopen(url, timeout=30).read() 192 break 193 except urllib2.URLError, e: 194 print "Error downloading from url %s: %s" % (url, str(e)) 195 except socket.timeout, e: 196 print "Time out accessing %s: %s" % (url, str(e)) 197 except socket.error, e: 198 print "Socket error when accessing %s: %s" % (url, str(e)) 199 print "Sleeping %d seconds before retrying" % sleeptime 200 time.sleep(sleeptime) 201 sleeptime = sleeptime * 2 202 if sleeptime > max_sleeptime: 203 sleeptime = max_sleeptime 204 n += 1 205 206 try: 207 f = open(file_name, 'w') 208 f.write(contents) 209 f.close() 210 except IOError, e: 211 print "Error writing downloaded contents to file %s: %s" % (file_name, str(e)) 212 raise SystemError(-1) 213 214 215# BaseConfig {{{1 216class BaseConfig(object): 217 """Basic config setting/getting. 218 """ 219 def __init__(self, config=None, initial_config_file=None, config_options=None, 220 all_actions=None, default_actions=None, 221 volatile_config=None, option_args=None, 222 require_config_file=False, 223 append_env_variables_from_configs=False, 224 usage="usage: %prog [options]"): 225 self._config = {} 226 self.all_cfg_files_and_dicts = [] 227 self.actions = [] 228 self.config_lock = False 229 self.require_config_file = require_config_file 230 # It allows to append env variables from multiple config files 231 self.append_env_variables_from_configs = append_env_variables_from_configs 232 233 if all_actions: 234 self.all_actions = all_actions[:] 235 else: 236 self.all_actions = ['clobber', 'build'] 237 if default_actions: 238 self.default_actions = default_actions[:] 239 else: 240 self.default_actions = self.all_actions[:] 241 if volatile_config is None: 242 self.volatile_config = { 243 'actions': None, 244 'add_actions': None, 245 'no_actions': None, 246 } 247 else: 248 self.volatile_config = deepcopy(volatile_config) 249 250 if config: 251 self.set_config(config) 252 if initial_config_file: 253 initial_config = parse_config_file(initial_config_file) 254 self.all_cfg_files_and_dicts.append( 255 (initial_config_file, initial_config) 256 ) 257 self.set_config(initial_config) 258 # Since initial_config_file is only set when running unit tests, 259 # if no option_args have been specified, then the parser will 260 # parse sys.argv which in this case would be the command line 261 # options specified to run the tests, e.g. nosetests -v. Clearly, 262 # the options passed to nosetests (such as -v) should not be 263 # interpreted by mozharness as mozharness options, so we specify 264 # a dummy command line with no options, so that the parser does 265 # not add anything from the test invocation command line 266 # arguments to the mozharness options. 267 if option_args is None: 268 option_args = ['dummy_mozharness_script_with_no_command_line_options.py'] 269 if config_options is None: 270 config_options = [] 271 self._create_config_parser(config_options, usage) 272 # we allow manually passing of option args for things like nosetests 273 self.parse_args(args=option_args) 274 275 def get_read_only_config(self): 276 return ReadOnlyDict(self._config) 277 278 def _create_config_parser(self, config_options, usage): 279 self.config_parser = ExtendedOptionParser(usage=usage) 280 self.config_parser.add_option( 281 "--work-dir", action="store", dest="work_dir", 282 type="string", default="build", 283 help="Specify the work_dir (subdir of base_work_dir)" 284 ) 285 self.config_parser.add_option( 286 "--base-work-dir", action="store", dest="base_work_dir", 287 type="string", default=os.getcwd(), 288 help="Specify the absolute path of the parent of the working directory" 289 ) 290 self.config_parser.add_option( 291 "--extra-config-path", action='extend', dest="config_paths", 292 type="string", help="Specify additional paths to search for config files.", 293 ) 294 self.config_parser.add_option( 295 "-c", "--config-file", "--cfg", action="extend", 296 dest="config_files", default=[], type="string", 297 help="Specify a config file; can be repeated", 298 ) 299 self.config_parser.add_option( 300 "-C", "--opt-config-file", "--opt-cfg", action="extend", 301 dest="opt_config_files", type="string", default=[], 302 help="Specify an optional config file, like --config-file but with no " 303 "error if the file is missing; can be repeated" 304 ) 305 self.config_parser.add_option( 306 "--dump-config", action="store_true", 307 dest="dump_config", 308 help="List and dump the config generated from this run to " 309 "a JSON file." 310 ) 311 self.config_parser.add_option( 312 "--dump-config-hierarchy", action="store_true", 313 dest="dump_config_hierarchy", 314 help="Like --dump-config but will list and dump which config " 315 "files were used making up the config and specify their own " 316 "keys/values that were not overwritten by another cfg -- " 317 "held the highest hierarchy." 318 ) 319 self.config_parser.add_option( 320 "--append-env-variables-from-configs", action="store_true", 321 dest="append_env_variables_from_configs", 322 help="Merge environment variables from config files." 323 ) 324 325 # Logging 326 log_option_group = OptionGroup(self.config_parser, "Logging") 327 log_option_group.add_option( 328 "--log-level", action="store", 329 type="choice", dest="log_level", default=INFO, 330 choices=[DEBUG, INFO, WARNING, ERROR, CRITICAL, FATAL], 331 help="Set log level (debug|info|warning|error|critical|fatal)" 332 ) 333 log_option_group.add_option( 334 "-q", "--quiet", action="store_false", dest="log_to_console", 335 default=True, help="Don't log to the console" 336 ) 337 log_option_group.add_option( 338 "--append-to-log", action="store_true", 339 dest="append_to_log", default=False, 340 help="Append to the log" 341 ) 342 log_option_group.add_option( 343 "--multi-log", action="store_const", const="multi", 344 dest="log_type", help="Log using MultiFileLogger" 345 ) 346 log_option_group.add_option( 347 "--simple-log", action="store_const", const="simple", 348 dest="log_type", help="Log using SimpleFileLogger" 349 ) 350 self.config_parser.add_option_group(log_option_group) 351 352 # Actions 353 action_option_group = OptionGroup( 354 self.config_parser, "Actions", 355 "Use these options to list or enable/disable actions." 356 ) 357 action_option_group.add_option( 358 "--list-actions", action="store_true", 359 dest="list_actions", 360 help="List all available actions, then exit" 361 ) 362 action_option_group.add_option( 363 "--add-action", action="extend", 364 dest="add_actions", metavar="ACTIONS", 365 help="Add action %s to the list of actions" % self.all_actions 366 ) 367 action_option_group.add_option( 368 "--no-action", action="extend", 369 dest="no_actions", metavar="ACTIONS", 370 help="Don't perform action" 371 ) 372 for action in self.all_actions: 373 action_option_group.add_option( 374 "--%s" % action, action="append_const", 375 dest="actions", const=action, 376 help="Add %s to the limited list of actions" % action 377 ) 378 action_option_group.add_option( 379 "--no-%s" % action, action="append_const", 380 dest="no_actions", const=action, 381 help="Remove %s from the list of actions to perform" % action 382 ) 383 self.config_parser.add_option_group(action_option_group) 384 # Child-specified options 385 # TODO error checking for overlapping options 386 if config_options: 387 for option in config_options: 388 self.config_parser.add_option(*option[0], **option[1]) 389 390 # Initial-config-specified options 391 config_options = self._config.get('config_options', None) 392 if config_options: 393 for option in config_options: 394 self.config_parser.add_option(*option[0], **option[1]) 395 396 def set_config(self, config, overwrite=False): 397 """This is probably doable some other way.""" 398 if self._config and not overwrite: 399 self._config.update(config) 400 else: 401 self._config = config 402 return self._config 403 404 def get_actions(self): 405 return self.actions 406 407 def verify_actions(self, action_list, quiet=False): 408 for action in action_list: 409 if action not in self.all_actions: 410 if not quiet: 411 print("Invalid action %s not in %s!" % (action, 412 self.all_actions)) 413 raise SystemExit(-1) 414 return action_list 415 416 def verify_actions_order(self, action_list): 417 try: 418 indexes = [self.all_actions.index(elt) for elt in action_list] 419 sorted_indexes = sorted(indexes) 420 for i in range(len(indexes)): 421 if indexes[i] != sorted_indexes[i]: 422 print(("Action %s comes in different order in %s\n" + 423 "than in %s") % (action_list[i], action_list, self.all_actions)) 424 raise SystemExit(-1) 425 except ValueError as e: 426 print("Invalid action found: " + str(e)) 427 raise SystemExit(-1) 428 429 def list_actions(self): 430 print "Actions available:" 431 for a in self.all_actions: 432 print " " + ("*" if a in self.default_actions else " "), a 433 raise SystemExit(0) 434 435 def get_cfgs_from_files(self, all_config_files, options): 436 """Returns the configuration derived from the list of configuration 437 files. The result is represented as a list of `(filename, 438 config_dict)` tuples; they will be combined with keys in later 439 dictionaries taking precedence over earlier. 440 441 `all_config_files` is all files specified with `--config-file` and 442 `--opt-config-file`; `options` is the argparse options object giving 443 access to any other command-line options. 444 445 This function is also responsible for downloading any configuration 446 files specified by URL. It uses ``parse_config_file`` in this module 447 to parse individual files. 448 449 This method can be overridden in a subclass to add extra logic to the 450 way that self.config is made up. See 451 `mozharness.mozilla.building.buildbase.BuildingConfig` for an example. 452 """ 453 config_paths = options.config_paths or ['.'] 454 all_cfg_files_and_dicts = [] 455 for cf in all_config_files: 456 try: 457 if '://' in cf: # config file is an url 458 file_name = os.path.basename(cf) 459 file_path = os.path.join(os.getcwd(), file_name) 460 download_config_file(cf, file_path) 461 all_cfg_files_and_dicts.append( 462 (file_path, parse_config_file(file_path, search_path=["."])) 463 ) 464 else: 465 all_cfg_files_and_dicts.append( 466 (cf, parse_config_file( 467 cf, 468 search_path=config_paths + [DEFAULT_CONFIG_PATH] 469 )) 470 ) 471 except Exception: 472 if cf in options.opt_config_files: 473 print( 474 "WARNING: optional config file not found %s" % cf 475 ) 476 else: 477 raise 478 479 if 'EXTRA_MOZHARNESS_CONFIG' in os.environ: 480 env_config = json.loads(os.environ['EXTRA_MOZHARNESS_CONFIG']) 481 all_cfg_files_and_dicts.append(("[EXTRA_MOZHARENSS_CONFIG]", env_config)) 482 483 return all_cfg_files_and_dicts 484 485 def parse_args(self, args=None): 486 """Parse command line arguments in a generic way. 487 Return the parser object after adding the basic options, so 488 child objects can manipulate it. 489 """ 490 self.command_line = ' '.join(sys.argv) 491 if args is None: 492 args = sys.argv[1:] 493 (options, args) = self.config_parser.parse_args(args) 494 495 defaults = self.config_parser.defaults.copy() 496 497 if not options.config_files: 498 if self.require_config_file: 499 if options.list_actions: 500 self.list_actions() 501 print("Required config file not set! (use --config-file option)") 502 raise SystemExit(-1) 503 504 # this is what get_cfgs_from_files returns. It will represent each 505 # config file name and its assoctiated dict 506 # eg ('builds/branch_specifics.py', {'foo': 'bar'}) 507 # let's store this to self for things like --interpret-config-files 508 self.all_cfg_files_and_dicts.extend(self.get_cfgs_from_files( 509 # append opt_config to allow them to overwrite previous configs 510 options.config_files + options.opt_config_files, options=options 511 )) 512 config = {} 513 if (self.append_env_variables_from_configs 514 or options.append_env_variables_from_configs): 515 # We only append values from various configs for the 'env' entry 516 # For everything else we follow the standard behaviour 517 for i, (c_file, c_dict) in enumerate(self.all_cfg_files_and_dicts): 518 for v in c_dict.keys(): 519 if v == 'env' and v in config: 520 config[v].update(c_dict[v]) 521 else: 522 config[v] = c_dict[v] 523 else: 524 for i, (c_file, c_dict) in enumerate(self.all_cfg_files_and_dicts): 525 config.update(c_dict) 526 # assign or update self._config depending on if it exists or not 527 # NOTE self._config will be passed to ReadOnlyConfig's init -- a 528 # dict subclass with immutable locking capabilities -- and serve 529 # as the keys/values that make up that instance. Ultimately, 530 # this becomes self.config during BaseScript's init 531 self.set_config(config) 532 533 for key in defaults.keys(): 534 value = getattr(options, key) 535 if value is None: 536 continue 537 # Don't override config_file defaults with config_parser defaults 538 if key in defaults and value == defaults[key] and key in self._config: 539 continue 540 self._config[key] = value 541 542 # The idea behind the volatile_config is we don't want to save this 543 # info over multiple runs. This defaults to the action-specific 544 # config options, but can be anything. 545 for key in self.volatile_config.keys(): 546 if self._config.get(key) is not None: 547 self.volatile_config[key] = self._config[key] 548 del(self._config[key]) 549 550 self.update_actions() 551 if options.list_actions: 552 self.list_actions() 553 554 # Keep? This is for saving the volatile config in the dump_config 555 self._config['volatile_config'] = self.volatile_config 556 557 self.options = options 558 self.args = args 559 return (self.options, self.args) 560 561 def update_actions(self): 562 """ Update actions after reading in config. 563 564 Seems a little complex, but the logic goes: 565 566 First, if default_actions is specified in the config, set our 567 default actions even if the script specifies other default actions. 568 569 Without any other action-specific options, run with default actions. 570 571 If we specify --ACTION or --only-ACTION once or multiple times, 572 we want to override the default_actions list with the one(s) we list. 573 574 Otherwise, if we specify --add-action ACTION, we want to add an 575 action to the list. 576 577 Finally, if we specify --no-ACTION, remove that from the list of 578 actions to perform. 579 """ 580 if self._config.get('default_actions'): 581 default_actions = self.verify_actions(self._config['default_actions']) 582 self.default_actions = default_actions 583 self.verify_actions_order(self.default_actions) 584 self.actions = self.default_actions[:] 585 if self.volatile_config['actions']: 586 actions = self.verify_actions(self.volatile_config['actions']) 587 self.actions = actions 588 elif self.volatile_config['add_actions']: 589 actions = self.verify_actions(self.volatile_config['add_actions']) 590 self.actions.extend(actions) 591 if self.volatile_config['no_actions']: 592 actions = self.verify_actions(self.volatile_config['no_actions']) 593 for action in actions: 594 if action in self.actions: 595 self.actions.remove(action) 596 597 598# __main__ {{{1 599if __name__ == '__main__': 600 pass 601