1# Copyright 2012 Red Hat, Inc. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); you may 4# not use this file except in compliance with the License. You may obtain 5# a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12# License for the specific language governing permissions and limitations 13# under the License. 14 15"""Primary module in oslo_config. 16""" 17 18import argparse 19import collections 20from collections import abc 21import copy 22import enum 23import errno 24import functools 25import glob 26import inspect 27import itertools 28import logging 29import os 30import string 31import sys 32 33# NOTE(bnemec): oslo.log depends on oslo.config, so we can't 34# have a hard dependency on oslo.log. However, in most cases 35# oslo.log will be installed so we can use it. 36try: 37 import oslo_log 38except ImportError: 39 oslo_log = None 40 41from oslo_config import iniparser 42from oslo_config import sources 43# Absolute import to avoid circular import in Python 2.7 44import oslo_config.sources._environment as _environment 45from oslo_config import types 46 47import stevedore 48 49LOG = logging.getLogger(__name__) 50 51_SOURCE_DRIVER_OPTION_HELP = ( 52 'The name of the driver that can load this ' 53 'configuration source.' 54) 55 56 57class Locations(enum.Enum): 58 opt_default = (1, False) 59 set_default = (2, False) 60 set_override = (3, False) 61 user = (4, True) 62 command_line = (5, True) 63 environment = (6, True) 64 65 def __init__(self, num, is_user_controlled): 66 self.num = num 67 self.is_user_controlled = is_user_controlled 68 69 70LocationInfo = collections.namedtuple('LocationInfo', ['location', 'detail']) 71 72 73class Error(Exception): 74 """Base class for cfg exceptions.""" 75 76 def __init__(self, msg=None): 77 self.msg = msg 78 79 def __str__(self): 80 return self.msg 81 82 83class NotInitializedError(Error): 84 """Raised if parser is not initialized yet.""" 85 86 def __str__(self): 87 return "call expression on parser has not been invoked" 88 89 90class ArgsAlreadyParsedError(Error): 91 """Raised if a CLI opt is registered after parsing.""" 92 93 def __str__(self): 94 ret = "arguments already parsed" 95 if self.msg: 96 ret += ": " + self.msg 97 return ret 98 99 100class NoSuchOptError(Error, AttributeError): 101 """Raised if an opt which doesn't exist is referenced.""" 102 103 def __init__(self, opt_name, group=None): 104 self.opt_name = opt_name 105 self.group = group 106 107 def __str__(self): 108 group_name = 'DEFAULT' if self.group is None else self.group.name 109 return "no such option %s in group [%s]" % (self.opt_name, group_name) 110 111 112class NoSuchGroupError(Error): 113 """Raised if a group which doesn't exist is referenced.""" 114 115 def __init__(self, group_name): 116 self.group_name = group_name 117 118 def __str__(self): 119 return "no such group [%s]" % self.group_name 120 121 122class DuplicateOptError(Error): 123 """Raised if multiple opts with the same name are registered.""" 124 125 def __init__(self, opt_name): 126 self.opt_name = opt_name 127 128 def __str__(self): 129 return "duplicate option: %s" % self.opt_name 130 131 132class RequiredOptError(Error): 133 """Raised if an option is required but no value is supplied by the user.""" 134 135 def __init__(self, opt_name, group=None): 136 self.opt_name = opt_name 137 self.group = group 138 139 def __str__(self): 140 group_name = 'DEFAULT' if self.group is None else self.group.name 141 return "value required for option %s in group [%s]" % (self.opt_name, 142 group_name) 143 144 145class TemplateSubstitutionError(Error): 146 """Raised if an error occurs substituting a variable in an opt value.""" 147 148 def __str__(self): 149 return "template substitution error: %s" % self.msg 150 151 152class ConfigFilesNotFoundError(Error): 153 """Raised if one or more config files are not found.""" 154 155 def __init__(self, config_files): 156 self.config_files = config_files 157 158 def __str__(self): 159 return ('Failed to find some config files: %s' % 160 ",".join(self.config_files)) 161 162 163class ConfigFilesPermissionDeniedError(Error): 164 """Raised if one or more config files are not readable.""" 165 166 def __init__(self, config_files): 167 self.config_files = config_files 168 169 def __str__(self): 170 return ('Failed to open some config files: %s' % 171 ",".join(self.config_files)) 172 173 174class ConfigDirNotFoundError(Error): 175 """Raised if the requested config-dir is not found.""" 176 177 def __init__(self, config_dir): 178 self.config_dir = config_dir 179 180 def __str__(self): 181 return ('Failed to read config file directory: %s' % self.config_dir) 182 183 184class ConfigFileParseError(Error): 185 """Raised if there is an error parsing a config file.""" 186 187 def __init__(self, config_file, msg): 188 self.config_file = config_file 189 self.msg = msg 190 191 def __str__(self): 192 return 'Failed to parse %s: %s' % (self.config_file, self.msg) 193 194 195class ConfigSourceValueError(Error, ValueError): 196 """Raised if a config source value does not match its opt type.""" 197 pass 198 199 200class ConfigFileValueError(ConfigSourceValueError): 201 """Raised if a config file value does not match its opt type.""" 202 pass 203 204 205class DefaultValueError(Error, ValueError): 206 """Raised if a default config type does not fit the opt type.""" 207 208 209def _fixpath(p): 210 """Apply tilde expansion and absolutization to a path.""" 211 return os.path.abspath(os.path.expanduser(p)) 212 213 214def _get_config_dirs(project=None): 215 """Return a list of directories where config files may be located. 216 217 :param project: an optional project name 218 219 If a project is specified, following directories are returned:: 220 221 ~/.${project}/ 222 ~/ 223 /usr/local/etc/${project}/ 224 /usr/local/etc/ 225 226 If a project is specified and installed from a snap package, following 227 directories are also returned: 228 229 ${SNAP_COMMON}/usr/local/etc/${project} 230 ${SNAP}/usr/local/etc/${project} 231 232 Otherwise, if project is not specified, these directories are returned: 233 234 ~/ 235 /usr/local/etc/ 236 """ 237 snap = os.environ.get('SNAP') 238 snap_c = os.environ.get('SNAP_COMMON') 239 240 cfg_dirs = [ 241 _fixpath(os.path.join('~', '.' + project)) if project else None, 242 _fixpath('~'), 243 os.path.join('/usr/local/etc', project) if project else None, 244 '/usr/local/etc', 245 os.path.join(snap_c, "etc", project) if snap_c and project else None, 246 os.path.join(snap, "etc", project) if snap and project else None, 247 ] 248 return [x for x in cfg_dirs if x] 249 250 251def _search_dirs(dirs, basename, extension=""): 252 """Search a list of directories for a given filename or directory name. 253 254 Iterator over the supplied directories, returning the first file 255 found with the supplied name and extension. 256 257 :param dirs: a list of directories 258 :param basename: the filename or directory name, for example 'glance-api' 259 :param extension: the file extension, for example '.conf' 260 :returns: the path to a matching file or directory, or None 261 """ 262 for d in dirs: 263 path = os.path.join(d, '%s%s' % (basename, extension)) 264 if os.path.exists(path): 265 return path 266 267 268def _find_config_files(project, prog, extension): 269 if prog is None: 270 prog = os.path.basename(sys.argv[0]) 271 if prog.endswith(".py"): 272 prog = prog[:-3] 273 274 cfg_dirs = _get_config_dirs(project) 275 config_files = (_search_dirs(cfg_dirs, p, extension) 276 for p in [project, prog] if p) 277 278 return [x for x in config_files if x] 279 280 281def find_config_files(project=None, prog=None, extension='.conf'): 282 """Return a list of default configuration files. 283 284 :param project: an optional project name 285 :param prog: the program name, defaulting to the basename of 286 sys.argv[0], without extension .py 287 :param extension: the type of the config file 288 289 We default to two config files: [${project}.conf, ${prog}.conf] 290 291 And we look for those config files in the following directories:: 292 293 ~/.${project}/ 294 ~/ 295 /usr/local/etc/${project}/ 296 /usr/local/etc/ 297 ${SNAP_COMMON}/usr/local/etc/${project} 298 ${SNAP}/usr/local/etc/${project} 299 300 We return an absolute path for (at most) one of each the default config 301 files, for the topmost directory it exists in. 302 303 For example, if project=foo, prog=bar and /usr/local/etc/foo/foo.conf, /usr/local/etc/bar.conf 304 and ~/.foo/bar.conf all exist, then we return ['/usr/local/etc/foo/foo.conf', 305 '~/.foo/bar.conf'] 306 307 If no project name is supplied, we only look for ${prog}.conf. 308 """ 309 return _find_config_files(project, prog, extension) 310 311 312def find_config_dirs(project=None, prog=None, extension='.conf.d'): 313 """Return a list of default configuration dirs. 314 315 :param project: an optional project name 316 :param prog: the program name, defaulting to the basename of 317 sys.argv[0], without extension .py 318 :param extension: the type of the config directory. Defaults to '.conf.d' 319 320 We default to two config dirs: [${project}.conf.d/, ${prog}.conf.d/]. 321 If no project name is supplied, we only look for ${prog.conf.d/}. 322 323 And we look for those config dirs in the following directories:: 324 325 ~/.${project}/ 326 ~/ 327 /usr/local/etc/${project}/ 328 /usr/local/etc/ 329 ${SNAP_COMMON}/usr/local/etc/${project} 330 ${SNAP}/usr/local/etc/${project} 331 332 We return an absolute path for each of the two config dirs, 333 in the first place we find it (iff we find it). 334 335 For example, if project=foo, prog=bar and /usr/local/etc/foo/foo.conf.d/, 336 /usr/local/etc/bar.conf.d/ and ~/.foo/bar.conf.d/ all exist, then we return 337 ['/usr/local/etc/foo/foo.conf.d/', '~/.foo/bar.conf.d/'] 338 """ 339 return _find_config_files(project, prog, extension) 340 341 342def _is_opt_registered(opts, opt): 343 """Check whether an opt with the same name is already registered. 344 345 The same opt may be registered multiple times, with only the first 346 registration having any effect. However, it is an error to attempt 347 to register a different opt with the same name. 348 349 :param opts: the set of opts already registered 350 :param opt: the opt to be registered 351 :returns: True if the opt was previously registered, False otherwise 352 :raises: DuplicateOptError if a naming conflict is detected 353 """ 354 if opt.dest in opts: 355 if opts[opt.dest]['opt'] != opt: 356 raise DuplicateOptError(opt.name) 357 return True 358 else: 359 return False 360 361 362_show_caller_details = bool(os.environ.get( 363 'OSLO_CONFIG_SHOW_CODE_LOCATIONS')) 364 365 366def _get_caller_detail(n=2): 367 """Return a string describing where this is being called from. 368 369 :param n: Number of steps up the stack to look. Defaults to ``2``. 370 :type n: int 371 :returns: str 372 """ 373 if not _show_caller_details: 374 return None 375 s = inspect.stack()[:n + 1] 376 try: 377 frame = s[n] 378 try: 379 return frame[1] 380 # WARNING(dhellmann): Using frame.lineno to include the 381 # line number in the return value causes some sort of 382 # memory or stack corruption that manifests in values not 383 # being cleaned up in the cfgfilter tests. 384 # return '%s:%s' % (frame[1], frame[2]) 385 finally: 386 del frame 387 finally: 388 del s 389 390 391def set_defaults(opts, **kwargs): 392 for opt in opts: 393 if opt.dest in kwargs: 394 opt.default = kwargs[opt.dest] 395 opt._set_location = LocationInfo(Locations.set_default, 396 _get_caller_detail()) 397 398 399def _normalize_group_name(group_name): 400 if group_name == 'DEFAULT': 401 return group_name 402 return group_name.lower() 403 404 405def _report_deprecation(format_str, format_dict): 406 """Report use of a deprecated option 407 408 Uses versionutils from oslo.log if it is available. If not, logs 409 a simple warning message. 410 411 :param format_str: The message to use for the report 412 :param format_dict: A dict containing keys for any parameters in format_str 413 """ 414 if oslo_log: 415 # We can't import versionutils at the module level because of circular 416 # imports. Importing just oslo_log at the module level and 417 # versionutils locally allows us to unit test this and still avoid the 418 # circular problem. 419 from oslo_log import versionutils 420 versionutils.report_deprecated_feature(LOG, format_str, 421 format_dict) 422 else: 423 LOG.warning(format_str, format_dict) 424 425 426@functools.total_ordering 427class Opt: 428 429 """Base class for all configuration options. 430 431 The only required parameter is the option's name. However, it is 432 common to also supply a default and help string for all options. 433 434 :param name: the option's name 435 :param type: the option's type. Must be a callable object that takes string 436 and returns converted and validated value 437 :param dest: the name of the corresponding :class:`.ConfigOpts` property 438 :param short: a single character CLI option name 439 :param default: the default value of the option 440 :param positional: True if the option is a positional CLI argument 441 :param metavar: the option argument to show in --help 442 :param help: an explanation of how the option is used 443 :param secret: true if the value should be obfuscated in log output 444 :param required: true if a value must be supplied for this option 445 :param deprecated_name: deprecated name option. Acts like an alias 446 :param deprecated_group: the group containing a deprecated alias 447 :param deprecated_opts: list of :class:`.DeprecatedOpt` 448 :param sample_default: a default string for sample config files 449 :param deprecated_for_removal: indicates whether this opt is planned for 450 removal in a future release 451 :param deprecated_reason: indicates why this opt is planned for removal in 452 a future release. Silently ignored if 453 deprecated_for_removal is False 454 :param deprecated_since: indicates which release this opt was deprecated 455 in. Accepts any string, though valid version 456 strings are encouraged. Silently ignored if 457 deprecated_for_removal is False 458 :param mutable: True if this option may be reloaded 459 :param advanced: a bool True/False value if this option has advanced usage 460 and is not normally used by the majority of users 461 462 An Opt object has no public methods, but has a number of public properties: 463 464 .. py:attribute:: name 465 466 the name of the option, which may include hyphens 467 468 .. py:attribute:: type 469 470 a callable object that takes string and returns converted and 471 validated value. Default types are available from 472 :class:`oslo_config.types` 473 474 .. py:attribute:: dest 475 476 the (hyphen-less) :class:`.ConfigOpts` property which contains the 477 option value 478 479 .. py:attribute:: short 480 481 a single character CLI option name 482 483 .. py:attribute:: default 484 485 the default value of the option 486 487 .. py:attribute:: sample_default 488 489 a sample default value string to include in sample config files 490 491 .. py:attribute:: positional 492 493 True if the option is a positional CLI argument 494 495 .. py:attribute:: metavar 496 497 the name shown as the argument to a CLI option in --help output 498 499 .. py:attribute:: help 500 501 a string explaining how the option's value is used 502 503 .. py:attribute:: advanced 504 505 in sample files, a bool value indicating the option is advanced 506 507 .. versionchanged:: 1.2 508 Added *deprecated_opts* parameter. 509 510 .. versionchanged:: 1.4 511 Added *sample_default* parameter. 512 513 .. versionchanged:: 1.9 514 Added *deprecated_for_removal* parameter. 515 516 .. versionchanged:: 2.7 517 An exception is now raised if the default value has the wrong type. 518 519 .. versionchanged:: 3.2 520 Added *deprecated_reason* parameter. 521 522 .. versionchanged:: 3.5 523 Added *mutable* parameter. 524 525 .. versionchanged:: 3.12 526 Added *deprecated_since* parameter. 527 528 .. versionchanged:: 3.15 529 Added *advanced* parameter and attribute. 530 """ 531 multi = False 532 533 def __init__(self, name, type=None, dest=None, short=None, 534 default=None, positional=False, metavar=None, help=None, 535 secret=False, required=None, 536 deprecated_name=None, deprecated_group=None, 537 deprecated_opts=None, sample_default=None, 538 deprecated_for_removal=False, deprecated_reason=None, 539 deprecated_since=None, mutable=False, advanced=False): 540 if name.startswith('_'): 541 raise ValueError('illegal name %s with prefix _' % (name,)) 542 self.name = name 543 544 if type is None: 545 type = types.String() 546 547 if not callable(type): 548 raise TypeError('type must be callable') 549 self.type = type 550 551 # By default, non-positional options are *optional*, and positional 552 # options are *required*. 553 if required is None: 554 required = True if positional else False 555 556 if dest is None: 557 self.dest = self.name.replace('-', '_') 558 else: 559 self.dest = dest 560 self.short = short 561 self.default = default 562 self.sample_default = sample_default 563 self.positional = positional 564 self.metavar = metavar 565 self.help = help 566 self.secret = secret 567 self.required = required 568 self.deprecated_for_removal = deprecated_for_removal 569 self.deprecated_reason = deprecated_reason 570 self.deprecated_since = deprecated_since 571 self._logged_deprecation = False 572 573 if self.__class__ is Opt: 574 stack_depth = 2 # someone instantiated Opt directly 575 else: 576 stack_depth = 3 # skip the call to the child class constructor 577 self._set_location = LocationInfo( 578 Locations.opt_default, 579 _get_caller_detail(stack_depth), 580 ) 581 582 self.deprecated_opts = copy.deepcopy(deprecated_opts) or [] 583 for o in self.deprecated_opts: 584 if '-' in o.name: 585 self.deprecated_opts.append(DeprecatedOpt( 586 o.name.replace('-', '_'), 587 group=o.group)) 588 if deprecated_name is not None or deprecated_group is not None: 589 self.deprecated_opts.append(DeprecatedOpt(deprecated_name, 590 group=deprecated_group)) 591 if deprecated_name and '-' in deprecated_name: 592 self.deprecated_opts.append(DeprecatedOpt( 593 deprecated_name.replace('-', '_'), 594 group=deprecated_group)) 595 self._check_default() 596 597 self.mutable = mutable 598 self.advanced = advanced 599 600 def _default_is_ref(self): 601 """Check if default is a reference to another var.""" 602 if isinstance(self.default, str): 603 tmpl = self.default.replace(r'\$', '').replace('$$', '') 604 return '$' in tmpl 605 return False 606 607 def _check_default(self): 608 if (self.default is not None 609 and not self._default_is_ref()): 610 try: 611 self.type(self.default) 612 except Exception: 613 raise DefaultValueError("Error processing default value " 614 "%(default)s for Opt type of %(opt)s." 615 % {'default': self.default, 616 'opt': self.type}) 617 618 def _vars_for_cmp(self): 619 # NOTE(dhellmann): Get the instance variables of this Opt and 620 # then make a new dictionary so we can modify the contents 621 # before returning it without removing any attributes of the 622 # object. 623 v = dict(vars(self)) 624 625 # NOTE(dhellmann): Ignore the location where the option is 626 # defined when comparing them. Ideally we could use this to 627 # detect duplicate settings in code bases, but as long as the 628 # options match otherwise they should be safe. 629 if '_set_location' in v: 630 del v['_set_location'] 631 632 return v 633 634 def __ne__(self, another): 635 return self._vars_for_cmp() != another._vars_for_cmp() 636 637 def __eq__(self, another): 638 return self._vars_for_cmp() == another._vars_for_cmp() 639 640 __hash__ = object.__hash__ 641 642 def _get_from_namespace(self, namespace, group_name): 643 """Retrieves the option value from a _Namespace object. 644 645 :param namespace: a _Namespace object 646 :param group_name: a group name 647 """ 648 names = [(group_name, self.dest)] 649 current_name = (group_name, self.name) 650 651 for opt in self.deprecated_opts: 652 dname, dgroup = opt.name, opt.group 653 if dname or dgroup: 654 names.append((dgroup if dgroup else group_name, 655 dname if dname else self.dest)) 656 657 value, loc = namespace._get_value( 658 names, multi=self.multi, 659 positional=self.positional, current_name=current_name) 660 # The previous line will raise a KeyError if no value is set in the 661 # config file, so we'll only log deprecations for set options. 662 if self.deprecated_for_removal and not self._logged_deprecation: 663 self._logged_deprecation = True 664 pretty_group = group_name or 'DEFAULT' 665 if self.deprecated_reason: 666 pretty_reason = ' ({})'.format(self.deprecated_reason) 667 else: 668 pretty_reason = '' 669 format_str = ('Option "%(option)s" from group "%(group)s" is ' 670 'deprecated for removal%(reason)s. Its value may ' 671 'be silently ignored in the future.') 672 format_dict = {'option': self.dest, 673 'group': pretty_group, 674 'reason': pretty_reason} 675 _report_deprecation(format_str, format_dict) 676 return (value, loc) 677 678 def _add_to_cli(self, parser, group=None): 679 """Makes the option available in the command line interface. 680 681 This is the method ConfigOpts uses to add the opt to the CLI interface 682 as appropriate for the opt type. Some opt types may extend this method, 683 others may just extend the helper methods it uses. 684 685 :param parser: the CLI option parser 686 :param group: an optional OptGroup object 687 """ 688 container = self._get_argparse_container(parser, group) 689 kwargs = self._get_argparse_kwargs(group) 690 prefix = self._get_argparse_prefix('', group.name if group else None) 691 deprecated_names = [] 692 for opt in self.deprecated_opts: 693 deprecated_name = self._get_deprecated_cli_name(opt.name, 694 opt.group) 695 if deprecated_name is not None: 696 deprecated_names.append(deprecated_name) 697 self._add_to_argparse(parser, container, self.name, self.short, 698 kwargs, prefix, 699 self.positional, deprecated_names) 700 701 def _add_to_argparse(self, parser, container, name, short, kwargs, 702 prefix='', positional=False, deprecated_names=None): 703 """Add an option to an argparse parser or group. 704 705 :param container: an argparse._ArgumentGroup object 706 :param name: the opt name 707 :param short: the short opt name 708 :param kwargs: the keyword arguments for add_argument() 709 :param prefix: an optional prefix to prepend to the opt name 710 :param positional: whether the option is a positional CLI argument 711 :param deprecated_names: list of deprecated option names 712 """ 713 def hyphen(arg): 714 return arg if not positional else '' 715 716 # Because we must omit the dest parameter when using a positional 717 # argument, the name supplied for the positional argument must not 718 # include hyphens. 719 if positional: 720 prefix = prefix.replace('-', '_') 721 name = name.replace('-', '_') 722 723 args = [hyphen('--') + prefix + name] 724 if short: 725 args.append(hyphen('-') + short) 726 for deprecated_name in deprecated_names: 727 args.append(hyphen('--') + deprecated_name) 728 729 parser.add_parser_argument(container, *args, **kwargs) 730 731 def _get_argparse_container(self, parser, group): 732 """Returns an argparse._ArgumentGroup. 733 734 :param parser: an argparse.ArgumentParser 735 :param group: an (optional) OptGroup object 736 :returns: an argparse._ArgumentGroup if group is given, else parser 737 """ 738 if group is not None: 739 return group._get_argparse_group(parser) 740 else: 741 return parser 742 743 def _get_argparse_kwargs(self, group, **kwargs): 744 r"""Build a dict of keyword arguments for argparse's add_argument(). 745 746 Most opt types extend this method to customize the behaviour of the 747 options added to argparse. 748 749 :param group: an optional group 750 :param \*\*kwargs: optional keyword arguments to add to 751 :returns: a dict of keyword arguments 752 """ 753 if not self.positional: 754 dest = self.dest 755 if group is not None: 756 dest = group.name + '_' + dest 757 kwargs['dest'] = dest 758 elif not self.required: 759 kwargs['nargs'] = '?' 760 kwargs.update({'default': None, 761 'metavar': self.metavar, 762 'help': self.help, }) 763 return kwargs 764 765 def _get_argparse_prefix(self, prefix, group_name): 766 """Build a prefix for the CLI option name, if required. 767 768 CLI options in a group are prefixed with the group's name in order 769 to avoid conflicts between similarly named options in different 770 groups. 771 772 :param prefix: an existing prefix to append to (for example 'no' or '') 773 :param group_name: an optional group name 774 :returns: a CLI option prefix including the group name, if appropriate 775 """ 776 if group_name is not None: 777 return group_name + '-' + prefix 778 else: 779 return prefix 780 781 def _get_deprecated_cli_name(self, dname, dgroup, prefix=''): 782 """Build a CLi arg name for deprecated options. 783 784 Either a deprecated name or a deprecated group or both or 785 neither can be supplied: 786 787 dname, dgroup -> dgroup + '-' + dname 788 dname -> dname 789 dgroup -> dgroup + '-' + self.name 790 neither -> None 791 792 :param dname: a deprecated name, which can be None 793 :param dgroup: a deprecated group, which can be None 794 :param prefix: an prefix to append to (for example 'no' or '') 795 :returns: a CLI argument name 796 """ 797 if dgroup == 'DEFAULT': 798 dgroup = None 799 800 if dname is None and dgroup is None: 801 return None 802 803 if dname is None: 804 dname = self.name 805 806 return self._get_argparse_prefix(prefix, dgroup) + dname 807 808 def __lt__(self, another): 809 return hash(self) < hash(another) 810 811 812class DeprecatedOpt: 813 814 """Represents a Deprecated option. 815 816 Here's how you can use it:: 817 818 oldopts = [cfg.DeprecatedOpt('oldopt1', group='group1'), 819 cfg.DeprecatedOpt('oldopt2', group='group2')] 820 cfg.CONF.register_group(cfg.OptGroup('group1')) 821 cfg.CONF.register_opt(cfg.StrOpt('newopt', deprecated_opts=oldopts), 822 group='group1') 823 824 For options which have a single value (like in the example above), 825 if the new option is present ("[group1]/newopt" above), it will override 826 any deprecated options present ("[group1]/oldopt1" and "[group2]/oldopt2" 827 above). 828 829 If no group is specified for a DeprecatedOpt option (i.e. the group is 830 None), lookup will happen within the same group the new option is in. 831 For example, if no group was specified for the second option 'oldopt2' in 832 oldopts list:: 833 834 oldopts = [cfg.DeprecatedOpt('oldopt1', group='group1'), 835 cfg.DeprecatedOpt('oldopt2')] 836 cfg.CONF.register_group(cfg.OptGroup('group1')) 837 cfg.CONF.register_opt(cfg.StrOpt('newopt', deprecated_opts=oldopts), 838 group='group1') 839 840 then lookup for that option will happen in group 'group1'. 841 842 If the new option is not present and multiple deprecated options are 843 present, the option corresponding to the first element of deprecated_opts 844 will be chosen. 845 846 Multi-value options will return all new and deprecated 847 options. So if we have a multi-value option "[group1]/opt1" whose 848 deprecated option is "[group2]/opt2", and the conf file has both these 849 options specified like so:: 850 851 [group1] 852 opt1=val10,val11 853 854 [group2] 855 opt2=val21,val22 856 857 Then the value of "[group1]/opt1" will be ['val10', 'val11', 'val21', 858 'val22']. 859 860 .. versionadded:: 1.2 861 """ 862 863 def __init__(self, name, group=None): 864 """Constructs an DeprecatedOpt object. 865 866 :param name: the name of the option 867 :param group: the group of the option 868 """ 869 self.name = name 870 self.group = group 871 872 def __key(self): 873 return (self.name, self.group) 874 875 def __eq__(self, other): 876 return self.__key() == other.__key() 877 878 def __hash__(self): 879 return hash(self.__key()) 880 881 882class StrOpt(Opt): 883 r"""Option with String type 884 885 Option with ``type`` :class:`oslo_config.types.String` 886 887 :param name: the option's name 888 :param choices: Optional sequence of either valid values or tuples of valid 889 values with descriptions. 890 :param quotes: If True and string is enclosed with single or double 891 quotes, will strip those quotes. 892 :param regex: Optional regular expression (string or compiled 893 regex) that the value must match on an unanchored 894 search. 895 :param ignore_case: If True case differences (uppercase vs. lowercase) 896 between 'choices' or 'regex' will be ignored. 897 :param max_length: If positive integer, the value must be less than or 898 equal to this parameter. 899 :param \*\*kwargs: arbitrary keyword arguments passed to :class:`Opt` 900 901 .. versionchanged:: 2.7 902 Added *quotes* parameter 903 904 .. versionchanged:: 2.7 905 Added *regex* parameter 906 907 .. versionchanged:: 2.7 908 Added *ignore_case* parameter 909 910 .. versionchanged:: 2.7 911 Added *max_length* parameter 912 913 .. versionchanged:: 5.2 914 The *choices* parameter will now accept a sequence of tuples, where each 915 tuple is of form (*choice*, *description*) 916 """ 917 918 def __init__(self, name, choices=None, quotes=None, 919 regex=None, ignore_case=False, max_length=None, **kwargs): 920 super(StrOpt, self).__init__(name, 921 type=types.String( 922 choices=choices, 923 quotes=quotes, 924 regex=regex, 925 ignore_case=ignore_case, 926 max_length=max_length), 927 **kwargs) 928 929 def _get_choice_text(self, choice): 930 if choice is None: 931 return '<None>' 932 elif choice == '': 933 return "''" 934 return str(choice) 935 936 def _get_argparse_kwargs(self, group, **kwargs): 937 """Extends the base argparse keyword dict for the config dir option.""" 938 kwargs = super(StrOpt, self)._get_argparse_kwargs(group) 939 940 if getattr(self.type, 'choices', None): 941 choices_text = ', '.join([self._get_choice_text(choice) 942 for choice in self.type.choices]) 943 if kwargs['help'] is None: 944 kwargs['help'] = '' 945 946 kwargs['help'].rstrip('\n') 947 kwargs['help'] += '\n Allowed values: %s\n' % choices_text 948 949 return kwargs 950 951 952class BoolOpt(Opt): 953 954 r"""Boolean options. 955 956 Bool opts are set to True or False on the command line using --optname or 957 --nooptname respectively. 958 959 In config files, boolean values are cast with Boolean type. 960 961 :param name: the option's name 962 :param \*\*kwargs: arbitrary keyword arguments passed to :class:`Opt` 963 """ 964 965 def __init__(self, name, **kwargs): 966 if 'positional' in kwargs: 967 raise ValueError('positional boolean args not supported') 968 super(BoolOpt, self).__init__(name, type=types.Boolean(), **kwargs) 969 970 def _add_to_cli(self, parser, group=None): 971 """Extends the base class method to add the --nooptname option.""" 972 super(BoolOpt, self)._add_to_cli(parser, group) 973 self._add_inverse_to_argparse(parser, group) 974 975 def _add_inverse_to_argparse(self, parser, group): 976 """Add the --nooptname option to the option parser.""" 977 container = self._get_argparse_container(parser, group) 978 kwargs = self._get_argparse_kwargs(group, action='store_false') 979 prefix = self._get_argparse_prefix('no', group.name if group else None) 980 deprecated_names = [] 981 for opt in self.deprecated_opts: 982 deprecated_name = self._get_deprecated_cli_name(opt.name, 983 opt.group, 984 prefix='no') 985 if deprecated_name is not None: 986 deprecated_names.append(deprecated_name) 987 kwargs["help"] = "The inverse of --" + self.name 988 self._add_to_argparse(parser, container, self.name, None, kwargs, 989 prefix, self.positional, deprecated_names) 990 991 def _get_argparse_kwargs(self, group, action='store_true', **kwargs): 992 """Extends the base argparse keyword dict for boolean options.""" 993 994 kwargs = super(BoolOpt, self)._get_argparse_kwargs(group, **kwargs) 995 # type has no effect for BoolOpt, it only matters for 996 # values that came from config files 997 if 'type' in kwargs: 998 del kwargs['type'] 999 1000 # metavar has no effect for BoolOpt 1001 if 'metavar' in kwargs: 1002 del kwargs['metavar'] 1003 1004 kwargs['action'] = action 1005 1006 return kwargs 1007 1008 1009class IntOpt(Opt): 1010 1011 r"""Option with Integer type 1012 1013 Option with ``type`` :class:`oslo_config.types.Integer` 1014 1015 :param name: the option's name 1016 :param min: minimum value the integer can take 1017 :param max: maximum value the integer can take 1018 :param \*\*kwargs: arbitrary keyword arguments passed to :class:`Opt` 1019 1020 .. versionchanged:: 1.15 1021 1022 Added *min* and *max* parameters. 1023 """ 1024 1025 def __init__(self, name, min=None, max=None, **kwargs): 1026 super(IntOpt, self).__init__(name, type=types.Integer(min, max), 1027 **kwargs) 1028 1029 1030class FloatOpt(Opt): 1031 1032 r"""Option with Float type 1033 1034 Option with ``type`` :class:`oslo_config.types.Float` 1035 1036 :param name: the option's name 1037 :param min: minimum value the float can take 1038 :param max: maximum value the float can take 1039 :param \*\*kwargs: arbitrary keyword arguments passed to :class:`Opt` 1040 1041 .. versionchanged:: 3.14 1042 1043 Added *min* and *max* parameters. 1044 """ 1045 1046 def __init__(self, name, min=None, max=None, **kwargs): 1047 super(FloatOpt, self).__init__(name, type=types.Float(min, max), 1048 **kwargs) 1049 1050 1051class ListOpt(Opt): 1052 1053 r"""Option with List(String) type 1054 1055 Option with ``type`` :class:`oslo_config.types.List` 1056 1057 :param name: the option's name 1058 :param item_type: type of items (see :class:`oslo_config.types`) 1059 :param bounds: if True the value should be inside "[" and "]" pair 1060 :param \*\*kwargs: arbitrary keyword arguments passed to :class:`Opt` 1061 1062 .. versionchanged:: 2.5 1063 Added *item_type* and *bounds* parameters. 1064 """ 1065 1066 def __init__(self, name, item_type=None, bounds=None, **kwargs): 1067 super(ListOpt, self).__init__(name, 1068 type=types.List(item_type=item_type, 1069 bounds=bounds), 1070 **kwargs) 1071 1072 1073class DictOpt(Opt): 1074 1075 r"""Option with Dict(String) type 1076 1077 Option with ``type`` :class:`oslo_config.types.Dict` 1078 1079 :param name: the option's name 1080 :param \*\*kwargs: arbitrary keyword arguments passed to :class:`Opt` 1081 1082 .. versionadded:: 1.2 1083 """ 1084 1085 def __init__(self, name, **kwargs): 1086 super(DictOpt, self).__init__(name, type=types.Dict(), **kwargs) 1087 1088 1089class IPOpt(Opt): 1090 1091 r"""Opt with IPAddress type 1092 1093 Option with ``type`` :class:`oslo_config.types.IPAddress` 1094 1095 :param name: the option's name 1096 :param version: one of either ``4``, ``6``, or ``None`` to specify 1097 either version. 1098 :param \*\*kwargs: arbitrary keyword arguments passed to :class:`Opt` 1099 1100 .. versionadded:: 1.4 1101 """ 1102 1103 def __init__(self, name, version=None, **kwargs): 1104 super(IPOpt, self).__init__(name, type=types.IPAddress(version), 1105 **kwargs) 1106 1107 1108class PortOpt(Opt): 1109 1110 r"""Option for a TCP/IP port number. Ports can range from 0 to 65535. 1111 1112 Option with ``type`` :class:`oslo_config.types.Integer` 1113 1114 :param name: the option's name 1115 :param min: minimum value the port can take 1116 :param max: maximum value the port can take 1117 :param choices: Optional sequence of either valid values or tuples of valid 1118 values with descriptions. 1119 :param \*\*kwargs: arbitrary keyword arguments passed to :class:`Opt` 1120 1121 .. versionadded:: 2.6 1122 .. versionchanged:: 3.2 1123 Added *choices* parameter. 1124 .. versionchanged:: 3.4 1125 Allow port number with 0. 1126 .. versionchanged:: 3.16 1127 Added *min* and *max* parameters. 1128 .. versionchanged:: 5.2 1129 The *choices* parameter will now accept a sequence of tuples, where each 1130 tuple is of form (*choice*, *description*) 1131 """ 1132 1133 def __init__(self, name, min=None, max=None, choices=None, **kwargs): 1134 type = types.Port(min=min, max=max, choices=choices, 1135 type_name='port value') 1136 super(PortOpt, self).__init__(name, type=type, **kwargs) 1137 1138 1139class HostnameOpt(Opt): 1140 1141 r"""Option for a hostname. Only accepts valid hostnames. 1142 1143 Option with ``type`` :class:`oslo_config.types.Hostname` 1144 1145 :param name: the option's name 1146 :param \*\*kwargs: arbitrary keyword arguments passed to :class:`Opt` 1147 1148 .. versionadded:: 3.8 1149 """ 1150 1151 def __init__(self, name, **kwargs): 1152 super(HostnameOpt, self).__init__(name, type=types.Hostname(), 1153 **kwargs) 1154 1155 1156class HostAddressOpt(Opt): 1157 1158 r"""Option for either an IP or a hostname. 1159 1160 Accepts valid hostnames and valid IP addresses. 1161 1162 Option with ``type`` :class:`oslo_config.types.HostAddress` 1163 1164 :param name: the option's name 1165 :param version: one of either ``4``, ``6``, or ``None`` to specify 1166 either version. 1167 :param \*\*kwargs: arbitrary keyword arguments passed to :class:`Opt` 1168 1169 .. versionadded:: 3.22 1170 """ 1171 1172 def __init__(self, name, version=None, **kwargs): 1173 super(HostAddressOpt, self).__init__(name, 1174 type=types.HostAddress(version), 1175 **kwargs) 1176 1177 1178class HostDomainOpt(Opt): 1179 1180 r"""Option for either an IP or a hostname. 1181 1182 Like HostAddress with the support of _ character. 1183 1184 Option with ``type`` :class:`oslo_config.types.HostDomain` 1185 1186 :param name: the option's name 1187 :param version: one of either ``4``, ``6``, or ``None`` to specify 1188 either version. 1189 :param \*\*kwargs: arbitrary keyword arguments passed to :class:`Opt` 1190 1191 .. versionadded:: 8.6 1192 """ 1193 1194 def __init__(self, name, version=None, **kwargs): 1195 super(HostDomainOpt, self).__init__(name, 1196 type=types.HostDomain(version), 1197 **kwargs) 1198 1199 1200class URIOpt(Opt): 1201 1202 r"""Opt with URI type 1203 1204 Option with ``type`` :class:`oslo_config.types.URI` 1205 1206 :param name: the option's name 1207 :param max_length: If positive integer, the value must be less than or 1208 equal to this parameter. 1209 :param schemes: list of valid URI schemes, e.g. 'https', 'ftp', 'git' 1210 :param \*\*kwargs: arbitrary keyword arguments passed to :class:`Opt` 1211 1212 .. versionadded:: 3.12 1213 1214 .. versionchanged:: 3.14 1215 Added *max_length* parameter 1216 .. versionchanged:: 3.18 1217 Added *schemes* parameter 1218 """ 1219 1220 def __init__(self, name, max_length=None, schemes=None, **kwargs): 1221 type = types.URI(max_length=max_length, schemes=schemes) 1222 super(URIOpt, self).__init__(name, type=type, **kwargs) 1223 1224 1225class MultiOpt(Opt): 1226 1227 r"""Multi-value option. 1228 1229 Multi opt values are typed opts which may be specified multiple times. 1230 The opt value is a list containing all the values specified. 1231 1232 :param name: the option's name 1233 :param item_type: Type of items (see :class:`oslo_config.types`) 1234 :param \*\*kwargs: arbitrary keyword arguments passed to :class:`Opt` 1235 1236 For example:: 1237 1238 cfg.MultiOpt('foo', 1239 item_type=types.Integer(), 1240 default=None, 1241 help="Multiple foo option") 1242 1243 The command line ``--foo=1 --foo=2`` would result in ``cfg.CONF.foo`` 1244 containing ``[1,2]`` 1245 1246 .. versionadded:: 1.3 1247 """ 1248 multi = True 1249 1250 def __init__(self, name, item_type, **kwargs): 1251 super(MultiOpt, self).__init__(name, item_type, **kwargs) 1252 1253 def _get_argparse_kwargs(self, group, **kwargs): 1254 """Extends the base argparse keyword dict for multi value options.""" 1255 kwargs = super(MultiOpt, self)._get_argparse_kwargs(group) 1256 if not self.positional: 1257 kwargs['action'] = 'append' 1258 else: 1259 kwargs['nargs'] = '*' 1260 return kwargs 1261 1262 1263class MultiStrOpt(MultiOpt): 1264 1265 r"""MultiOpt with a MultiString ``item_type``. 1266 1267 MultiOpt with a default :class:`oslo_config.types.MultiString` item 1268 type. 1269 1270 :param name: the option's name 1271 :param \*\*kwargs: arbitrary keyword arguments passed to :class:`MultiOpt` 1272 """ 1273 1274 def __init__(self, name, **kwargs): 1275 super(MultiStrOpt, self).__init__(name, 1276 item_type=types.MultiString(), 1277 **kwargs) 1278 1279 1280class SubCommandOpt(Opt): 1281 1282 """Sub-command options. 1283 1284 Sub-command options allow argparse sub-parsers to be used to parse 1285 additional command line arguments. 1286 1287 The handler argument to the SubCommandOpt constructor is a callable 1288 which is supplied an argparse subparsers object. Use this handler 1289 callable to add sub-parsers. 1290 1291 The opt value is SubCommandAttr object with the name of the chosen 1292 sub-parser stored in the 'name' attribute and the values of other 1293 sub-parser arguments available as additional attributes. 1294 1295 :param name: the option's name 1296 :param dest: the name of the corresponding :class:`.ConfigOpts` property 1297 :param handler: callable which is supplied subparsers object when invoked 1298 :param title: title of the sub-commands group in help output 1299 :param description: description of the group in help output 1300 :param help: a help string giving an overview of available sub-commands 1301 """ 1302 1303 def __init__(self, name, dest=None, handler=None, 1304 title=None, description=None, help=None): 1305 """Construct an sub-command parsing option. 1306 1307 This behaves similarly to other Opt sub-classes but adds a 1308 'handler' argument. The handler is a callable which is supplied 1309 an subparsers object when invoked. The add_parser() method on 1310 this subparsers object can be used to register parsers for 1311 sub-commands. 1312 """ 1313 super(SubCommandOpt, self).__init__(name, type=types.String(), 1314 dest=dest, help=help) 1315 self.handler = handler 1316 self.title = title 1317 self.description = description 1318 1319 def _add_to_cli(self, parser, group=None): 1320 """Add argparse sub-parsers and invoke the handler method.""" 1321 dest = self.dest 1322 if group is not None: 1323 dest = group.name + '_' + dest 1324 1325 subparsers = parser.add_subparsers(dest=dest, 1326 title=self.title, 1327 description=self.description, 1328 help=self.help) 1329 # NOTE(jd) Set explicitly to True for Python 3 1330 # See http://bugs.python.org/issue9253 for context 1331 subparsers.required = True 1332 1333 if self.handler is not None: 1334 self.handler(subparsers) 1335 1336 1337class _ConfigFileOpt(Opt): 1338 1339 """The --config-file option. 1340 1341 This is an private option type which handles the special processing 1342 required for --config-file options. 1343 1344 As each --config-file option is encountered on the command line, we 1345 parse the file and store the parsed values in the _Namespace object. 1346 This allows us to properly handle the precedence of --config-file 1347 options over previous command line arguments, but not over subsequent 1348 arguments. 1349 1350 .. versionadded:: 1.2 1351 """ 1352 1353 class ConfigFileAction(argparse.Action): 1354 1355 """An argparse action for --config-file. 1356 1357 As each --config-file option is encountered, this action adds the 1358 value to the config_file attribute on the _Namespace object but also 1359 parses the configuration file and stores the values found also in 1360 the _Namespace object. 1361 """ 1362 1363 def __call__(self, parser, namespace, values, option_string=None): 1364 """Handle a --config-file command line argument. 1365 1366 :raises: ConfigFileParseError, ConfigFileValueError 1367 """ 1368 if getattr(namespace, self.dest, None) is None: 1369 setattr(namespace, self.dest, []) 1370 items = getattr(namespace, self.dest) 1371 items.append(values) 1372 1373 ConfigParser._parse_file(values, namespace) 1374 1375 def __init__(self, name, **kwargs): 1376 super(_ConfigFileOpt, self).__init__(name, lambda x: x, **kwargs) 1377 1378 def _get_argparse_kwargs(self, group, **kwargs): 1379 """Extends the base argparse keyword dict for the config file opt.""" 1380 kwargs = super(_ConfigFileOpt, self)._get_argparse_kwargs(group) 1381 kwargs['action'] = self.ConfigFileAction 1382 return kwargs 1383 1384 1385class _ConfigDirOpt(Opt): 1386 1387 """The --config-dir option. 1388 1389 This is an private option type which handles the special processing 1390 required for --config-dir options. 1391 1392 As each --config-dir option is encountered on the command line, we 1393 parse the files in that directory and store the parsed values in the 1394 _Namespace object. This allows us to properly handle the precedence of 1395 --config-dir options over previous command line arguments, but not 1396 over subsequent arguments. 1397 1398 .. versionadded:: 1.2 1399 """ 1400 1401 class ConfigDirAction(argparse.Action): 1402 1403 """An argparse action for --config-dir. 1404 1405 As each --config-dir option is encountered, this action sets the 1406 config_dir attribute on the _Namespace object but also parses the 1407 configuration files and stores the values found also in the 1408 _Namespace object. 1409 """ 1410 1411 def __call__(self, parser, namespace, values, option_string=None): 1412 """Handle a --config-dir command line argument. 1413 1414 :raises: ConfigFileParseError, ConfigFileValueError, 1415 ConfigDirNotFoundError 1416 """ 1417 namespace._config_dirs.append(values) 1418 setattr(namespace, self.dest, values) 1419 1420 values = os.path.expanduser(values) 1421 1422 if not os.path.exists(values): 1423 raise ConfigDirNotFoundError(values) 1424 1425 config_dir_glob = os.path.join(values, '*.conf') 1426 1427 for config_file in sorted(glob.glob(config_dir_glob)): 1428 ConfigParser._parse_file(config_file, namespace) 1429 1430 def __init__(self, name, **kwargs): 1431 super(_ConfigDirOpt, self).__init__(name, type=types.List(), 1432 **kwargs) 1433 1434 def _get_argparse_kwargs(self, group, **kwargs): 1435 """Extends the base argparse keyword dict for the config dir option.""" 1436 kwargs = super(_ConfigDirOpt, self)._get_argparse_kwargs(group) 1437 kwargs['action'] = self.ConfigDirAction 1438 return kwargs 1439 1440 1441class OptGroup: 1442 1443 """Represents a group of opts. 1444 1445 CLI opts in the group are automatically prefixed with the group name. 1446 1447 Each group corresponds to a section in config files. 1448 1449 An OptGroup object has no public methods, but has a number of public string 1450 properties: 1451 1452 .. py:attribute:: name 1453 1454 the name of the group 1455 1456 .. py:attribute:: title 1457 1458 the group title as displayed in --help 1459 1460 .. py:attribute:: help 1461 1462 the group description as displayed in --help 1463 1464 :param name: the group name 1465 :type name: str 1466 :param title: the group title for --help 1467 :type title: str 1468 :param help: the group description for --help 1469 :type help: str 1470 :param dynamic_group_owner: The name of the option that controls 1471 repeated instances of this group. 1472 :type dynamic_group_owner: str 1473 :param driver_option: The name of the option within the group that 1474 controls which driver will register options. 1475 :type driver_option: str 1476 1477 """ 1478 1479 def __init__(self, name, title=None, help=None, 1480 dynamic_group_owner='', 1481 driver_option=''): 1482 """Constructs an OptGroup object.""" 1483 self.name = name 1484 self.title = "%s options" % name if title is None else title 1485 self.help = help 1486 self.dynamic_group_owner = dynamic_group_owner 1487 self.driver_option = driver_option 1488 1489 self._opts = {} # dict of dicts of (opt:, override:, default:) 1490 self._argparse_group = None 1491 self._driver_opts = {} # populated by the config generator 1492 1493 def _save_driver_opts(self, opts): 1494 """Save known driver opts. 1495 1496 :param opts: mapping between driver name and list of opts 1497 :type opts: dict 1498 1499 """ 1500 self._driver_opts.update(opts) 1501 1502 def _get_generator_data(self): 1503 "Return a dict with data for the sample generator." 1504 return { 1505 'help': self.help or '', 1506 'dynamic_group_owner': self.dynamic_group_owner, 1507 'driver_option': self.driver_option, 1508 'driver_opts': self._driver_opts, 1509 } 1510 1511 def _register_opt(self, opt, cli=False): 1512 """Add an opt to this group. 1513 1514 :param opt: an Opt object 1515 :param cli: whether this is a CLI option 1516 :returns: False if previously registered, True otherwise 1517 :raises: DuplicateOptError if a naming conflict is detected 1518 """ 1519 if _is_opt_registered(self._opts, opt): 1520 return False 1521 1522 self._opts[opt.dest] = {'opt': opt, 'cli': cli} 1523 1524 return True 1525 1526 def _unregister_opt(self, opt): 1527 """Remove an opt from this group. 1528 1529 :param opt: an Opt object 1530 """ 1531 if opt.dest in self._opts: 1532 del self._opts[opt.dest] 1533 1534 def _get_argparse_group(self, parser): 1535 if self._argparse_group is None: 1536 """Build an argparse._ArgumentGroup for this group.""" 1537 self._argparse_group = parser.add_argument_group(self.title, 1538 self.help) 1539 return self._argparse_group 1540 1541 def _clear(self): 1542 """Clear this group's option parsing state.""" 1543 self._argparse_group = None 1544 1545 def __str__(self): 1546 return self.name 1547 1548 1549class ParseError(iniparser.ParseError): 1550 def __init__(self, msg, lineno, line, filename): 1551 super(ParseError, self).__init__(msg, lineno, line) 1552 self.filename = filename 1553 1554 def __str__(self): 1555 return 'at %s:%d, %s: %r' % (self.filename, self.lineno, 1556 self.msg, self.line) 1557 1558 1559class ConfigParser(iniparser.BaseParser): 1560 """Parses a single config file, populating 'sections' to look like:: 1561 1562 {'DEFAULT': {'key': [value, ...], ...}, 1563 ...} 1564 1565 Also populates self._normalized which looks the same but with normalized 1566 section names. 1567 """ 1568 1569 def __init__(self, filename, sections): 1570 super(ConfigParser, self).__init__() 1571 self.filename = filename 1572 self.sections = sections 1573 self._normalized = None 1574 self.section = None 1575 1576 def _add_normalized(self, normalized): 1577 self._normalized = normalized 1578 1579 def parse(self): 1580 with open(self.filename) as f: 1581 return super(ConfigParser, self).parse(f.readlines()) 1582 1583 def new_section(self, section): 1584 self.section = section 1585 self.sections.setdefault(self.section, {}) 1586 1587 if self._normalized is not None: 1588 self._normalized.setdefault(_normalize_group_name(self.section), 1589 {}) 1590 1591 def assignment(self, key, value): 1592 if not self.section: 1593 raise self.error_no_section() 1594 1595 value = '\n'.join(value) 1596 1597 def append(sections, section): 1598 sections[section].setdefault(key, []) 1599 sections[section][key].append(value) 1600 1601 append(self.sections, self.section) 1602 if self._normalized is not None: 1603 append(self._normalized, _normalize_group_name(self.section)) 1604 1605 def parse_exc(self, msg, lineno, line=None): 1606 return ParseError(msg, lineno, line, self.filename) 1607 1608 def error_no_section(self): 1609 return self.parse_exc('Section must be started before assignment', 1610 self.lineno) 1611 1612 @classmethod 1613 def _parse_file(cls, config_file, namespace): 1614 """Parse a config file and store any values in the namespace. 1615 1616 :raises: ConfigFileParseError, ConfigFileValueError 1617 """ 1618 config_file = _fixpath(config_file) 1619 1620 sections = {} 1621 normalized = {} 1622 parser = cls(config_file, sections) 1623 parser._add_normalized(normalized) 1624 1625 try: 1626 parser.parse() 1627 except iniparser.ParseError as pe: 1628 raise ConfigFileParseError(pe.filename, str(pe)) 1629 except IOError as err: 1630 if err.errno == errno.ENOENT: 1631 namespace._file_not_found(config_file) 1632 return 1633 if err.errno == errno.EACCES: 1634 namespace._file_permission_denied(config_file) 1635 return 1636 raise 1637 1638 namespace._add_parsed_config_file(config_file, sections, normalized) 1639 namespace._parse_cli_opts_from_config_file( 1640 config_file, sections, normalized) 1641 1642 1643class _Namespace(argparse.Namespace): 1644 """An argparse namespace which also stores config file values. 1645 1646 As we parse command line arguments, the values get set as attributes 1647 on a namespace object. However, we also want to parse config files as 1648 they are specified on the command line and collect the values alongside 1649 the option values parsed from the command line. 1650 1651 Note, we don't actually assign values from config files as attributes 1652 on the namespace because config file options be registered after the 1653 command line has been parsed, so we may not know how to properly parse 1654 or convert a config file value at this point. 1655 """ 1656 1657 _deprecated_opt_message = ('Option "%(dep_option)s" from group ' 1658 '"%(dep_group)s" is deprecated. Use option ' 1659 '"%(option)s" from group "%(group)s".') 1660 1661 def __init__(self, conf): 1662 self._conf = conf 1663 self._parsed = [] 1664 self._normalized = [] 1665 self._emitted_deprecations = set() 1666 self._files_not_found = [] 1667 self._files_permission_denied = [] 1668 self._config_dirs = [] 1669 self._sections_to_file = {} 1670 1671 def _parse_cli_opts_from_config_file(self, config_file, sections, 1672 normalized): 1673 """Parse CLI options from a config file. 1674 1675 CLI options are special - we require they be registered before the 1676 command line is parsed. This means that as we parse config files, we 1677 can go ahead and apply the appropriate option-type specific conversion 1678 to the values in config files for CLI options. We can't do this for 1679 non-CLI options, because the schema describing those options may not be 1680 registered until after the config files are parsed. 1681 1682 This method relies on that invariant in order to enforce proper 1683 priority of option values - i.e. that the order in which an option 1684 value is parsed, whether the value comes from the CLI or a config file, 1685 determines which value specified for a given option wins. 1686 1687 The way we implement this ordering is that as we parse each config 1688 file, we look for values in that config file for CLI options only. Any 1689 values for CLI options found in the config file are treated like they 1690 had appeared on the command line and set as attributes on the namespace 1691 objects. Values in later config files or on the command line will 1692 override values found in this file. 1693 """ 1694 namespace = _Namespace(self._conf) 1695 namespace._add_parsed_config_file(config_file, sections, normalized) 1696 1697 for opt, group in self._conf._all_cli_opts(): 1698 group_name = group.name if group is not None else None 1699 try: 1700 value, loc = opt._get_from_namespace(namespace, group_name) 1701 except KeyError: 1702 continue 1703 except ValueError as ve: 1704 raise ConfigFileValueError( 1705 "Value for option %s is not valid: %s" 1706 % (opt.name, str(ve))) 1707 1708 if group_name is None: 1709 dest = opt.dest 1710 else: 1711 dest = group_name + '_' + opt.dest 1712 1713 if opt.multi: 1714 if getattr(self, dest, None) is None: 1715 setattr(self, dest, []) 1716 values = getattr(self, dest) 1717 values.extend(value) 1718 else: 1719 setattr(self, dest, value) 1720 1721 def _add_parsed_config_file(self, filename, sections, normalized): 1722 """Add a parsed config file to the list of parsed files. 1723 1724 :param filename: the full name of the file that was parsed 1725 :param sections: a mapping of section name to dicts of config values 1726 :param normalized: sections mapping with section names normalized 1727 :raises: ConfigFileValueError 1728 """ 1729 for s in sections: 1730 self._sections_to_file[s] = filename 1731 self._parsed.insert(0, sections) 1732 self._normalized.insert(0, normalized) 1733 1734 def _file_not_found(self, config_file): 1735 """Record that we were unable to open a config file. 1736 1737 :param config_file: the path to the failed file 1738 """ 1739 self._files_not_found.append(config_file) 1740 1741 def _file_permission_denied(self, config_file): 1742 """Record that we have no permission to open a config file. 1743 1744 :param config_file: the path to the failed file 1745 """ 1746 self._files_permission_denied.append(config_file) 1747 1748 def _get_cli_value(self, names, positional=False): 1749 """Fetch a CLI option value. 1750 1751 Look up the value of a CLI option. The value itself may have come from 1752 parsing the command line or parsing config files specified on the 1753 command line. Type conversion have already been performed for CLI 1754 options at this point. 1755 1756 :param names: a list of (section, name) tuples 1757 :param positional: whether this is a positional option 1758 """ 1759 for group_name, name in names: 1760 name = name if group_name is None else group_name + '_' + name 1761 value = getattr(self, name, None) 1762 if value is not None: 1763 # argparse ignores default=None for nargs='*' and returns [] 1764 if positional and not value: 1765 continue 1766 1767 return value 1768 1769 raise KeyError 1770 1771 def _get_file_value( 1772 self, names, multi=False, normalized=False, current_name=None): 1773 """Fetch a config file value from the parsed files. 1774 1775 :param names: a list of (section, name) tuples 1776 :param multi: a boolean indicating whether to return multiple values 1777 :param normalized: whether to normalize group names to lowercase 1778 :param current_name: current name in tuple being checked 1779 """ 1780 rvalue = [] 1781 1782 def normalize(name): 1783 if name is None: 1784 name = 'DEFAULT' 1785 return _normalize_group_name(name) if normalized else name 1786 1787 names = [(normalize(section), name) for section, name in names] 1788 1789 loc = None 1790 for sections in (self._normalized if normalized else self._parsed): 1791 for section, name in names: 1792 if section not in sections: 1793 continue 1794 if name in sections[section]: 1795 current_name = current_name or names[0] 1796 self._check_deprecated((section, name), current_name, 1797 names[1:]) 1798 val = sections[section][name] 1799 if loc is None: 1800 loc = LocationInfo( 1801 Locations.user, 1802 self._sections_to_file.get(section, ''), 1803 ) 1804 if multi: 1805 rvalue = val + rvalue 1806 else: 1807 return (val, loc) 1808 if multi and rvalue != []: 1809 return (rvalue, loc) 1810 raise KeyError 1811 1812 def _check_deprecated(self, name, current, deprecated): 1813 """Check for usage of deprecated names. 1814 1815 :param name: A tuple of the form (group, name) representing the group 1816 and name where an opt value was found. 1817 :param current: A tuple of the form (group, name) representing the 1818 current name for an option. 1819 :param deprecated: A list of tuples with the same format as the name 1820 param which represent any deprecated names for an option. 1821 If the name param matches any entries in this list a 1822 deprecation warning will be logged. 1823 """ 1824 if name in deprecated and name not in self._emitted_deprecations: 1825 self._emitted_deprecations.add(name) 1826 current = (current[0] or 'DEFAULT', current[1]) 1827 format_dict = {'dep_option': name[1], 'dep_group': name[0], 1828 'option': current[1], 'group': current[0]} 1829 _report_deprecation(self._deprecated_opt_message, format_dict) 1830 1831 def _get_value(self, names, multi=False, positional=False, 1832 current_name=None, normalized=True): 1833 """Fetch a value from config files. 1834 1835 Multiple names for a given configuration option may be supplied so 1836 that we can transparently handle files containing deprecated option 1837 names or groups. 1838 1839 :param names: a list of (section, name) tuples 1840 :param positional: whether this is a positional option 1841 :param multi: a boolean indicating whether to return multiple values 1842 :param normalized: whether to normalize group names to lowercase 1843 """ 1844 # NOTE(dhellmann): We don't have a way to track which options 1845 # that are registered as command line values show up on the 1846 # command line or in the configuration files. So we look up 1847 # the value in the file first to get the location, and then 1848 # try looking it up as a CLI value in case it was set there. 1849 1850 # Set a default location indicating that the value came from 1851 # the command line. This will be overridden if we find a value 1852 # in a file. 1853 loc = LocationInfo(Locations.command_line, '') 1854 1855 try: 1856 file_names = [(g if g is not None else 'DEFAULT', n) 1857 for g, n in names] 1858 values, loc = self._get_file_value( 1859 file_names, multi=multi, normalized=normalized, 1860 current_name=current_name) 1861 except KeyError: 1862 # If we receive a KeyError when looking for the CLI, just 1863 # go ahead and throw it because we know we don't have a 1864 # value. 1865 raise_later = True 1866 else: 1867 raise_later = False 1868 1869 # Now try the CLI 1870 try: 1871 value = self._get_cli_value(names, positional) 1872 return (value, loc) 1873 except KeyError: 1874 if raise_later: 1875 # Re-raise to indicate that we haven't found the value 1876 # anywhere. 1877 raise 1878 1879 # Return the value we found in the file. 1880 return (values if multi else values[-1], loc) 1881 1882 def _sections(self): 1883 for sections in self._parsed: 1884 for section in sections: 1885 yield section 1886 1887 1888class _CachedArgumentParser(argparse.ArgumentParser): 1889 1890 """class for caching/collecting command line arguments. 1891 1892 It also sorts the arguments before initializing the ArgumentParser. 1893 We need to do this since ArgumentParser by default does not sort 1894 the argument options and the only way to influence the order of 1895 arguments in '--help' is to ensure they are added in the sorted 1896 order. 1897 """ 1898 1899 def __init__(self, prog=None, usage=None, **kwargs): 1900 super(_CachedArgumentParser, self).__init__(prog, usage, **kwargs) 1901 self._args_cache = {} 1902 1903 def add_parser_argument(self, container, *args, **kwargs): 1904 values = [] 1905 if container in self._args_cache: 1906 values = self._args_cache[container] 1907 values.append({'args': args, 'kwargs': kwargs}) 1908 self._args_cache[container] = values 1909 1910 def initialize_parser_arguments(self): 1911 # NOTE(mfedosin): The code below looks a little bit weird, but 1912 # it's done because we need to sort only optional opts and do 1913 # not touch positional. For the reason optional opts go first in 1914 # the values we only need to find an index of the first positional 1915 # option and then sort the values slice. 1916 for container, values in self._args_cache.items(): 1917 index = 0 1918 has_positional = False 1919 for index, argument in enumerate(values): 1920 if not argument['args'][0].startswith('-'): 1921 has_positional = True 1922 break 1923 size = index if has_positional else len(values) 1924 values[:size] = sorted(values[:size], key=lambda x: x['args']) 1925 for argument in values: 1926 try: 1927 container.add_argument(*argument['args'], 1928 **argument['kwargs']) 1929 except argparse.ArgumentError: 1930 options = ','.join(argument['args']) 1931 raise DuplicateOptError(options) 1932 self._args_cache = {} 1933 1934 def parse_args(self, args=None, namespace=None): 1935 self.initialize_parser_arguments() 1936 return super(_CachedArgumentParser, self).parse_args(args, namespace) 1937 1938 def print_help(self, file=None): 1939 self.initialize_parser_arguments() 1940 super(_CachedArgumentParser, self).print_help(file) 1941 1942 def print_usage(self, file=None): 1943 self.initialize_parser_arguments() 1944 super(_CachedArgumentParser, self).print_usage(file) 1945 1946 1947class ConfigOpts(abc.Mapping): 1948 1949 """Config options which may be set on the command line or in config files. 1950 1951 ConfigOpts is a configuration option manager with APIs for registering 1952 option schemas, grouping options, parsing option values and retrieving 1953 the values of options. 1954 1955 It has built-in support for :oslo.config:option:`config_file` and 1956 :oslo.config:option:`config_dir` options. 1957 1958 """ 1959 disallow_names = ('project', 'prog', 'version', 1960 'usage', 'default_config_files', 'default_config_dirs') 1961 1962 # NOTE(dhellmann): This instance is reused by list_opts(). 1963 _config_source_opt = ListOpt( 1964 'config_source', 1965 metavar='SOURCE', 1966 default=[], 1967 help=('Lists configuration groups that provide more ' 1968 'details for accessing configuration settings ' 1969 'from locations other than local files.'), 1970 ) 1971 1972 def __init__(self): 1973 """Construct a ConfigOpts object.""" 1974 self._opts = {} # dict of dicts of (opt:, override:, default:) 1975 self._groups = {} 1976 self._deprecated_opts = {} 1977 1978 self._args = None 1979 1980 self._oparser = None 1981 self._namespace = None 1982 self._mutable_ns = None 1983 self._mutate_hooks = set([]) 1984 self.__cache = {} 1985 self.__drivers_cache = {} 1986 self._config_opts = [] 1987 self._cli_opts = collections.deque() 1988 self._validate_default_values = False 1989 self._sources = [] 1990 self._ext_mgr = None 1991 # Though the env_driver is a Source, we load it by default. 1992 self._use_env = True 1993 self._env_driver = _environment.EnvironmentConfigurationSource() 1994 1995 self.register_opt(self._config_source_opt) 1996 1997 def _pre_setup(self, project, prog, version, usage, description, epilog, 1998 default_config_files, default_config_dirs): 1999 """Initialize a ConfigCliParser object for option parsing.""" 2000 2001 if prog is None: 2002 prog = os.path.basename(sys.argv[0]) 2003 if prog.endswith(".py"): 2004 prog = prog[:-3] 2005 2006 if default_config_files is None: 2007 default_config_files = find_config_files(project, prog) 2008 2009 if default_config_dirs is None: 2010 default_config_dirs = find_config_dirs(project, prog) 2011 2012 self._oparser = _CachedArgumentParser( 2013 prog=prog, usage=usage, description=description, epilog=epilog) 2014 2015 if version is not None: 2016 self._oparser.add_parser_argument(self._oparser, 2017 '--version', 2018 action='version', 2019 version=version) 2020 2021 return prog, default_config_files, default_config_dirs 2022 2023 @staticmethod 2024 def _make_config_options(default_config_files, default_config_dirs): 2025 return [ 2026 _ConfigFileOpt('config-file', 2027 default=default_config_files, 2028 metavar='PATH', 2029 help=('Path to a config file to use. Multiple ' 2030 'config files can be specified, with values ' 2031 'in later files taking precedence. Defaults ' 2032 'to %(default)s. This option must be set ' 2033 'from the command-line.')), 2034 _ConfigDirOpt('config-dir', 2035 metavar='DIR', 2036 default=default_config_dirs, 2037 help='Path to a config directory to pull `*.conf` ' 2038 'files from. This file set is sorted, so as to ' 2039 'provide a predictable parse order if ' 2040 'individual options are over-ridden. The set ' 2041 'is parsed after the file(s) specified via ' 2042 'previous --config-file, arguments hence ' 2043 'over-ridden options in the directory take ' 2044 'precedence. This option must be set from ' 2045 'the command-line.'), 2046 ] 2047 2048 @classmethod 2049 def _list_options_for_discovery(cls, 2050 default_config_files, 2051 default_config_dirs): 2052 "Return options to be used by list_opts() for the sample generator." 2053 options = cls._make_config_options(default_config_files, 2054 default_config_dirs) 2055 options.append(cls._config_source_opt) 2056 return options 2057 2058 def _setup(self, project, prog, version, usage, default_config_files, 2059 default_config_dirs, use_env): 2060 """Initialize a ConfigOpts object for option parsing.""" 2061 self._config_opts = self._make_config_options(default_config_files, 2062 default_config_dirs) 2063 self.register_cli_opts(self._config_opts) 2064 2065 self.project = project 2066 self.prog = prog 2067 self.version = version 2068 self.usage = usage 2069 self.default_config_files = default_config_files 2070 self.default_config_dirs = default_config_dirs 2071 self._use_env = use_env 2072 2073 def __clear_cache(f): 2074 @functools.wraps(f) 2075 def __inner(self, *args, **kwargs): 2076 if kwargs.pop('clear_cache', True): 2077 result = f(self, *args, **kwargs) 2078 self.__cache.clear() 2079 return result 2080 else: 2081 return f(self, *args, **kwargs) 2082 2083 return __inner 2084 2085 def __clear_drivers_cache(f): 2086 @functools.wraps(f) 2087 def __inner(self, *args, **kwargs): 2088 if kwargs.pop('clear_drivers_cache', True): 2089 result = f(self, *args, **kwargs) 2090 self.__drivers_cache.clear() 2091 return result 2092 else: 2093 return f(self, *args, **kwargs) 2094 2095 return __inner 2096 2097 def __call__(self, 2098 args=None, 2099 project=None, 2100 prog=None, 2101 version=None, 2102 usage=None, 2103 default_config_files=None, 2104 default_config_dirs=None, 2105 validate_default_values=False, 2106 description=None, 2107 epilog=None, 2108 use_env=True): 2109 """Parse command line arguments and config files. 2110 2111 Calling a ConfigOpts object causes the supplied command line arguments 2112 and config files to be parsed, causing opt values to be made available 2113 as attributes of the object. 2114 2115 The object may be called multiple times, each time causing the previous 2116 set of values to be overwritten. 2117 2118 Automatically registers the --config-file option with either a supplied 2119 list of default config files, or a list from find_config_files(). 2120 2121 If the --config-dir option is set, any *.conf files from this 2122 directory are pulled in, after all the file(s) specified by the 2123 --config-file option. 2124 2125 :param args: command line arguments (defaults to sys.argv[1:]) 2126 :param project: the toplevel project name, used to locate config files 2127 :param prog: the name of the program (defaults to sys.argv[0] 2128 basename, without extension .py) 2129 :param version: the program version (for --version) 2130 :param usage: a usage string (%prog will be expanded) 2131 :param description: A description of what the program does 2132 :param epilog: Text following the argument descriptions 2133 :param default_config_files: config files to use by default 2134 :param default_config_dirs: config dirs to use by default 2135 :param validate_default_values: whether to validate the default values 2136 :param use_env: If True (the default) look in the environment as one 2137 source of option values. 2138 :raises: SystemExit, ConfigFilesNotFoundError, ConfigFileParseError, 2139 ConfigFilesPermissionDeniedError, 2140 RequiredOptError, DuplicateOptError 2141 """ 2142 self.clear() 2143 2144 self._validate_default_values = validate_default_values 2145 2146 prog, default_config_files, default_config_dirs = self._pre_setup( 2147 project, prog, version, usage, description, epilog, 2148 default_config_files, default_config_dirs) 2149 2150 self._setup(project, prog, version, usage, default_config_files, 2151 default_config_dirs, use_env) 2152 2153 self._namespace = self._parse_cli_opts(args if args is not None 2154 else sys.argv[1:]) 2155 if self._namespace._files_not_found: 2156 raise ConfigFilesNotFoundError(self._namespace._files_not_found) 2157 if self._namespace._files_permission_denied: 2158 raise ConfigFilesPermissionDeniedError( 2159 self._namespace._files_permission_denied) 2160 2161 self._load_alternative_sources() 2162 2163 self._check_required_opts() 2164 2165 def _load_alternative_sources(self): 2166 # Look for other sources of option data. 2167 for source_group_name in self.config_source: 2168 source = self._open_source_from_opt_group(source_group_name) 2169 if source is not None: 2170 self._sources.append(source) 2171 2172 def _open_source_from_opt_group(self, group_name): 2173 if not self._ext_mgr: 2174 self._ext_mgr = stevedore.ExtensionManager( 2175 "oslo.config.driver", 2176 invoke_on_load=True) 2177 2178 self.register_opt( 2179 StrOpt('driver', 2180 choices=self._ext_mgr.names(), 2181 help=_SOURCE_DRIVER_OPTION_HELP), 2182 group=group_name) 2183 2184 try: 2185 driver_name = self[group_name].driver 2186 except ConfigFileValueError as err: 2187 LOG.error( 2188 "could not load configuration from %r. %s", 2189 group_name, err.msg) 2190 return None 2191 2192 if driver_name is None: 2193 LOG.error( 2194 "could not load configuration from %r, no 'driver' is set.", 2195 group_name) 2196 return None 2197 2198 LOG.info('loading configuration from %r using %r', 2199 group_name, driver_name) 2200 2201 driver = self._ext_mgr[driver_name].obj 2202 2203 try: 2204 return driver.open_source_from_opt_group(self, group_name) 2205 except Exception as err: 2206 LOG.error( 2207 "could not load configuration from %r using %s driver: %s", 2208 group_name, driver_name, err) 2209 return None 2210 2211 def __getattr__(self, name): 2212 """Look up an option value and perform string substitution. 2213 2214 :param name: the opt name (or 'dest', more precisely) 2215 :returns: the option value (after string substitution) or a GroupAttr 2216 :raises: ValueError or NoSuchOptError 2217 """ 2218 try: 2219 return self._get(name) 2220 except ValueError: 2221 raise 2222 except Exception: 2223 raise NoSuchOptError(name) 2224 2225 def __getitem__(self, key): 2226 """Look up an option value and perform string substitution.""" 2227 return self.__getattr__(key) 2228 2229 def __contains__(self, key): 2230 """Return True if key is the name of a registered opt or group.""" 2231 return key in self._opts or key in self._groups 2232 2233 def __iter__(self): 2234 """Iterate over all registered opt and group names.""" 2235 for key in itertools.chain(list(self._opts.keys()), 2236 list(self._groups.keys())): 2237 yield key 2238 2239 def __len__(self): 2240 """Return the number of options and option groups.""" 2241 return len(self._opts) + len(self._groups) 2242 2243 def reset(self): 2244 """Clear the object state and unset overrides and defaults.""" 2245 self._unset_defaults_and_overrides() 2246 self.clear() 2247 2248 @__clear_cache 2249 def clear(self): 2250 """Reset the state of the object to before options were registered. 2251 2252 This method removes all registered options and discards the data 2253 from the command line and configuration files. 2254 2255 Any subparsers added using the add_cli_subparsers() will also be 2256 removed as a side-effect of this method. 2257 """ 2258 self._args = None 2259 self._oparser = None 2260 self._namespace = None 2261 self._mutable_ns = None 2262 # Keep _mutate_hooks 2263 self._validate_default_values = False 2264 self.unregister_opts(self._config_opts) 2265 for group in self._groups.values(): 2266 group._clear() 2267 2268 def _add_cli_opt(self, opt, group): 2269 if {'opt': opt, 'group': group} in self._cli_opts: 2270 return 2271 if opt.positional: 2272 self._cli_opts.append({'opt': opt, 'group': group}) 2273 else: 2274 self._cli_opts.appendleft({'opt': opt, 'group': group}) 2275 2276 def _track_deprecated_opts(self, opt, group=None): 2277 if hasattr(opt, 'deprecated_opts'): 2278 for dep_opt in opt.deprecated_opts: 2279 dep_group = dep_opt.group or 'DEFAULT' 2280 dep_dest = dep_opt.name 2281 if dep_dest: 2282 dep_dest = dep_dest.replace('-', '_') 2283 if dep_group not in self._deprecated_opts: 2284 self._deprecated_opts[dep_group] = { 2285 dep_dest: { 2286 'opt': opt, 2287 'group': group 2288 } 2289 } 2290 else: 2291 self._deprecated_opts[dep_group][dep_dest] = { 2292 'opt': opt, 2293 'group': group 2294 } 2295 2296 @__clear_cache 2297 def register_opt(self, opt, group=None, cli=False): 2298 """Register an option schema. 2299 2300 Registering an option schema makes any option value which is previously 2301 or subsequently parsed from the command line or config files available 2302 as an attribute of this object. 2303 2304 :param opt: an instance of an Opt sub-class 2305 :param group: an optional OptGroup object or group name 2306 :param cli: whether this is a CLI option 2307 :return: False if the opt was already registered, True otherwise 2308 :raises: DuplicateOptError 2309 """ 2310 if group is not None: 2311 group = self._get_group(group, autocreate=True) 2312 if cli: 2313 self._add_cli_opt(opt, group) 2314 self._track_deprecated_opts(opt, group=group) 2315 return group._register_opt(opt, cli) 2316 2317 # NOTE(gcb) We can't use some names which are same with attributes of 2318 # Opts in default group. They includes project, prog, version, usage, 2319 # default_config_files and default_config_dirs. 2320 if group is None: 2321 if opt.name in self.disallow_names: 2322 raise ValueError('Name %s was reserved for oslo.config.' 2323 % opt.name) 2324 2325 if cli: 2326 self._add_cli_opt(opt, None) 2327 2328 if _is_opt_registered(self._opts, opt): 2329 return False 2330 2331 self._opts[opt.dest] = {'opt': opt, 'cli': cli} 2332 self._track_deprecated_opts(opt) 2333 return True 2334 2335 @__clear_cache 2336 def register_opts(self, opts, group=None): 2337 """Register multiple option schemas at once.""" 2338 for opt in opts: 2339 self.register_opt(opt, group, clear_cache=False) 2340 2341 @__clear_cache 2342 def register_cli_opt(self, opt, group=None): 2343 """Register a CLI option schema. 2344 2345 CLI option schemas must be registered before the command line and 2346 config files are parsed. This is to ensure that all CLI options are 2347 shown in --help and option validation works as expected. 2348 2349 :param opt: an instance of an Opt sub-class 2350 :param group: an optional OptGroup object or group name 2351 :return: False if the opt was already registered, True otherwise 2352 :raises: DuplicateOptError, ArgsAlreadyParsedError 2353 """ 2354 if self._args is not None: 2355 raise ArgsAlreadyParsedError("cannot register CLI option") 2356 2357 return self.register_opt(opt, group, cli=True, clear_cache=False) 2358 2359 @__clear_cache 2360 def register_cli_opts(self, opts, group=None): 2361 """Register multiple CLI option schemas at once.""" 2362 for opt in opts: 2363 self.register_cli_opt(opt, group, clear_cache=False) 2364 2365 def register_group(self, group): 2366 """Register an option group. 2367 2368 An option group must be registered before options can be registered 2369 with the group. 2370 2371 :param group: an OptGroup object 2372 """ 2373 if group.name in self._groups: 2374 return 2375 2376 self._groups[group.name] = copy.copy(group) 2377 2378 @__clear_cache 2379 def unregister_opt(self, opt, group=None): 2380 """Unregister an option. 2381 2382 :param opt: an Opt object 2383 :param group: an optional OptGroup object or group name 2384 :raises: ArgsAlreadyParsedError, NoSuchGroupError 2385 """ 2386 if self._args is not None: 2387 raise ArgsAlreadyParsedError("reset before unregistering options") 2388 2389 remitem = None 2390 for item in self._cli_opts: 2391 if (item['opt'].dest == opt.dest and 2392 (group is None or 2393 self._get_group(group).name == item['group'].name)): 2394 remitem = item 2395 break 2396 if remitem is not None: 2397 self._cli_opts.remove(remitem) 2398 2399 if group is not None: 2400 self._get_group(group)._unregister_opt(opt) 2401 elif opt.dest in self._opts: 2402 del self._opts[opt.dest] 2403 2404 @__clear_cache 2405 def unregister_opts(self, opts, group=None): 2406 """Unregister multiple CLI option schemas at once.""" 2407 for opt in opts: 2408 self.unregister_opt(opt, group, clear_cache=False) 2409 2410 def import_opt(self, name, module_str, group=None): 2411 """Import an option definition from a module. 2412 2413 Import a module and check that a given option is registered. 2414 2415 This is intended for use with global configuration objects 2416 like cfg.CONF where modules commonly register options with 2417 CONF at module load time. If one module requires an option 2418 defined by another module it can use this method to explicitly 2419 declare the dependency. 2420 2421 :param name: the name/dest of the opt 2422 :param module_str: the name of a module to import 2423 :param group: an option OptGroup object or group name 2424 :raises: NoSuchOptError, NoSuchGroupError 2425 """ 2426 __import__(module_str) 2427 self._get_opt_info(name, group) 2428 2429 def import_group(self, group, module_str): 2430 """Import an option group from a module. 2431 2432 Import a module and check that a given option group is registered. 2433 2434 This is intended for use with global configuration objects 2435 like cfg.CONF where modules commonly register options with 2436 CONF at module load time. If one module requires an option group 2437 defined by another module it can use this method to explicitly 2438 declare the dependency. 2439 2440 :param group: an option OptGroup object or group name 2441 :param module_str: the name of a module to import 2442 :raises: ImportError, NoSuchGroupError 2443 """ 2444 __import__(module_str) 2445 self._get_group(group) 2446 2447 @__clear_cache 2448 def set_override(self, name, override, group=None): 2449 """Override an opt value. 2450 2451 Override the command line, config file and default values of a 2452 given option. 2453 2454 :param name: the name/dest of the opt 2455 :param override: the override value 2456 :param group: an option OptGroup object or group name 2457 2458 :raises: NoSuchOptError, NoSuchGroupError 2459 """ 2460 opt_info = self._get_opt_info(name, group) 2461 opt_info['override'] = self._get_enforced_type_value( 2462 opt_info['opt'], override) 2463 opt_info['location'] = LocationInfo( 2464 Locations.set_override, 2465 _get_caller_detail(3), # this function has a decorator to skip 2466 ) 2467 2468 @__clear_cache 2469 def set_default(self, name, default, group=None): 2470 """Override an opt's default value. 2471 2472 Override the default value of given option. A command line or 2473 config file value will still take precedence over this default. 2474 2475 :param name: the name/dest of the opt 2476 :param default: the default value 2477 :param group: an option OptGroup object or group name 2478 2479 :raises: NoSuchOptError, NoSuchGroupError 2480 """ 2481 opt_info = self._get_opt_info(name, group) 2482 opt_info['default'] = self._get_enforced_type_value( 2483 opt_info['opt'], default) 2484 opt_info['location'] = LocationInfo( 2485 Locations.set_default, 2486 _get_caller_detail(3), # this function has a decorator to skip 2487 ) 2488 2489 def _get_enforced_type_value(self, opt, value): 2490 if value is None: 2491 return None 2492 2493 return self._convert_value(value, opt) 2494 2495 @__clear_cache 2496 def clear_override(self, name, group=None): 2497 """Clear an override an opt value. 2498 2499 Clear a previously set override of the command line, config file 2500 and default values of a given option. 2501 2502 :param name: the name/dest of the opt 2503 :param group: an option OptGroup object or group name 2504 :raises: NoSuchOptError, NoSuchGroupError 2505 """ 2506 opt_info = self._get_opt_info(name, group) 2507 opt_info.pop('override', None) 2508 2509 @__clear_cache 2510 def clear_default(self, name, group=None): 2511 """Clear an override an opt's default value. 2512 2513 Clear a previously set override of the default value of given option. 2514 2515 :param name: the name/dest of the opt 2516 :param group: an option OptGroup object or group name 2517 :raises: NoSuchOptError, NoSuchGroupError 2518 """ 2519 opt_info = self._get_opt_info(name, group) 2520 opt_info.pop('default', None) 2521 2522 def _all_opt_infos(self): 2523 """A generator function for iteration opt infos.""" 2524 for info in self._opts.values(): 2525 yield info, None 2526 for group in self._groups.values(): 2527 for info in group._opts.values(): 2528 yield info, group 2529 2530 def _all_cli_opts(self): 2531 """A generator function for iterating CLI opts.""" 2532 for item in self._cli_opts: 2533 yield item['opt'], item['group'] 2534 2535 def _unset_defaults_and_overrides(self): 2536 """Unset any default or override on all options.""" 2537 for info, group in self._all_opt_infos(): 2538 info.pop('default', None) 2539 info.pop('override', None) 2540 2541 @property 2542 def config_dirs(self): 2543 if self._namespace is None: 2544 return [] 2545 return self._namespace._config_dirs 2546 2547 def find_file(self, name): 2548 """Locate a file located alongside the config files. 2549 2550 Search for a file with the supplied basename in the directories 2551 which we have already loaded config files from and other known 2552 configuration directories. 2553 2554 The directory, if any, supplied by the config_dir option is 2555 searched first. Then the config_file option is iterated over 2556 and each of the base directories of the config_files values 2557 are searched. Failing both of these, the standard directories 2558 searched by the module level find_config_files() function is 2559 used. The first matching file is returned. 2560 2561 :param name: the filename, for example 'policy.json' 2562 :returns: the path to a matching file, or None 2563 """ 2564 if not self._namespace: 2565 raise NotInitializedError() 2566 dirs = [] 2567 if self._namespace._config_dirs: 2568 for directory in self._namespace._config_dirs: 2569 dirs.append(_fixpath(directory)) 2570 2571 for cf in reversed(self.config_file): 2572 dirs.append(os.path.dirname(_fixpath(cf))) 2573 2574 dirs.extend(_get_config_dirs(self.project)) 2575 2576 return _search_dirs(dirs, name) 2577 2578 def log_opt_values(self, logger, lvl): 2579 """Log the value of all registered opts. 2580 2581 It's often useful for an app to log its configuration to a log file at 2582 startup for debugging. This method dumps to the entire config state to 2583 the supplied logger at a given log level. 2584 2585 :param logger: a logging.Logger object 2586 :param lvl: the log level (for example logging.DEBUG) arg to 2587 logger.log() 2588 """ 2589 logger.log(lvl, "*" * 80) 2590 logger.log(lvl, "Configuration options gathered from:") 2591 logger.log(lvl, "command line args: %s", self._args) 2592 logger.log(lvl, "config files: %s", 2593 hasattr(self, 'config_file') and self.config_file or []) 2594 logger.log(lvl, "=" * 80) 2595 2596 def _sanitize(opt, value): 2597 """Obfuscate values of options declared secret.""" 2598 return value if not opt.secret else '*' * 4 2599 2600 for opt_name in sorted(self._opts): 2601 opt = self._get_opt_info(opt_name)['opt'] 2602 logger.log(lvl, "%-30s = %s", opt_name, 2603 _sanitize(opt, getattr(self, opt_name))) 2604 2605 for group_name in list(self._groups): 2606 group_attr = self.GroupAttr(self, self._get_group(group_name)) 2607 for opt_name in sorted(self._groups[group_name]._opts): 2608 opt = self._get_opt_info(opt_name, group_name)['opt'] 2609 logger.log(lvl, "%-30s = %s", 2610 "%s.%s" % (group_name, opt_name), 2611 _sanitize(opt, getattr(group_attr, opt_name))) 2612 2613 logger.log(lvl, "*" * 80) 2614 2615 def print_usage(self, file=None): 2616 """Print the usage message for the current program. 2617 2618 This method is for use after all CLI options are known 2619 registered using __call__() method. If this method is called 2620 before the __call__() is invoked, it throws NotInitializedError 2621 2622 :param file: the File object (if None, output is on sys.stdout) 2623 :raises: NotInitializedError 2624 """ 2625 if not self._oparser: 2626 raise NotInitializedError() 2627 self._oparser.print_usage(file) 2628 2629 def print_help(self, file=None): 2630 """Print the help message for the current program. 2631 2632 This method is for use after all CLI options are known 2633 registered using __call__() method. If this method is called 2634 before the __call__() is invoked, it throws NotInitializedError 2635 2636 :param file: the File object (if None, output is on sys.stdout) 2637 :raises: NotInitializedError 2638 """ 2639 if not self._oparser: 2640 raise NotInitializedError() 2641 self._oparser.print_help(file) 2642 2643 def _get(self, name, group=None, namespace=None): 2644 if isinstance(group, OptGroup): 2645 key = (group.name, name) 2646 else: 2647 key = (group, name) 2648 if namespace is None: 2649 try: 2650 return self.__cache[key] 2651 except KeyError: # nosec: Valid control flow instruction 2652 pass 2653 value, loc = self._do_get(name, group, namespace) 2654 self.__cache[key] = value 2655 return value 2656 2657 def _do_get(self, name, group=None, namespace=None): 2658 """Look up an option value. 2659 2660 :param name: the opt name (or 'dest', more precisely) 2661 :param group: an OptGroup 2662 :param namespace: the namespace object to get the option value from 2663 :returns: 2-tuple of the option value or a GroupAttr object 2664 and LocationInfo or None 2665 :raises: NoSuchOptError, NoSuchGroupError, ConfigFileValueError, 2666 TemplateSubstitutionError 2667 """ 2668 if group is None and name in self._groups: 2669 return (self.GroupAttr(self, self._get_group(name)), None) 2670 2671 info = self._get_opt_info(name, group) 2672 opt = info['opt'] 2673 if 'location' in info: 2674 loc = info['location'] 2675 else: 2676 loc = opt._set_location 2677 2678 if isinstance(opt, SubCommandOpt): 2679 return (self.SubCommandAttr(self, group, opt.dest), None) 2680 2681 if 'override' in info: 2682 return (self._substitute(info['override']), loc) 2683 2684 def convert(value): 2685 return self._convert_value( 2686 self._substitute(value, group, namespace), opt) 2687 2688 group_name = group.name if group else None 2689 key = (group_name, name) 2690 2691 # If use_env is true, get a value from the environment but don't use 2692 # it yet. We will look at the command line first, below. 2693 env_val = (sources._NoValue, None) 2694 if self._use_env: 2695 env_val = self._env_driver.get(group_name, name, opt) 2696 2697 if opt.mutable and namespace is None: 2698 namespace = self._mutable_ns 2699 if namespace is None: 2700 namespace = self._namespace 2701 if namespace is not None: 2702 try: 2703 alt_loc = None 2704 try: 2705 val, alt_loc = opt._get_from_namespace(namespace, 2706 group_name) 2707 # Try command line first 2708 if (val != sources._NoValue 2709 and alt_loc.location == Locations.command_line): 2710 return (convert(val), alt_loc) 2711 # Environment source second 2712 if env_val[0] != sources._NoValue: 2713 return (convert(env_val[0]), env_val[1]) 2714 # Default file source third 2715 if val != sources._NoValue: 2716 return (convert(val), alt_loc) 2717 except KeyError: # nosec: Valid control flow instruction 2718 alt_loc = LocationInfo( 2719 Locations.environment, 2720 self._env_driver.get_name(group_name, name), 2721 ) 2722 # If there was a KeyError looking at config files or 2723 # command line, retry the env_val. 2724 if env_val[0] != sources._NoValue: 2725 return (convert(env_val[0]), env_val[1]) 2726 except ValueError as ve: 2727 message = "Value for option %s from %s is not valid: %s" % ( 2728 opt.name, alt_loc, str(ve)) 2729 # Preserve backwards compatibility for file-based value 2730 # errors. 2731 if alt_loc.location == Locations.user: 2732 raise ConfigFileValueError(message) 2733 raise ConfigSourceValueError(message) 2734 2735 try: 2736 return self.__drivers_cache[key] 2737 except KeyError: # nosec: Valid control flow instruction 2738 pass 2739 2740 for source in self._sources: 2741 val = source.get(group_name, name, opt) 2742 if val[0] != sources._NoValue: 2743 result = (convert(val[0]), val[1]) 2744 self.__drivers_cache[key] = result 2745 return result 2746 2747 if 'default' in info: 2748 return (self._substitute(info['default']), loc) 2749 2750 if self._validate_default_values: 2751 if opt.default is not None: 2752 try: 2753 convert(opt.default) 2754 except ValueError as e: 2755 raise ConfigFileValueError( 2756 "Default value for option %s is not valid: %s" 2757 % (opt.name, str(e))) 2758 2759 if opt.default is not None: 2760 return (convert(opt.default), loc) 2761 2762 return (None, None) 2763 2764 def _substitute(self, value, group=None, namespace=None): 2765 """Perform string template substitution. 2766 2767 Substitute any template variables (for example $foo, ${bar}) in 2768 the supplied string value(s) with opt values. 2769 2770 :param value: the string value, or list of string values 2771 :param group: the group that retrieves the option value from 2772 :param namespace: the namespace object that retrieves the option 2773 value from 2774 :returns: the substituted string(s) 2775 """ 2776 if isinstance(value, list): 2777 return [self._substitute(i, group=group, namespace=namespace) 2778 for i in value] 2779 elif isinstance(value, str): 2780 # Treat a backslash followed by the dollar sign "\$" 2781 # the same as the string template escape "$$" as it is 2782 # a bit more natural for users 2783 if r'\$' in value: 2784 value = value.replace(r'\$', '$$') 2785 tmpl = self.Template(value) 2786 ret = tmpl.safe_substitute( 2787 self.StrSubWrapper(self, group=group, namespace=namespace)) 2788 return ret 2789 elif isinstance(value, dict): 2790 # Substitute template variables in both key and value 2791 return {self._substitute(key, group=group, namespace=namespace): 2792 self._substitute(val, group=group, namespace=namespace) 2793 for key, val in value.items()} 2794 else: 2795 return value 2796 2797 class Template(string.Template): 2798 idpattern = r'[_a-z][\._a-z0-9]*' 2799 2800 def _convert_value(self, value, opt): 2801 """Perform value type conversion. 2802 2803 Converts values using option's type. Handles cases when value is 2804 actually a list of values (for example for multi opts). 2805 2806 :param value: the string value, or list of string values 2807 :param opt: option definition (instance of Opt class or its subclasses) 2808 :returns: converted value 2809 """ 2810 if opt.multi: 2811 return [opt.type(v) for v in value] 2812 else: 2813 return opt.type(value) 2814 2815 def _get_group(self, group_or_name, autocreate=False): 2816 """Looks up a OptGroup object. 2817 2818 Helper function to return an OptGroup given a parameter which can 2819 either be the group's name or an OptGroup object. 2820 2821 The OptGroup object returned is from the internal dict of OptGroup 2822 objects, which will be a copy of any OptGroup object that users of 2823 the API have access to. 2824 2825 If autocreate is True, the group will be created if it's not found. If 2826 group is an instance of OptGroup, that same instance will be 2827 registered, otherwise a new instance of OptGroup will be created. 2828 2829 :param group_or_name: the group's name or the OptGroup object itself 2830 :param autocreate: whether to auto-create the group if it's not found 2831 :raises: NoSuchGroupError 2832 """ 2833 group = group_or_name if isinstance(group_or_name, OptGroup) else None 2834 group_name = group.name if group else group_or_name 2835 2836 if group_name not in self._groups: 2837 if not autocreate: 2838 raise NoSuchGroupError(group_name) 2839 2840 self.register_group(group or OptGroup(name=group_name)) 2841 2842 return self._groups[group_name] 2843 2844 def _find_deprecated_opts(self, opt_name, group=None): 2845 real_opt_name = None 2846 real_group_name = None 2847 group_name = group or 'DEFAULT' 2848 if hasattr(group_name, 'name'): 2849 group_name = group_name.name 2850 dep_group = self._deprecated_opts.get(group_name) 2851 if dep_group: 2852 real_opt_dict = dep_group.get(opt_name) 2853 if real_opt_dict: 2854 real_opt_name = real_opt_dict['opt'].name 2855 if real_opt_dict['group']: 2856 real_group_name = real_opt_dict['group'].name 2857 return real_opt_name, real_group_name 2858 2859 def _get_opt_info(self, opt_name, group=None): 2860 """Return the (opt, override, default) dict for an opt. 2861 2862 :param opt_name: an opt name/dest 2863 :param group: an optional group name or OptGroup object 2864 :raises: NoSuchOptError, NoSuchGroupError 2865 """ 2866 if group is None: 2867 opts = self._opts 2868 else: 2869 group = self._get_group(group) 2870 opts = group._opts 2871 2872 if opt_name not in opts: 2873 real_opt_name, real_group_name = self._find_deprecated_opts( 2874 opt_name, group=group) 2875 if not real_opt_name: 2876 raise NoSuchOptError(opt_name, group) 2877 log_real_group_name = real_group_name or 'DEFAULT' 2878 dep_message = ('Config option %(dep_group)s.%(dep_option)s ' 2879 ' is deprecated. Use option %(group)s.' 2880 '%(option)s instead.') 2881 LOG.warning(dep_message, {'dep_option': opt_name, 2882 'dep_group': group, 2883 'option': real_opt_name, 2884 'group': log_real_group_name}) 2885 opt_name = real_opt_name 2886 if real_group_name: 2887 group = self._get_group(real_group_name) 2888 opts = group._opts 2889 2890 return opts[opt_name] 2891 2892 def _check_required_opts(self, namespace=None): 2893 """Check that all opts marked as required have values specified. 2894 2895 :param namespace: the namespace object be checked the required options 2896 :raises: RequiredOptError 2897 """ 2898 for info, group in self._all_opt_infos(): 2899 opt = info['opt'] 2900 2901 if opt.required: 2902 if 'default' in info or 'override' in info: 2903 continue 2904 2905 if self._get(opt.dest, group, namespace) is None: 2906 raise RequiredOptError(opt.name, group) 2907 2908 def _parse_cli_opts(self, args): 2909 """Parse command line options. 2910 2911 Initializes the command line option parser and parses the supplied 2912 command line arguments. 2913 2914 :param args: the command line arguments 2915 :returns: a _Namespace object containing the parsed option values 2916 :raises: SystemExit, DuplicateOptError 2917 ConfigFileParseError, ConfigFileValueError 2918 2919 """ 2920 self._args = args 2921 for opt, group in self._all_cli_opts(): 2922 opt._add_to_cli(self._oparser, group) 2923 2924 return self._parse_config_files() 2925 2926 def _parse_config_files(self): 2927 """Parse configure files options. 2928 2929 :raises: SystemExit, ConfigFilesNotFoundError, ConfigFileParseError, 2930 ConfigFilesPermissionDeniedError, 2931 RequiredOptError, DuplicateOptError 2932 """ 2933 namespace = _Namespace(self) 2934 2935 # handle --config-file args or the default_config_files 2936 for arg in self._args: 2937 if arg == '--config-file' or arg.startswith('--config-file='): 2938 break 2939 else: 2940 for config_file in self.default_config_files: 2941 ConfigParser._parse_file(config_file, namespace) 2942 2943 # handle --config-dir args or the default_config_dirs 2944 for arg in self._args: 2945 if arg == '--config-dir' or arg.startswith('--config-dir='): 2946 break 2947 else: 2948 for config_dir in self.default_config_dirs: 2949 # for the default config-dir directories we just continue 2950 # if the directories do not exist. This is different to the 2951 # case where --config-dir is given on the command line. 2952 if not os.path.exists(config_dir): 2953 continue 2954 2955 config_dir_glob = os.path.join(config_dir, '*.conf') 2956 2957 for config_file in sorted(glob.glob(config_dir_glob)): 2958 ConfigParser._parse_file(config_file, namespace) 2959 2960 self._oparser.parse_args(self._args, namespace) 2961 2962 self._validate_cli_options(namespace) 2963 2964 return namespace 2965 2966 def _validate_cli_options(self, namespace): 2967 for opt, group in sorted(self._all_cli_opts(), 2968 key=lambda x: x[0].name): 2969 group_name = group.name if group else None 2970 try: 2971 value, loc = opt._get_from_namespace(namespace, group_name) 2972 except KeyError: 2973 continue 2974 2975 value = self._substitute(value, group=group, namespace=namespace) 2976 2977 try: 2978 self._convert_value(value, opt) 2979 except ValueError: 2980 sys.stderr.write("argument --%s: Invalid %s value: %s\n" % ( 2981 opt.dest, repr(opt.type), value)) 2982 raise SystemExit 2983 2984 def _reload_config_files(self): 2985 namespace = self._parse_config_files() 2986 if namespace._files_not_found: 2987 raise ConfigFilesNotFoundError(namespace._files_not_found) 2988 if namespace._files_permission_denied: 2989 raise ConfigFilesPermissionDeniedError( 2990 namespace._files_permission_denied) 2991 self._check_required_opts(namespace) 2992 return namespace 2993 2994 @__clear_cache 2995 @__clear_drivers_cache 2996 def reload_config_files(self): 2997 """Reload configure files and parse all options 2998 2999 :return: False if reload configure files failed or else return True 3000 """ 3001 3002 try: 3003 namespace = self._reload_config_files() 3004 except SystemExit as exc: 3005 LOG.warning("Caught SystemExit while reloading configure " 3006 "files with exit code: %d", exc.code) 3007 return False 3008 except Error as err: 3009 LOG.warning("Caught Error while reloading configure files: " 3010 "%s", err) 3011 return False 3012 else: 3013 self._namespace = namespace 3014 return True 3015 3016 def register_mutate_hook(self, hook): 3017 """Registers a hook to be called by mutate_config_files. 3018 3019 :param hook: a function accepting this ConfigOpts object and a dict of 3020 config mutations, as returned by mutate_config_files. 3021 :return: None 3022 """ 3023 self._mutate_hooks.add(hook) 3024 3025 def mutate_config_files(self): 3026 """Reload configure files and parse all options. 3027 3028 Only options marked as 'mutable' will appear to change. 3029 3030 Hooks are called in a NON-DETERMINISTIC ORDER. Do not expect hooks to 3031 be called in the same order as they were added. 3032 3033 :return: {(None or 'group', 'optname'): (old_value, new_value), ... } 3034 :raises: Error if reloading fails 3035 """ 3036 self.__cache.clear() 3037 3038 old_mutate_ns = self._mutable_ns or self._namespace 3039 self._mutable_ns = self._reload_config_files() 3040 self._warn_immutability() 3041 fresh = self._diff_ns(old_mutate_ns, self._mutable_ns) 3042 3043 def key_fn(item): 3044 # Py3 won't sort heterogeneous types. Sort None as TAB which has a 3045 # very low ASCII value. 3046 (groupname, optname) = item[0] 3047 return item[0] if groupname else ('\t', optname) 3048 sorted_fresh = sorted(fresh.items(), key=key_fn) 3049 for (groupname, optname), (old, new) in sorted_fresh: 3050 groupname = groupname if groupname else 'DEFAULT' 3051 LOG.info("Option %(group)s.%(option)s changed from " 3052 "[%(old_val)s] to [%(new_val)s]", 3053 {'group': groupname, 3054 'option': optname, 3055 'old_val': old, 3056 'new_val': new}) 3057 for hook in self._mutate_hooks: 3058 hook(self, fresh) 3059 return fresh 3060 3061 def _warn_immutability(self): 3062 """Check immutable opts have not changed. 3063 3064 _do_get won't return the new values but presumably someone changed the 3065 config file expecting them to change so we should warn them they won't. 3066 """ 3067 for info, group in self._all_opt_infos(): 3068 opt = info['opt'] 3069 if opt.mutable: 3070 continue 3071 groupname = group.name if group else 'DEFAULT' 3072 try: 3073 old, _ = opt._get_from_namespace(self._namespace, groupname) 3074 except KeyError: 3075 old = None 3076 try: 3077 new, _ = opt._get_from_namespace(self._mutable_ns, groupname) 3078 except KeyError: 3079 new = None 3080 if old != new: 3081 LOG.warning("Ignoring change to immutable option " 3082 "%(group)s.%(option)s", 3083 {"group": groupname, "option": opt.name}) 3084 3085 def _diff_ns(self, old_ns, new_ns): 3086 """Compare mutable option values between two namespaces. 3087 3088 This can be used to only reconfigure stateful sessions when necessary. 3089 3090 :return {(None or 'group', 'optname'): (old_value, new_value), ... } 3091 """ 3092 diff = {} 3093 for info, group in self._all_opt_infos(): 3094 opt = info['opt'] 3095 if not opt.mutable: 3096 continue 3097 groupname = group.name if group else None 3098 try: 3099 old, _ = opt._get_from_namespace(old_ns, groupname) 3100 except KeyError: 3101 old = None 3102 try: 3103 new, _ = opt._get_from_namespace(new_ns, groupname) 3104 except KeyError: 3105 new = None 3106 if old != new: 3107 diff[(groupname, opt.name)] = (old, new) 3108 return diff 3109 3110 def list_all_sections(self): 3111 """List all sections from the configuration. 3112 3113 Returns a sorted list of all section names found in the 3114 configuration files, whether declared beforehand or not. 3115 """ 3116 s = set([]) 3117 if self._mutable_ns: 3118 s |= set(self._mutable_ns._sections()) 3119 if self._namespace: 3120 s |= set(self._namespace._sections()) 3121 return sorted(s) 3122 3123 def get_location(self, name, group=None): 3124 """Return the location where the option is being set. 3125 3126 :param name: The name of the option. 3127 :type name: str 3128 :param group: The name of the group of the option. Defaults to 3129 ``'DEFAULT'``. 3130 :type group: str 3131 :return: LocationInfo 3132 3133 .. seealso:: 3134 3135 :doc:`/reference/locations` 3136 3137 .. versionadded:: 5.3.0 3138 """ 3139 opt_group = OptGroup(group) if group is not None else None 3140 value, loc = self._do_get(name, opt_group, None) 3141 return loc 3142 3143 class GroupAttr(abc.Mapping): 3144 3145 """Helper class. 3146 3147 Represents the option values of a group as a mapping and attributes. 3148 """ 3149 3150 def __init__(self, conf, group): 3151 """Construct a GroupAttr object. 3152 3153 :param conf: a ConfigOpts object 3154 :param group: an OptGroup object 3155 """ 3156 self._conf = conf 3157 self._group = group 3158 3159 def __getattr__(self, name): 3160 """Look up an option value and perform template substitution.""" 3161 return self._conf._get(name, self._group) 3162 3163 def __getitem__(self, key): 3164 """Look up an option value and perform string substitution.""" 3165 return self.__getattr__(key) 3166 3167 def __contains__(self, key): 3168 """Return True if key is the name of a registered opt or group.""" 3169 return key in self._group._opts 3170 3171 def __iter__(self): 3172 """Iterate over all registered opt and group names.""" 3173 for key in self._group._opts.keys(): 3174 yield key 3175 3176 def __len__(self): 3177 """Return the number of options and option groups.""" 3178 return len(self._group._opts) 3179 3180 class SubCommandAttr: 3181 3182 """Helper class. 3183 3184 Represents the name and arguments of an argparse sub-parser. 3185 """ 3186 3187 def __init__(self, conf, group, dest): 3188 """Construct a SubCommandAttr object. 3189 3190 :param conf: a ConfigOpts object 3191 :param group: an OptGroup object 3192 :param dest: the name of the sub-parser 3193 """ 3194 self._conf = conf 3195 self._group = group 3196 self._dest = dest 3197 3198 def __getattr__(self, name): 3199 """Look up a sub-parser name or argument value.""" 3200 if name == 'name': 3201 name = self._dest 3202 if self._group is not None: 3203 name = self._group.name + '_' + name 3204 return getattr(self._conf._namespace, name) 3205 3206 if name in self._conf: 3207 raise DuplicateOptError(name) 3208 3209 try: 3210 return getattr(self._conf._namespace, name) 3211 except AttributeError: 3212 raise NoSuchOptError(name) 3213 3214 class StrSubWrapper: 3215 3216 """Helper class. 3217 3218 Exposes opt values as a dict for string substitution. 3219 """ 3220 3221 def __init__(self, conf, group=None, namespace=None): 3222 """Construct a StrSubWrapper object. 3223 3224 :param conf: a ConfigOpts object 3225 :param group: an OptGroup object 3226 :param namespace: the namespace object that retrieves the option 3227 value from 3228 """ 3229 self.conf = conf 3230 self.group = group 3231 self.namespace = namespace 3232 3233 def __getitem__(self, key): 3234 """Look up an opt value from the ConfigOpts object. 3235 3236 :param key: an opt name 3237 :returns: an opt value 3238 """ 3239 try: 3240 group_name, option = key.split(".", 1) 3241 except ValueError: 3242 group = self.group 3243 option = key 3244 else: 3245 group = OptGroup(name=group_name) 3246 try: 3247 value = self.conf._get(option, group=group, 3248 namespace=self.namespace) 3249 except NoSuchOptError: 3250 value = self.conf._get(key, namespace=self.namespace) 3251 if isinstance(value, self.conf.GroupAttr): 3252 raise TemplateSubstitutionError( 3253 'substituting group %s not supported' % key) 3254 if value is None: 3255 return '' 3256 return value 3257 3258 3259CONF = ConfigOpts() 3260