1"""A simple configuration system.""" 2 3# Copyright (c) IPython Development Team. 4# Distributed under the terms of the Modified BSD License. 5 6import argparse 7import copy 8import os 9import re 10import sys 11import json 12import warnings 13 14from ..utils import cast_unicode, filefind 15 16from traitlets.traitlets import ( 17 HasTraits, Container, List, Dict, Any, Undefined, 18) 19 20#----------------------------------------------------------------------------- 21# Exceptions 22#----------------------------------------------------------------------------- 23 24 25class ConfigError(Exception): 26 pass 27 28class ConfigLoaderError(ConfigError): 29 pass 30 31class ConfigFileNotFound(ConfigError): 32 pass 33 34class ArgumentError(ConfigLoaderError): 35 pass 36 37#----------------------------------------------------------------------------- 38# Argparse fix 39#----------------------------------------------------------------------------- 40 41# Unfortunately argparse by default prints help messages to stderr instead of 42# stdout. This makes it annoying to capture long help screens at the command 43# line, since one must know how to pipe stderr, which many users don't know how 44# to do. So we override the print_help method with one that defaults to 45# stdout and use our class instead. 46 47 48class _Sentinel: 49 def __repr__(self): 50 return "<Sentinel deprecated>" 51 52 def __str__(self): 53 return "<deprecated>" 54 55 56_deprecated = _Sentinel() 57 58 59class ArgumentParser(argparse.ArgumentParser): 60 """Simple argparse subclass that prints help to stdout by default.""" 61 62 def print_help(self, file=None): 63 if file is None: 64 file = sys.stdout 65 return super(ArgumentParser, self).print_help(file) 66 67 print_help.__doc__ = argparse.ArgumentParser.print_help.__doc__ 68 69#----------------------------------------------------------------------------- 70# Config class for holding config information 71#----------------------------------------------------------------------------- 72 73def execfile(fname, glob): 74 with open(fname, 'rb') as f: 75 exec(compile(f.read(), fname, 'exec'), glob, glob) 76 77class LazyConfigValue(HasTraits): 78 """Proxy object for exposing methods on configurable containers 79 80 These methods allow appending/extending/updating 81 to add to non-empty defaults instead of clobbering them. 82 83 Exposes: 84 85 - append, extend, insert on lists 86 - update on dicts 87 - update, add on sets 88 """ 89 90 _value = None 91 92 # list methods 93 _extend = List() 94 _prepend = List() 95 _inserts = List() 96 97 def append(self, obj): 98 """Append an item to a List""" 99 self._extend.append(obj) 100 101 def extend(self, other): 102 """Extend a list""" 103 self._extend.extend(other) 104 105 def prepend(self, other): 106 """like list.extend, but for the front""" 107 self._prepend[:0] = other 108 109 110 def merge_into(self, other): 111 """ 112 Merge with another earlier LazyConfigValue or an earlier container. 113 This is useful when having global system-wide configuration files. 114 115 Self is expected to have higher precedence. 116 117 Parameters 118 ---------- 119 other : LazyConfigValue or container 120 121 Returns 122 ------- 123 LazyConfigValue 124 if ``other`` is also lazy, a reified container otherwise. 125 """ 126 if isinstance(other, LazyConfigValue): 127 other._extend.extend(self._extend) 128 self._extend = other._extend 129 130 self._prepend.extend(other._prepend) 131 132 other._inserts.extend(self._inserts) 133 self._inserts = other._inserts 134 135 if self._update: 136 other.update(self._update) 137 self._update = other._update 138 return self 139 else: 140 # other is a container, reify now. 141 return self.get_value(other) 142 143 def insert(self, index, other): 144 if not isinstance(index, int): 145 raise TypeError("An integer is required") 146 self._inserts.append((index, other)) 147 148 # dict methods 149 # update is used for both dict and set 150 _update = Any() 151 152 def update(self, other): 153 """Update either a set or dict""" 154 if self._update is None: 155 if isinstance(other, dict): 156 self._update = {} 157 else: 158 self._update = set() 159 self._update.update(other) 160 161 # set methods 162 def add(self, obj): 163 """Add an item to a set""" 164 self.update({obj}) 165 166 def get_value(self, initial): 167 """construct the value from the initial one 168 169 after applying any insert / extend / update changes 170 """ 171 if self._value is not None: 172 return self._value 173 value = copy.deepcopy(initial) 174 if isinstance(value, list): 175 for idx, obj in self._inserts: 176 value.insert(idx, obj) 177 value[:0] = self._prepend 178 value.extend(self._extend) 179 180 elif isinstance(value, dict): 181 if self._update: 182 value.update(self._update) 183 elif isinstance(value, set): 184 if self._update: 185 value.update(self._update) 186 self._value = value 187 return value 188 189 def to_dict(self): 190 """return JSONable dict form of my data 191 192 Currently update as dict or set, extend, prepend as lists, and inserts as list of tuples. 193 """ 194 d = {} 195 if self._update: 196 d['update'] = self._update 197 if self._extend: 198 d['extend'] = self._extend 199 if self._prepend: 200 d['prepend'] = self._prepend 201 elif self._inserts: 202 d['inserts'] = self._inserts 203 return d 204 205 def __repr__(self): 206 if self._value is not None: 207 return "<%s value=%r>" % (self.__class__.__name__, self._value) 208 else: 209 return "<%s %r>" % (self.__class__.__name__, self.to_dict()) 210 211 212def _is_section_key(key): 213 """Is a Config key a section name (does it start with a capital)?""" 214 if key and key[0].upper()==key[0] and not key.startswith('_'): 215 return True 216 else: 217 return False 218 219 220class Config(dict): 221 """An attribute-based dict that can do smart merges. 222 223 Accessing a field on a config object for the first time populates the key 224 with either a nested Config object for keys starting with capitals 225 or :class:`.LazyConfigValue` for lowercase keys, 226 allowing quick assignments such as:: 227 228 c = Config() 229 c.Class.int_trait = 5 230 c.Class.list_trait.append("x") 231 232 """ 233 234 def __init__(self, *args, **kwds): 235 dict.__init__(self, *args, **kwds) 236 self._ensure_subconfig() 237 238 def _ensure_subconfig(self): 239 """ensure that sub-dicts that should be Config objects are 240 241 casts dicts that are under section keys to Config objects, 242 which is necessary for constructing Config objects from dict literals. 243 """ 244 for key in self: 245 obj = self[key] 246 if _is_section_key(key) \ 247 and isinstance(obj, dict) \ 248 and not isinstance(obj, Config): 249 setattr(self, key, Config(obj)) 250 251 def _merge(self, other): 252 """deprecated alias, use Config.merge()""" 253 self.merge(other) 254 255 def merge(self, other): 256 """merge another config object into this one""" 257 to_update = {} 258 for k, v in other.items(): 259 if k not in self: 260 to_update[k] = v 261 else: # I have this key 262 if isinstance(v, Config) and isinstance(self[k], Config): 263 # Recursively merge common sub Configs 264 self[k].merge(v) 265 elif isinstance(v, LazyConfigValue): 266 self[k] = v.merge_into(self[k]) 267 else: 268 # Plain updates for non-Configs 269 to_update[k] = v 270 271 self.update(to_update) 272 273 def collisions(self, other): 274 """Check for collisions between two config objects. 275 276 Returns a dict of the form {"Class": {"trait": "collision message"}}`, 277 indicating which values have been ignored. 278 279 An empty dict indicates no collisions. 280 """ 281 collisions = {} 282 for section in self: 283 if section not in other: 284 continue 285 mine = self[section] 286 theirs = other[section] 287 for key in mine: 288 if key in theirs and mine[key] != theirs[key]: 289 collisions.setdefault(section, {}) 290 collisions[section][key] = "%r ignored, using %r" % (mine[key], theirs[key]) 291 return collisions 292 293 def __contains__(self, key): 294 # allow nested contains of the form `"Section.key" in config` 295 if '.' in key: 296 first, remainder = key.split('.', 1) 297 if first not in self: 298 return False 299 return remainder in self[first] 300 301 return super(Config, self).__contains__(key) 302 303 # .has_key is deprecated for dictionaries. 304 has_key = __contains__ 305 306 def _has_section(self, key): 307 return _is_section_key(key) and key in self 308 309 def copy(self): 310 return type(self)(dict.copy(self)) 311 312 def __copy__(self): 313 return self.copy() 314 315 def __deepcopy__(self, memo): 316 new_config = type(self)() 317 for key, value in self.items(): 318 if isinstance(value, (Config, LazyConfigValue)): 319 # deep copy config objects 320 value = copy.deepcopy(value, memo) 321 elif type(value) in {dict, list, set, tuple}: 322 # shallow copy plain container traits 323 value = copy.copy(value) 324 new_config[key] = value 325 return new_config 326 327 def __getitem__(self, key): 328 try: 329 return dict.__getitem__(self, key) 330 except KeyError: 331 if _is_section_key(key): 332 c = Config() 333 dict.__setitem__(self, key, c) 334 return c 335 elif not key.startswith('_'): 336 # undefined, create lazy value, used for container methods 337 v = LazyConfigValue() 338 dict.__setitem__(self, key, v) 339 return v 340 else: 341 raise KeyError 342 343 def __setitem__(self, key, value): 344 if _is_section_key(key): 345 if not isinstance(value, Config): 346 raise ValueError('values whose keys begin with an uppercase ' 347 'char must be Config instances: %r, %r' % (key, value)) 348 dict.__setitem__(self, key, value) 349 350 def __getattr__(self, key): 351 if key.startswith('__'): 352 return dict.__getattr__(self, key) 353 try: 354 return self.__getitem__(key) 355 except KeyError as e: 356 raise AttributeError(e) 357 358 def __setattr__(self, key, value): 359 if key.startswith('__'): 360 return dict.__setattr__(self, key, value) 361 try: 362 self.__setitem__(key, value) 363 except KeyError as e: 364 raise AttributeError(e) 365 366 def __delattr__(self, key): 367 if key.startswith('__'): 368 return dict.__delattr__(self, key) 369 try: 370 dict.__delitem__(self, key) 371 except KeyError as e: 372 raise AttributeError(e) 373 374 375class DeferredConfig: 376 """Class for deferred-evaluation of config from CLI""" 377 pass 378 379 def get_value(self, trait): 380 raise NotImplementedError("Implement in subclasses") 381 382 def _super_repr(self): 383 # explicitly call super on direct parent 384 return super(self.__class__, self).__repr__() 385 386 387class DeferredConfigString(str, DeferredConfig): 388 """Config value for loading config from a string 389 390 Interpretation is deferred until it is loaded into the trait. 391 392 Subclass of str for backward compatibility. 393 394 This class is only used for values that are not listed 395 in the configurable classes. 396 397 When config is loaded, `trait.from_string` will be used. 398 399 If an error is raised in `.from_string`, 400 the original string is returned. 401 402 .. versionadded:: 5.0 403 """ 404 def get_value(self, trait): 405 """Get the value stored in this string""" 406 s = str(self) 407 try: 408 return trait.from_string(s) 409 except Exception: 410 # exception casting from string, 411 # let the original string lie. 412 # this will raise a more informative error when config is loaded. 413 return s 414 415 def __repr__(self): 416 return '%s(%s)' % (self.__class__.__name__, self._super_repr()) 417 418 419class DeferredConfigList(list, DeferredConfig): 420 """Config value for loading config from a list of strings 421 422 Interpretation is deferred until it is loaded into the trait. 423 424 This class is only used for values that are not listed 425 in the configurable classes. 426 427 When config is loaded, `trait.from_string_list` will be used. 428 429 If an error is raised in `.from_string_list`, 430 the original string list is returned. 431 432 .. versionadded:: 5.0 433 """ 434 def get_value(self, trait): 435 """Get the value stored in this string""" 436 if hasattr(trait, "from_string_list"): 437 src = list(self) 438 cast = trait.from_string_list 439 else: 440 # only allow one item 441 if len(self) > 1: 442 raise ValueError(f"{trait.name} only accepts one value, got {len(self)}: {list(self)}") 443 src = self[0] 444 cast = trait.from_string 445 446 try: 447 return cast(src) 448 except Exception: 449 # exception casting from string, 450 # let the original value lie. 451 # this will raise a more informative error when config is loaded. 452 return src 453 454 def __repr__(self): 455 return '%s(%s)' % (self.__class__.__name__, self._super_repr()) 456 457 458#----------------------------------------------------------------------------- 459# Config loading classes 460#----------------------------------------------------------------------------- 461 462 463class ConfigLoader(object): 464 """A object for loading configurations from just about anywhere. 465 466 The resulting configuration is packaged as a :class:`Config`. 467 468 Notes 469 ----- 470 A :class:`ConfigLoader` does one thing: load a config from a source 471 (file, command line arguments) and returns the data as a :class:`Config` object. 472 There are lots of things that :class:`ConfigLoader` does not do. It does 473 not implement complex logic for finding config files. It does not handle 474 default values or merge multiple configs. These things need to be 475 handled elsewhere. 476 """ 477 478 def _log_default(self): 479 from traitlets.log import get_logger 480 return get_logger() 481 482 def __init__(self, log=None): 483 """A base class for config loaders. 484 485 log : instance of :class:`logging.Logger` to use. 486 By default logger of :meth:`traitlets.config.application.Application.instance()` 487 will be used 488 489 Examples 490 -------- 491 >>> cl = ConfigLoader() 492 >>> config = cl.load_config() 493 >>> config 494 {} 495 """ 496 self.clear() 497 if log is None: 498 self.log = self._log_default() 499 self.log.debug('Using default logger') 500 else: 501 self.log = log 502 503 def clear(self): 504 self.config = Config() 505 506 def load_config(self): 507 """Load a config from somewhere, return a :class:`Config` instance. 508 509 Usually, this will cause self.config to be set and then returned. 510 However, in most cases, :meth:`ConfigLoader.clear` should be called 511 to erase any previous state. 512 """ 513 self.clear() 514 return self.config 515 516 517class FileConfigLoader(ConfigLoader): 518 """A base class for file based configurations. 519 520 As we add more file based config loaders, the common logic should go 521 here. 522 """ 523 524 def __init__(self, filename, path=None, **kw): 525 """Build a config loader for a filename and path. 526 527 Parameters 528 ---------- 529 filename : str 530 The file name of the config file. 531 path : str, list, tuple 532 The path to search for the config file on, or a sequence of 533 paths to try in order. 534 """ 535 super(FileConfigLoader, self).__init__(**kw) 536 self.filename = filename 537 self.path = path 538 self.full_filename = '' 539 540 def _find_file(self): 541 """Try to find the file by searching the paths.""" 542 self.full_filename = filefind(self.filename, self.path) 543 544class JSONFileConfigLoader(FileConfigLoader): 545 """A JSON file loader for config 546 547 Can also act as a context manager that rewrite the configuration file to disk on exit. 548 549 Example:: 550 551 with JSONFileConfigLoader('myapp.json','/home/jupyter/configurations/') as c: 552 c.MyNewConfigurable.new_value = 'Updated' 553 554 """ 555 556 def load_config(self): 557 """Load the config from a file and return it as a Config object.""" 558 self.clear() 559 try: 560 self._find_file() 561 except IOError as e: 562 raise ConfigFileNotFound(str(e)) 563 dct = self._read_file_as_dict() 564 self.config = self._convert_to_config(dct) 565 return self.config 566 567 def _read_file_as_dict(self): 568 with open(self.full_filename) as f: 569 return json.load(f) 570 571 def _convert_to_config(self, dictionary): 572 if 'version' in dictionary: 573 version = dictionary.pop('version') 574 else: 575 version = 1 576 577 if version == 1: 578 return Config(dictionary) 579 else: 580 raise ValueError('Unknown version of JSON config file: {version}'.format(version=version)) 581 582 def __enter__(self): 583 self.load_config() 584 return self.config 585 586 def __exit__(self, exc_type, exc_value, traceback): 587 """ 588 Exit the context manager but do not handle any errors. 589 590 In case of any error, we do not want to write the potentially broken 591 configuration to disk. 592 """ 593 self.config.version = 1 594 json_config = json.dumps(self.config, indent=2) 595 with open(self.full_filename, 'w') as f: 596 f.write(json_config) 597 598 599 600class PyFileConfigLoader(FileConfigLoader): 601 """A config loader for pure python files. 602 603 This is responsible for locating a Python config file by filename and 604 path, then executing it to construct a Config object. 605 """ 606 607 def load_config(self): 608 """Load the config from a file and return it as a Config object.""" 609 self.clear() 610 try: 611 self._find_file() 612 except IOError as e: 613 raise ConfigFileNotFound(str(e)) 614 self._read_file_as_dict() 615 return self.config 616 617 def load_subconfig(self, fname, path=None): 618 """Injected into config file namespace as load_subconfig""" 619 if path is None: 620 path = self.path 621 622 loader = self.__class__(fname, path) 623 try: 624 sub_config = loader.load_config() 625 except ConfigFileNotFound: 626 # Pass silently if the sub config is not there, 627 # treat it as an empty config file. 628 pass 629 else: 630 self.config.merge(sub_config) 631 632 def _read_file_as_dict(self): 633 """Load the config file into self.config, with recursive loading.""" 634 def get_config(): 635 """Unnecessary now, but a deprecation warning is more trouble than it's worth.""" 636 return self.config 637 638 namespace = dict( 639 c=self.config, 640 load_subconfig=self.load_subconfig, 641 get_config=get_config, 642 __file__=self.full_filename, 643 ) 644 conf_filename = self.full_filename 645 with open(conf_filename, 'rb') as f: 646 exec(compile(f.read(), conf_filename, 'exec'), namespace, namespace) 647 648 649class CommandLineConfigLoader(ConfigLoader): 650 """A config loader for command line arguments. 651 652 As we add more command line based loaders, the common logic should go 653 here. 654 """ 655 656 def _exec_config_str(self, lhs, rhs, trait=None): 657 """execute self.config.<lhs> = <rhs> 658 659 * expands ~ with expanduser 660 * interprets value with trait if available 661 """ 662 value = rhs 663 if isinstance(value, DeferredConfig): 664 if trait: 665 # trait available, reify config immediately 666 value = value.get_value(trait) 667 elif isinstance(rhs, DeferredConfigList) and len(rhs) == 1: 668 # single item, make it a deferred str 669 value = DeferredConfigString(os.path.expanduser(rhs[0])) 670 else: 671 if trait: 672 value = trait.from_string(value) 673 else: 674 value = DeferredConfigString(value) 675 676 *path, key = lhs.split(".") 677 section = self.config 678 for part in path: 679 section = section[part] 680 section[key] = value 681 return 682 683 def _load_flag(self, cfg): 684 """update self.config from a flag, which can be a dict or Config""" 685 if isinstance(cfg, (dict, Config)): 686 # don't clobber whole config sections, update 687 # each section from config: 688 for sec, c in cfg.items(): 689 self.config[sec].update(c) 690 else: 691 raise TypeError("Invalid flag: %r" % cfg) 692 693# match --Class.trait keys for argparse 694# matches: 695# --Class.trait 696# --x 697# -x 698 699class_trait_opt_pattern = re.compile(r'^\-?\-[A-Za-z][\w]*(\.[\w]+)*$') 700 701_DOT_REPLACEMENT = "__DOT__" 702_DASH_REPLACEMENT = "__DASH__" 703 704 705class _KVAction(argparse.Action): 706 """Custom argparse action for handling --Class.trait=x 707 708 Always 709 """ 710 def __call__(self, parser, namespace, values, option_string=None): 711 if isinstance(values, str): 712 values = [values] 713 values = ["-" if v is _DASH_REPLACEMENT else v for v in values] 714 items = getattr(namespace, self.dest, None) 715 if items is None: 716 items = DeferredConfigList() 717 else: 718 items = DeferredConfigList(items) 719 items.extend(values) 720 setattr(namespace, self.dest, items) 721 722 723class _DefaultOptionDict(dict): 724 """Like the default options dict 725 726 but acts as if all --Class.trait options are predefined 727 """ 728 def _add_kv_action(self, key): 729 self[key] = _KVAction( 730 option_strings=[key], 731 dest=key.lstrip("-").replace(".", _DOT_REPLACEMENT), 732 # use metavar for display purposes 733 metavar=key.lstrip("-"), 734 ) 735 736 def __contains__(self, key): 737 if '=' in key: 738 return False 739 if super().__contains__(key): 740 return True 741 742 if key.startswith("-") and class_trait_opt_pattern.match(key): 743 self._add_kv_action(key) 744 return True 745 return False 746 747 def __getitem__(self, key): 748 if key in self: 749 return super().__getitem__(key) 750 else: 751 raise KeyError(key) 752 753 def get(self, key, default=None): 754 try: 755 return self[key] 756 except KeyError: 757 return default 758 759 760class _KVArgParser(argparse.ArgumentParser): 761 """subclass of ArgumentParser where any --Class.trait option is implicitly defined""" 762 def parse_known_args(self, args=None, namespace=None): 763 # must be done immediately prior to parsing because if we do it in init, 764 # registration of explicit actions via parser.add_option will fail during setup 765 for container in (self, self._optionals): 766 container._option_string_actions = _DefaultOptionDict( 767 container._option_string_actions) 768 return super().parse_known_args(args, namespace) 769 770 771class ArgParseConfigLoader(CommandLineConfigLoader): 772 """A loader that uses the argparse module to load from the command line.""" 773 774 parser_class = ArgumentParser 775 776 def __init__(self, argv=None, aliases=None, flags=None, log=None, classes=(), 777 *parser_args, **parser_kw): 778 """Create a config loader for use with argparse. 779 780 Parameters 781 ---------- 782 classes : optional, list 783 The classes to scan for *container* config-traits and decide 784 for their "multiplicity" when adding them as *argparse* arguments. 785 argv : optional, list 786 If given, used to read command-line arguments from, otherwise 787 sys.argv[1:] is used. 788 *parser_args : tuple 789 A tuple of positional arguments that will be passed to the 790 constructor of :class:`argparse.ArgumentParser`. 791 **parser_kw : dict 792 A tuple of keyword arguments that will be passed to the 793 constructor of :class:`argparse.ArgumentParser`. 794 aliases : dict of str to str 795 Dict of aliases to full traitlests names for CLI parsing 796 flags : dict of str to str 797 Dict of flags to full traitlests names for CLI parsing 798 log 799 Passed to `ConfigLoader` 800 801 Returns 802 ------- 803 config : Config 804 The resulting Config object. 805 """ 806 super(CommandLineConfigLoader, self).__init__(log=log) 807 self.clear() 808 if argv is None: 809 argv = sys.argv[1:] 810 self.argv = argv 811 self.aliases = aliases or {} 812 self.flags = flags or {} 813 self.classes = classes 814 815 self.parser_args = parser_args 816 self.version = parser_kw.pop("version", None) 817 kwargs = dict(argument_default=argparse.SUPPRESS) 818 kwargs.update(parser_kw) 819 self.parser_kw = kwargs 820 821 def load_config(self, argv=None, aliases=None, flags=_deprecated, classes=None): 822 """Parse command line arguments and return as a Config object. 823 824 Parameters 825 ---------- 826 argv : optional, list 827 If given, a list with the structure of sys.argv[1:] to parse 828 arguments from. If not given, the instance's self.argv attribute 829 (given at construction time) is used. 830 flags 831 Deprecated in traitlets 5.0, instanciate the config loader with the flags. 832 833 """ 834 835 if flags is not _deprecated: 836 warnings.warn( 837 "The `flag` argument to load_config is deprecated since Traitlets " 838 f"5.0 and will be ignored, pass flags the `{type(self)}` constructor.", 839 DeprecationWarning, 840 stacklevel=2, 841 ) 842 843 self.clear() 844 if argv is None: 845 argv = self.argv 846 if aliases is not None: 847 self.aliases = aliases 848 if classes is not None: 849 self.classes = classes 850 self._create_parser() 851 self._parse_args(argv) 852 self._convert_to_config() 853 return self.config 854 855 def get_extra_args(self): 856 if hasattr(self, 'extra_args'): 857 return self.extra_args 858 else: 859 return [] 860 861 def _create_parser(self): 862 self.parser = self.parser_class(*self.parser_args, **self.parser_kw) 863 self._add_arguments(self.aliases, self.flags, self.classes) 864 865 def _add_arguments(self, aliases, flags, classes): 866 raise NotImplementedError("subclasses must implement _add_arguments") 867 868 def _parse_args(self, args): 869 """self.parser->self.parsed_data""" 870 uargs = [cast_unicode(a) for a in args] 871 872 unpacked_aliases = {} 873 if self.aliases: 874 unpacked_aliases = {} 875 for alias, alias_target in self.aliases.items(): 876 if alias in self.flags: 877 continue 878 if not isinstance(alias, tuple): 879 short_alias, alias = alias, None 880 else: 881 short_alias, alias = alias 882 for al in (short_alias, alias): 883 if al is None: 884 continue 885 if len(al) == 1: 886 unpacked_aliases["-" + al] = "--" + alias_target 887 unpacked_aliases["--" + al] = "--" + alias_target 888 889 def _replace(arg): 890 if arg == "-": 891 return _DASH_REPLACEMENT 892 for k, v in unpacked_aliases.items(): 893 if arg == k: 894 return v 895 if arg.startswith(k + "="): 896 return v + "=" + arg[len(k) + 1:] 897 return arg 898 899 if '--' in uargs: 900 idx = uargs.index('--') 901 extra_args = uargs[idx+1:] 902 to_parse = uargs[:idx] 903 else: 904 extra_args = [] 905 to_parse = uargs 906 to_parse = [_replace(a) for a in to_parse] 907 908 self.parsed_data = self.parser.parse_args(to_parse) 909 self.extra_args = extra_args 910 911 def _convert_to_config(self): 912 """self.parsed_data->self.config""" 913 for k, v in vars(self.parsed_data).items(): 914 *path, key = k.split(".") 915 section = self.config 916 for p in path: 917 section = section[p] 918 setattr(section, key, v) 919 920 921class _FlagAction(argparse.Action): 922 """ArgParse action to handle a flag""" 923 def __init__(self, *args, **kwargs): 924 self.flag = kwargs.pop('flag') 925 self.alias = kwargs.pop('alias', None) 926 kwargs['const'] = Undefined 927 if not self.alias: 928 kwargs['nargs'] = 0 929 super(_FlagAction, self).__init__(*args, **kwargs) 930 931 def __call__(self, parser, namespace, values, option_string=None): 932 if self.nargs == 0 or values is Undefined: 933 if not hasattr(namespace, '_flags'): 934 namespace._flags = [] 935 namespace._flags.append(self.flag) 936 else: 937 setattr(namespace, self.alias, values) 938 939 940class KVArgParseConfigLoader(ArgParseConfigLoader): 941 """A config loader that loads aliases and flags with argparse, 942 943 as well as arbitrary --Class.trait value 944 """ 945 946 parser_class = _KVArgParser 947 948 def _add_arguments(self, aliases, flags, classes): 949 alias_flags = {} 950 paa = self.parser.add_argument 951 self.parser.set_defaults(_flags=[]) 952 paa("extra_args", nargs="*") 953 954 ## An index of all container traits collected:: 955 # 956 # { <traitname>: (<trait>, <argparse-kwds>) } 957 # 958 # Used to add the correct type into the `config` tree. 959 # Used also for aliases, not to re-collect them. 960 self.argparse_traits = argparse_traits = {} 961 for cls in classes: 962 for traitname, trait in cls.class_traits(config=True).items(): 963 argname = '%s.%s' % (cls.__name__, traitname) 964 argparse_kwds = {'type': str} 965 if isinstance(trait, (Container, Dict)): 966 multiplicity = trait.metadata.get('multiplicity', 'append') 967 if multiplicity == 'append': 968 argparse_kwds['action'] = multiplicity 969 else: 970 argparse_kwds['nargs'] = multiplicity 971 argparse_traits[argname] = (trait, argparse_kwds) 972 973 for keys, (value, _) in flags.items(): 974 if not isinstance(keys, tuple): 975 keys = (keys,) 976 for key in keys: 977 if key in aliases: 978 alias_flags[aliases[key]] = value 979 continue 980 keys = ('-' + key, '--' + key) if len(key) == 1 else ('--' + key,) 981 paa(*keys, action=_FlagAction, flag=value) 982 983 for keys, traitname in aliases.items(): 984 if not isinstance(keys, tuple): 985 keys = (keys,) 986 987 for key in keys: 988 argparse_kwds = { 989 'type': str, 990 'dest': traitname.replace(".", _DOT_REPLACEMENT), 991 'metavar': traitname, 992 } 993 if traitname in argparse_traits: 994 argparse_kwds.update(argparse_traits[traitname][1]) 995 if 'action' in argparse_kwds and traitname in alias_flags: 996 # flag sets 'action', so can't have flag & alias with custom action 997 # on the same name 998 raise ArgumentError( 999 "The alias `%s` for the 'append' sequence " 1000 "config-trait `%s` cannot be also a flag!'" 1001 % (key, traitname)) 1002 if traitname in alias_flags: 1003 # alias and flag. 1004 # when called with 0 args: flag 1005 # when called with >= 1: alias 1006 argparse_kwds.setdefault('nargs', '?') 1007 argparse_kwds['action'] = _FlagAction 1008 argparse_kwds['flag'] = alias_flags[traitname] 1009 argparse_kwds['alias'] = traitname 1010 keys = ('-' + key, '--' + key) if len(key) == 1 else ('--'+ key,) 1011 paa(*keys, **argparse_kwds) 1012 1013 def _convert_to_config(self): 1014 """self.parsed_data->self.config, parse unrecognized extra args via KVLoader.""" 1015 extra_args = self.extra_args 1016 1017 for lhs, rhs in vars(self.parsed_data).items(): 1018 if lhs == "extra_args": 1019 self.extra_args = ["-" if a == _DASH_REPLACEMENT else a for a in rhs] + extra_args 1020 continue 1021 elif lhs == '_flags': 1022 # _flags will be handled later 1023 continue 1024 1025 lhs = lhs.replace(_DOT_REPLACEMENT, ".") 1026 if '.' not in lhs: 1027 # probably a mistyped alias, but not technically illegal 1028 self.log.warning("Unrecognized alias: '%s', it will have no effect.", lhs) 1029 trait = None 1030 1031 if isinstance(rhs, list): 1032 rhs = DeferredConfigList(rhs) 1033 elif isinstance(rhs, str): 1034 rhs = DeferredConfigString(rhs) 1035 1036 trait = self.argparse_traits.get(lhs) 1037 if trait: 1038 trait = trait[0] 1039 1040 # eval the KV assignment 1041 try: 1042 self._exec_config_str(lhs, rhs, trait) 1043 except Exception as e: 1044 # cast deferred to nicer repr for the error 1045 # DeferredList->list, etc 1046 if isinstance(rhs, DeferredConfig): 1047 rhs = rhs._super_repr() 1048 raise ArgumentError(f"Error loading argument {lhs}={rhs}, {e}") 1049 1050 for subc in self.parsed_data._flags: 1051 self._load_flag(subc) 1052 1053 1054class KeyValueConfigLoader(KVArgParseConfigLoader): 1055 """Deprecated in traitlets 5.0 1056 1057 Use KVArgParseConfigLoader 1058 """ 1059 def __init__(self, *args, **kwargs): 1060 warnings.warn( 1061 "KeyValueConfigLoader is deprecated since Traitlets 5.0." 1062 " Use KVArgParseConfigLoader instead.", 1063 DeprecationWarning, 1064 stacklevel=2, 1065 ) 1066 super().__init__(*args, **kwargs) 1067 1068 1069def load_pyconfig_files(config_files, path): 1070 """Load multiple Python config files, merging each of them in turn. 1071 1072 Parameters 1073 ---------- 1074 config_files : list of str 1075 List of config files names to load and merge into the config. 1076 path : unicode 1077 The full path to the location of the config files. 1078 """ 1079 config = Config() 1080 for cf in config_files: 1081 loader = PyFileConfigLoader(cf, path=path) 1082 try: 1083 next_config = loader.load_config() 1084 except ConfigFileNotFound: 1085 pass 1086 except: 1087 raise 1088 else: 1089 config.merge(next_config) 1090 return config 1091