1# Copyright: (c) 2017, Ansible Project 2# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) 3 4from __future__ import (absolute_import, division, print_function) 5__metaclass__ = type 6 7import atexit 8import io 9import os 10import os.path 11import sys 12import stat 13import tempfile 14import traceback 15from collections import namedtuple 16 17from yaml import load as yaml_load 18try: 19 # use C version if possible for speedup 20 from yaml import CSafeLoader as SafeLoader 21except ImportError: 22 from yaml import SafeLoader 23 24from ansible.config.data import ConfigData 25from ansible.errors import AnsibleOptionsError, AnsibleError 26from ansible.module_utils._text import to_text, to_bytes, to_native 27from ansible.module_utils.common._collections_compat import Mapping, Sequence 28from ansible.module_utils.six import PY3, string_types 29from ansible.module_utils.six.moves import configparser 30from ansible.module_utils.parsing.convert_bool import boolean 31from ansible.parsing.quoting import unquote 32from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode 33from ansible.utils import py3compat 34from ansible.utils.path import cleanup_tmp_file, makedirs_safe, unfrackpath 35 36 37Plugin = namedtuple('Plugin', 'name type') 38Setting = namedtuple('Setting', 'name value origin type') 39 40INTERNAL_DEFS = {'lookup': ('_terms',)} 41 42 43def _get_entry(plugin_type, plugin_name, config): 44 ''' construct entry for requested config ''' 45 entry = '' 46 if plugin_type: 47 entry += 'plugin_type: %s ' % plugin_type 48 if plugin_name: 49 entry += 'plugin: %s ' % plugin_name 50 entry += 'setting: %s ' % config 51 return entry 52 53 54# FIXME: see if we can unify in module_utils with similar function used by argspec 55def ensure_type(value, value_type, origin=None): 56 ''' return a configuration variable with casting 57 :arg value: The value to ensure correct typing of 58 :kwarg value_type: The type of the value. This can be any of the following strings: 59 :boolean: sets the value to a True or False value 60 :bool: Same as 'boolean' 61 :integer: Sets the value to an integer or raises a ValueType error 62 :int: Same as 'integer' 63 :float: Sets the value to a float or raises a ValueType error 64 :list: Treats the value as a comma separated list. Split the value 65 and return it as a python list. 66 :none: Sets the value to None 67 :path: Expands any environment variables and tilde's in the value. 68 :tmppath: Create a unique temporary directory inside of the directory 69 specified by value and return its path. 70 :temppath: Same as 'tmppath' 71 :tmp: Same as 'tmppath' 72 :pathlist: Treat the value as a typical PATH string. (On POSIX, this 73 means colon separated strings.) Split the value and then expand 74 each part for environment variables and tildes. 75 :pathspec: Treat the value as a PATH string. Expands any environment variables 76 tildes's in the value. 77 :str: Sets the value to string types. 78 :string: Same as 'str' 79 ''' 80 81 errmsg = '' 82 basedir = None 83 if origin and os.path.isabs(origin) and os.path.exists(to_bytes(origin)): 84 basedir = origin 85 86 if value_type: 87 value_type = value_type.lower() 88 89 if value is not None: 90 if value_type in ('boolean', 'bool'): 91 value = boolean(value, strict=False) 92 93 elif value_type in ('integer', 'int'): 94 value = int(value) 95 96 elif value_type == 'float': 97 value = float(value) 98 99 elif value_type == 'list': 100 if isinstance(value, string_types): 101 value = [unquote(x.strip()) for x in value.split(',')] 102 elif not isinstance(value, Sequence): 103 errmsg = 'list' 104 105 elif value_type == 'none': 106 if value == "None": 107 value = None 108 109 if value is not None: 110 errmsg = 'None' 111 112 elif value_type == 'path': 113 if isinstance(value, string_types): 114 value = resolve_path(value, basedir=basedir) 115 else: 116 errmsg = 'path' 117 118 elif value_type in ('tmp', 'temppath', 'tmppath'): 119 if isinstance(value, string_types): 120 value = resolve_path(value, basedir=basedir) 121 if not os.path.exists(value): 122 makedirs_safe(value, 0o700) 123 prefix = 'ansible-local-%s' % os.getpid() 124 value = tempfile.mkdtemp(prefix=prefix, dir=value) 125 atexit.register(cleanup_tmp_file, value, warn=True) 126 else: 127 errmsg = 'temppath' 128 129 elif value_type == 'pathspec': 130 if isinstance(value, string_types): 131 value = value.split(os.pathsep) 132 133 if isinstance(value, Sequence): 134 value = [resolve_path(x, basedir=basedir) for x in value] 135 else: 136 errmsg = 'pathspec' 137 138 elif value_type == 'pathlist': 139 if isinstance(value, string_types): 140 value = [x.strip() for x in value.split(',')] 141 142 if isinstance(value, Sequence): 143 value = [resolve_path(x, basedir=basedir) for x in value] 144 else: 145 errmsg = 'pathlist' 146 147 elif value_type in ('dict', 'dictionary'): 148 if not isinstance(value, Mapping): 149 errmsg = 'dictionary' 150 151 elif value_type in ('str', 'string'): 152 if isinstance(value, (string_types, AnsibleVaultEncryptedUnicode, bool, int, float, complex)): 153 value = unquote(to_text(value, errors='surrogate_or_strict')) 154 else: 155 errmsg = 'string' 156 157 # defaults to string type 158 elif isinstance(value, (string_types, AnsibleVaultEncryptedUnicode)): 159 value = unquote(to_text(value, errors='surrogate_or_strict')) 160 161 if errmsg: 162 raise ValueError('Invalid type provided for "%s": %s' % (errmsg, to_native(value))) 163 164 return to_text(value, errors='surrogate_or_strict', nonstring='passthru') 165 166 167# FIXME: see if this can live in utils/path 168def resolve_path(path, basedir=None): 169 ''' resolve relative or 'variable' paths ''' 170 if '{{CWD}}' in path: # allow users to force CWD using 'magic' {{CWD}} 171 path = path.replace('{{CWD}}', os.getcwd()) 172 173 return unfrackpath(path, follow=False, basedir=basedir) 174 175 176# FIXME: generic file type? 177def get_config_type(cfile): 178 179 ftype = None 180 if cfile is not None: 181 ext = os.path.splitext(cfile)[-1] 182 if ext in ('.ini', '.cfg'): 183 ftype = 'ini' 184 elif ext in ('.yaml', '.yml'): 185 ftype = 'yaml' 186 else: 187 raise AnsibleOptionsError("Unsupported configuration file extension for %s: %s" % (cfile, to_native(ext))) 188 189 return ftype 190 191 192# FIXME: can move to module_utils for use for ini plugins also? 193def get_ini_config_value(p, entry): 194 ''' returns the value of last ini entry found ''' 195 value = None 196 if p is not None: 197 try: 198 value = p.get(entry.get('section', 'defaults'), entry.get('key', ''), raw=True) 199 except Exception: # FIXME: actually report issues here 200 pass 201 return value 202 203 204def find_ini_config_file(warnings=None): 205 ''' Load INI Config File order(first found is used): ENV, CWD, HOME, /usr/local/etc/ansible ''' 206 # FIXME: eventually deprecate ini configs 207 208 if warnings is None: 209 # Note: In this case, warnings does nothing 210 warnings = set() 211 212 # A value that can never be a valid path so that we can tell if ANSIBLE_CONFIG was set later 213 # We can't use None because we could set path to None. 214 SENTINEL = object 215 216 potential_paths = [] 217 218 # Environment setting 219 path_from_env = os.getenv("ANSIBLE_CONFIG", SENTINEL) 220 if path_from_env is not SENTINEL: 221 path_from_env = unfrackpath(path_from_env, follow=False) 222 if os.path.isdir(to_bytes(path_from_env)): 223 path_from_env = os.path.join(path_from_env, "ansible.cfg") 224 potential_paths.append(path_from_env) 225 226 # Current working directory 227 warn_cmd_public = False 228 try: 229 cwd = os.getcwd() 230 perms = os.stat(cwd) 231 cwd_cfg = os.path.join(cwd, "ansible.cfg") 232 if perms.st_mode & stat.S_IWOTH: 233 # Working directory is world writable so we'll skip it. 234 # Still have to look for a file here, though, so that we know if we have to warn 235 if os.path.exists(cwd_cfg): 236 warn_cmd_public = True 237 else: 238 potential_paths.append(to_text(cwd_cfg, errors='surrogate_or_strict')) 239 except OSError: 240 # If we can't access cwd, we'll simply skip it as a possible config source 241 pass 242 243 # Per user location 244 potential_paths.append(unfrackpath("~/.ansible.cfg", follow=False)) 245 246 # System location 247 potential_paths.append("/usr/local/etc/ansible/ansible.cfg") 248 249 for path in potential_paths: 250 b_path = to_bytes(path) 251 if os.path.exists(b_path) and os.access(b_path, os.R_OK): 252 break 253 else: 254 path = None 255 256 # Emit a warning if all the following are true: 257 # * We did not use a config from ANSIBLE_CONFIG 258 # * There's an ansible.cfg in the current working directory that we skipped 259 if path_from_env != path and warn_cmd_public: 260 warnings.add(u"Ansible is being run in a world writable directory (%s)," 261 u" ignoring it as an ansible.cfg source." 262 u" For more information see" 263 u" https://docs.ansible.com/ansible/devel/reference_appendices/config.html#cfg-in-world-writable-dir" 264 % to_text(cwd)) 265 266 return path 267 268 269def _add_base_defs_deprecations(base_defs): 270 '''Add deprecation source 'ansible.builtin' to deprecations in base.yml''' 271 def process(entry): 272 if 'deprecated' in entry: 273 entry['deprecated']['collection_name'] = 'ansible.builtin' 274 275 for dummy, data in base_defs.items(): 276 process(data) 277 for section in ('ini', 'env', 'vars'): 278 if section in data: 279 for entry in data[section]: 280 process(entry) 281 282 283class ConfigManager(object): 284 285 DEPRECATED = [] 286 WARNINGS = set() 287 288 def __init__(self, conf_file=None, defs_file=None): 289 290 self._base_defs = {} 291 self._plugins = {} 292 self._parsers = {} 293 294 self._config_file = conf_file 295 self.data = ConfigData() 296 297 self._base_defs = self._read_config_yaml_file(defs_file or ('%s/base.yml' % os.path.dirname(__file__))) 298 _add_base_defs_deprecations(self._base_defs) 299 300 if self._config_file is None: 301 # set config using ini 302 self._config_file = find_ini_config_file(self.WARNINGS) 303 304 # consume configuration 305 if self._config_file: 306 # initialize parser and read config 307 self._parse_config_file() 308 309 # update constants 310 self.update_config_data() 311 312 def _read_config_yaml_file(self, yml_file): 313 # TODO: handle relative paths as relative to the directory containing the current playbook instead of CWD 314 # Currently this is only used with absolute paths to the `ansible/config` directory 315 yml_file = to_bytes(yml_file) 316 if os.path.exists(yml_file): 317 with open(yml_file, 'rb') as config_def: 318 return yaml_load(config_def, Loader=SafeLoader) or {} 319 raise AnsibleError( 320 "Missing base YAML definition file (bad install?): %s" % to_native(yml_file)) 321 322 def _parse_config_file(self, cfile=None): 323 ''' return flat configuration settings from file(s) ''' 324 # TODO: take list of files with merge/nomerge 325 326 if cfile is None: 327 cfile = self._config_file 328 329 ftype = get_config_type(cfile) 330 if cfile is not None: 331 if ftype == 'ini': 332 kwargs = {} 333 if PY3: 334 kwargs['inline_comment_prefixes'] = (';',) 335 self._parsers[cfile] = configparser.ConfigParser(**kwargs) 336 with open(to_bytes(cfile), 'rb') as f: 337 try: 338 cfg_text = to_text(f.read(), errors='surrogate_or_strict') 339 except UnicodeError as e: 340 raise AnsibleOptionsError("Error reading config file(%s) because the config file was not utf8 encoded: %s" % (cfile, to_native(e))) 341 try: 342 if PY3: 343 self._parsers[cfile].read_string(cfg_text) 344 else: 345 cfg_file = io.StringIO(cfg_text) 346 self._parsers[cfile].readfp(cfg_file) 347 except configparser.Error as e: 348 raise AnsibleOptionsError("Error reading config file (%s): %s" % (cfile, to_native(e))) 349 # FIXME: this should eventually handle yaml config files 350 # elif ftype == 'yaml': 351 # with open(cfile, 'rb') as config_stream: 352 # self._parsers[cfile] = yaml.safe_load(config_stream) 353 else: 354 raise AnsibleOptionsError("Unsupported configuration file type: %s" % to_native(ftype)) 355 356 def _find_yaml_config_files(self): 357 ''' Load YAML Config Files in order, check merge flags, keep origin of settings''' 358 pass 359 360 def get_plugin_options(self, plugin_type, name, keys=None, variables=None, direct=None): 361 362 options = {} 363 defs = self.get_configuration_definitions(plugin_type, name) 364 for option in defs: 365 options[option] = self.get_config_value(option, plugin_type=plugin_type, plugin_name=name, keys=keys, variables=variables, direct=direct) 366 367 return options 368 369 def get_plugin_vars(self, plugin_type, name): 370 371 pvars = [] 372 for pdef in self.get_configuration_definitions(plugin_type, name).values(): 373 if 'vars' in pdef and pdef['vars']: 374 for var_entry in pdef['vars']: 375 pvars.append(var_entry['name']) 376 return pvars 377 378 def get_configuration_definition(self, name, plugin_type=None, plugin_name=None): 379 380 ret = {} 381 if plugin_type is None: 382 ret = self._base_defs.get(name, None) 383 elif plugin_name is None: 384 ret = self._plugins.get(plugin_type, {}).get(name, None) 385 else: 386 ret = self._plugins.get(plugin_type, {}).get(plugin_name, {}).get(name, None) 387 388 return ret 389 390 def get_configuration_definitions(self, plugin_type=None, name=None, ignore_private=False): 391 ''' just list the possible settings, either base or for specific plugins or plugin ''' 392 393 ret = {} 394 if plugin_type is None: 395 ret = self._base_defs 396 elif name is None: 397 ret = self._plugins.get(plugin_type, {}) 398 else: 399 ret = self._plugins.get(plugin_type, {}).get(name, {}) 400 401 if ignore_private: 402 for cdef in list(ret.keys()): 403 if cdef.startswith('_'): 404 del ret[cdef] 405 406 return ret 407 408 def _loop_entries(self, container, entry_list): 409 ''' repeat code for value entry assignment ''' 410 411 value = None 412 origin = None 413 for entry in entry_list: 414 name = entry.get('name') 415 try: 416 temp_value = container.get(name, None) 417 except UnicodeEncodeError: 418 self.WARNINGS.add(u'value for config entry {0} contains invalid characters, ignoring...'.format(to_text(name))) 419 continue 420 if temp_value is not None: # only set if entry is defined in container 421 # inline vault variables should be converted to a text string 422 if isinstance(temp_value, AnsibleVaultEncryptedUnicode): 423 temp_value = to_text(temp_value, errors='surrogate_or_strict') 424 425 value = temp_value 426 origin = name 427 428 # deal with deprecation of setting source, if used 429 if 'deprecated' in entry: 430 self.DEPRECATED.append((entry['name'], entry['deprecated'])) 431 432 return value, origin 433 434 def get_config_value(self, config, cfile=None, plugin_type=None, plugin_name=None, keys=None, variables=None, direct=None): 435 ''' wrapper ''' 436 437 try: 438 value, _drop = self.get_config_value_and_origin(config, cfile=cfile, plugin_type=plugin_type, plugin_name=plugin_name, 439 keys=keys, variables=variables, direct=direct) 440 except AnsibleError: 441 raise 442 except Exception as e: 443 raise AnsibleError("Unhandled exception when retrieving %s:\n%s" % (config, to_native(e)), orig_exc=e) 444 return value 445 446 def get_config_value_and_origin(self, config, cfile=None, plugin_type=None, plugin_name=None, keys=None, variables=None, direct=None): 447 ''' Given a config key figure out the actual value and report on the origin of the settings ''' 448 if cfile is None: 449 # use default config 450 cfile = self._config_file 451 452 # Note: sources that are lists listed in low to high precedence (last one wins) 453 value = None 454 origin = None 455 456 defs = self.get_configuration_definitions(plugin_type, plugin_name) 457 if config in defs: 458 459 aliases = defs[config].get('aliases', []) 460 461 # direct setting via plugin arguments, can set to None so we bypass rest of processing/defaults 462 direct_aliases = [] 463 if direct: 464 direct_aliases = [direct[alias] for alias in aliases if alias in direct] 465 if direct and config in direct: 466 value = direct[config] 467 origin = 'Direct' 468 elif direct and direct_aliases: 469 value = direct_aliases[0] 470 origin = 'Direct' 471 472 else: 473 # Use 'variable overrides' if present, highest precedence, but only present when querying running play 474 if variables and defs[config].get('vars'): 475 value, origin = self._loop_entries(variables, defs[config]['vars']) 476 origin = 'var: %s' % origin 477 478 # use playbook keywords if you have em 479 if value is None and keys: 480 if config in keys: 481 value = keys[config] 482 keyword = config 483 484 elif aliases: 485 for alias in aliases: 486 if alias in keys: 487 value = keys[alias] 488 keyword = alias 489 break 490 491 if value is not None: 492 origin = 'keyword: %s' % keyword 493 494 if value is None and 'cli' in defs[config]: 495 # avoid circular import .. until valid 496 from ansible import context 497 value, origin = self._loop_entries(context.CLIARGS, defs[config]['cli']) 498 origin = 'cli: %s' % origin 499 500 # env vars are next precedence 501 if value is None and defs[config].get('env'): 502 value, origin = self._loop_entries(py3compat.environ, defs[config]['env']) 503 origin = 'env: %s' % origin 504 505 # try config file entries next, if we have one 506 if self._parsers.get(cfile, None) is None: 507 self._parse_config_file(cfile) 508 509 if value is None and cfile is not None: 510 ftype = get_config_type(cfile) 511 if ftype and defs[config].get(ftype): 512 if ftype == 'ini': 513 # load from ini config 514 try: # FIXME: generalize _loop_entries to allow for files also, most of this code is dupe 515 for ini_entry in defs[config]['ini']: 516 temp_value = get_ini_config_value(self._parsers[cfile], ini_entry) 517 if temp_value is not None: 518 value = temp_value 519 origin = cfile 520 if 'deprecated' in ini_entry: 521 self.DEPRECATED.append(('[%s]%s' % (ini_entry['section'], ini_entry['key']), ini_entry['deprecated'])) 522 except Exception as e: 523 sys.stderr.write("Error while loading ini config %s: %s" % (cfile, to_native(e))) 524 elif ftype == 'yaml': 525 # FIXME: implement, also , break down key from defs (. notation???) 526 origin = cfile 527 528 # set default if we got here w/o a value 529 if value is None: 530 if defs[config].get('required', False): 531 if not plugin_type or config not in INTERNAL_DEFS.get(plugin_type, {}): 532 raise AnsibleError("No setting was provided for required configuration %s" % 533 to_native(_get_entry(plugin_type, plugin_name, config))) 534 else: 535 value = defs[config].get('default') 536 origin = 'default' 537 # skip typing as this is a templated default that will be resolved later in constants, which has needed vars 538 if plugin_type is None and isinstance(value, string_types) and (value.startswith('{{') and value.endswith('}}')): 539 return value, origin 540 541 # ensure correct type, can raise exceptions on mismatched types 542 try: 543 value = ensure_type(value, defs[config].get('type'), origin=origin) 544 except ValueError as e: 545 if origin.startswith('env:') and value == '': 546 # this is empty env var for non string so we can set to default 547 origin = 'default' 548 value = ensure_type(defs[config].get('default'), defs[config].get('type'), origin=origin) 549 else: 550 raise AnsibleOptionsError('Invalid type for configuration option %s: %s' % 551 (to_native(_get_entry(plugin_type, plugin_name, config)), to_native(e))) 552 553 # deal with restricted values 554 if value is not None and 'choices' in defs[config] and defs[config]['choices'] is not None: 555 invalid_choices = True # assume the worst! 556 if defs[config].get('type') == 'list': 557 # for a list type, compare all values in type are allowed 558 invalid_choices = not all(choice in defs[config]['choices'] for choice in value) 559 else: 560 # these should be only the simple data types (string, int, bool, float, etc) .. ignore dicts for now 561 invalid_choices = value not in defs[config]['choices'] 562 563 if invalid_choices: 564 raise AnsibleOptionsError('Invalid value "%s" for configuration option "%s", valid values are: %s' % 565 (value, to_native(_get_entry(plugin_type, plugin_name, config)), defs[config]['choices'])) 566 567 # deal with deprecation of the setting 568 if 'deprecated' in defs[config] and origin != 'default': 569 self.DEPRECATED.append((config, defs[config].get('deprecated'))) 570 else: 571 raise AnsibleError('Requested entry (%s) was not defined in configuration.' % to_native(_get_entry(plugin_type, plugin_name, config))) 572 573 return value, origin 574 575 def initialize_plugin_configuration_definitions(self, plugin_type, name, defs): 576 577 if plugin_type not in self._plugins: 578 self._plugins[plugin_type] = {} 579 580 self._plugins[plugin_type][name] = defs 581 582 def update_config_data(self, defs=None, configfile=None): 583 ''' really: update constants ''' 584 585 if defs is None: 586 defs = self._base_defs 587 588 if configfile is None: 589 configfile = self._config_file 590 591 if not isinstance(defs, dict): 592 raise AnsibleOptionsError("Invalid configuration definition type: %s for %s" % (type(defs), defs)) 593 594 # update the constant for config file 595 self.data.update_setting(Setting('CONFIG_FILE', configfile, '', 'string')) 596 597 origin = None 598 # env and config defs can have several entries, ordered in list from lowest to highest precedence 599 for config in defs: 600 if not isinstance(defs[config], dict): 601 raise AnsibleOptionsError("Invalid configuration definition '%s': type is %s" % (to_native(config), type(defs[config]))) 602 603 # get value and origin 604 try: 605 value, origin = self.get_config_value_and_origin(config, configfile) 606 except Exception as e: 607 # Printing the problem here because, in the current code: 608 # (1) we can't reach the error handler for AnsibleError before we 609 # hit a different error due to lack of working config. 610 # (2) We don't have access to display yet because display depends on config 611 # being properly loaded. 612 # 613 # If we start getting double errors printed from this section of code, then the 614 # above problem #1 has been fixed. Revamp this to be more like the try: except 615 # in get_config_value() at that time. 616 sys.stderr.write("Unhandled error:\n %s\n\n" % traceback.format_exc()) 617 raise AnsibleError("Invalid settings supplied for %s: %s\n" % (config, to_native(e)), orig_exc=e) 618 619 # set the constant 620 self.data.update_setting(Setting(config, value, origin, defs[config].get('type', 'string'))) 621