1# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> 2# 3# This file is part of Ansible 4# 5# Ansible is free software: you can redistribute it and/or modify 6# it under the terms of the GNU General Public License as published by 7# the Free Software Foundation, either version 3 of the License, or 8# (at your option) any later version. 9# 10# Ansible is distributed in the hope that it will be useful, 11# but WITHOUT ANY WARRANTY; without even the implied warranty of 12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13# GNU General Public License for more details. 14# 15# You should have received a copy of the GNU General Public License 16# along with Ansible. If not, see <http://www.gnu.org/licenses/>. 17 18# Make coding more python3-ish 19from __future__ import (absolute_import, division, print_function) 20__metaclass__ = type 21 22import os 23 24from ansible import constants as C 25from ansible.errors import AnsibleError, AnsibleParserError, AnsibleAssertionError 26from ansible.module_utils._text import to_text 27from ansible.module_utils.six import iteritems, binary_type, text_type 28from ansible.module_utils.common._collections_compat import Container, Mapping, Set, Sequence 29from ansible.playbook.attribute import FieldAttribute 30from ansible.playbook.base import Base 31from ansible.playbook.collectionsearch import CollectionSearch 32from ansible.playbook.conditional import Conditional 33from ansible.playbook.helpers import load_list_of_blocks 34from ansible.playbook.role.metadata import RoleMetadata 35from ansible.playbook.taggable import Taggable 36from ansible.plugins.loader import add_all_plugin_dirs 37from ansible.utils.collection_loader import AnsibleCollectionConfig 38from ansible.utils.vars import combine_vars 39 40 41__all__ = ['Role', 'hash_params'] 42 43# TODO: this should be a utility function, but can't be a member of 44# the role due to the fact that it would require the use of self 45# in a static method. This is also used in the base class for 46# strategies (ansible/plugins/strategy/__init__.py) 47 48 49def hash_params(params): 50 """ 51 Construct a data structure of parameters that is hashable. 52 53 This requires changing any mutable data structures into immutable ones. 54 We chose a frozenset because role parameters have to be unique. 55 56 .. warning:: this does not handle unhashable scalars. Two things 57 mitigate that limitation: 58 59 1) There shouldn't be any unhashable scalars specified in the yaml 60 2) Our only choice would be to return an error anyway. 61 """ 62 # Any container is unhashable if it contains unhashable items (for 63 # instance, tuple() is a Hashable subclass but if it contains a dict, it 64 # cannot be hashed) 65 if isinstance(params, Container) and not isinstance(params, (text_type, binary_type)): 66 if isinstance(params, Mapping): 67 try: 68 # Optimistically hope the contents are all hashable 69 new_params = frozenset(params.items()) 70 except TypeError: 71 new_params = set() 72 for k, v in params.items(): 73 # Hash each entry individually 74 new_params.add((k, hash_params(v))) 75 new_params = frozenset(new_params) 76 77 elif isinstance(params, (Set, Sequence)): 78 try: 79 # Optimistically hope the contents are all hashable 80 new_params = frozenset(params) 81 except TypeError: 82 new_params = set() 83 for v in params: 84 # Hash each entry individually 85 new_params.add(hash_params(v)) 86 new_params = frozenset(new_params) 87 else: 88 # This is just a guess. 89 new_params = frozenset(params) 90 return new_params 91 92 # Note: We do not handle unhashable scalars but our only choice would be 93 # to raise an error there anyway. 94 return frozenset((params,)) 95 96 97class Role(Base, Conditional, Taggable, CollectionSearch): 98 99 _delegate_to = FieldAttribute(isa='string') 100 _delegate_facts = FieldAttribute(isa='bool') 101 102 def __init__(self, play=None, from_files=None, from_include=False, validate=True): 103 self._role_name = None 104 self._role_path = None 105 self._role_collection = None 106 self._role_params = dict() 107 self._loader = None 108 109 self._metadata = None 110 self._play = play 111 self._parents = [] 112 self._dependencies = [] 113 self._task_blocks = [] 114 self._handler_blocks = [] 115 self._compiled_handler_blocks = None 116 self._default_vars = dict() 117 self._role_vars = dict() 118 self._had_task_run = dict() 119 self._completed = dict() 120 self._should_validate = validate 121 122 if from_files is None: 123 from_files = {} 124 self._from_files = from_files 125 126 # Indicates whether this role was included via include/import_role 127 self.from_include = from_include 128 129 super(Role, self).__init__() 130 131 def __repr__(self): 132 return self.get_name() 133 134 def get_name(self, include_role_fqcn=True): 135 if include_role_fqcn: 136 return '.'.join(x for x in (self._role_collection, self._role_name) if x) 137 return self._role_name 138 139 @staticmethod 140 def load(role_include, play, parent_role=None, from_files=None, from_include=False, validate=True): 141 142 if from_files is None: 143 from_files = {} 144 try: 145 # The ROLE_CACHE is a dictionary of role names, with each entry 146 # containing another dictionary corresponding to a set of parameters 147 # specified for a role as the key and the Role() object itself. 148 # We use frozenset to make the dictionary hashable. 149 150 params = role_include.get_role_params() 151 if role_include.when is not None: 152 params['when'] = role_include.when 153 if role_include.tags is not None: 154 params['tags'] = role_include.tags 155 if from_files is not None: 156 params['from_files'] = from_files 157 if role_include.vars: 158 params['vars'] = role_include.vars 159 160 params['from_include'] = from_include 161 162 hashed_params = hash_params(params) 163 if role_include.get_name() in play.ROLE_CACHE: 164 for (entry, role_obj) in iteritems(play.ROLE_CACHE[role_include.get_name()]): 165 if hashed_params == entry: 166 if parent_role: 167 role_obj.add_parent(parent_role) 168 return role_obj 169 170 # TODO: need to fix cycle detection in role load (maybe use an empty dict 171 # for the in-flight in role cache as a sentinel that we're already trying to load 172 # that role?) 173 # see https://github.com/ansible/ansible/issues/61527 174 r = Role(play=play, from_files=from_files, from_include=from_include, validate=validate) 175 r._load_role_data(role_include, parent_role=parent_role) 176 177 if role_include.get_name() not in play.ROLE_CACHE: 178 play.ROLE_CACHE[role_include.get_name()] = dict() 179 180 # FIXME: how to handle cache keys for collection-based roles, since they're technically adjustable per task? 181 play.ROLE_CACHE[role_include.get_name()][hashed_params] = r 182 return r 183 184 except RuntimeError: 185 raise AnsibleError("A recursion loop was detected with the roles specified. Make sure child roles do not have dependencies on parent roles", 186 obj=role_include._ds) 187 188 def _load_role_data(self, role_include, parent_role=None): 189 self._role_name = role_include.role 190 self._role_path = role_include.get_role_path() 191 self._role_collection = role_include._role_collection 192 self._role_params = role_include.get_role_params() 193 self._variable_manager = role_include.get_variable_manager() 194 self._loader = role_include.get_loader() 195 196 if parent_role: 197 self.add_parent(parent_role) 198 199 # copy over all field attributes from the RoleInclude 200 # update self._attributes directly, to avoid squashing 201 for (attr_name, _) in iteritems(self._valid_attrs): 202 if attr_name in ('when', 'tags'): 203 self._attributes[attr_name] = self._extend_value( 204 self._attributes[attr_name], 205 role_include._attributes[attr_name], 206 ) 207 else: 208 self._attributes[attr_name] = role_include._attributes[attr_name] 209 210 # vars and default vars are regular dictionaries 211 self._role_vars = self._load_role_yaml('vars', main=self._from_files.get('vars'), allow_dir=True) 212 if self._role_vars is None: 213 self._role_vars = {} 214 elif not isinstance(self._role_vars, Mapping): 215 raise AnsibleParserError("The vars/main.yml file for role '%s' must contain a dictionary of variables" % self._role_name) 216 217 self._default_vars = self._load_role_yaml('defaults', main=self._from_files.get('defaults'), allow_dir=True) 218 if self._default_vars is None: 219 self._default_vars = {} 220 elif not isinstance(self._default_vars, Mapping): 221 raise AnsibleParserError("The defaults/main.yml file for role '%s' must contain a dictionary of variables" % self._role_name) 222 223 # load the role's other files, if they exist 224 metadata = self._load_role_yaml('meta') 225 if metadata: 226 self._metadata = RoleMetadata.load(metadata, owner=self, variable_manager=self._variable_manager, loader=self._loader) 227 self._dependencies = self._load_dependencies() 228 else: 229 self._metadata = RoleMetadata() 230 231 # reset collections list; roles do not inherit collections from parents, just use the defaults 232 # FUTURE: use a private config default for this so we can allow it to be overridden later 233 self.collections = [] 234 235 # configure plugin/collection loading; either prepend the current role's collection or configure legacy plugin loading 236 # FIXME: need exception for explicit ansible.legacy? 237 if self._role_collection: # this is a collection-hosted role 238 self.collections.insert(0, self._role_collection) 239 else: # this is a legacy role, but set the default collection if there is one 240 default_collection = AnsibleCollectionConfig.default_collection 241 if default_collection: 242 self.collections.insert(0, default_collection) 243 # legacy role, ensure all plugin dirs under the role are added to plugin search path 244 add_all_plugin_dirs(self._role_path) 245 246 # collections can be specified in metadata for legacy or collection-hosted roles 247 if self._metadata.collections: 248 self.collections.extend((c for c in self._metadata.collections if c not in self.collections)) 249 250 # if any collections were specified, ensure that core or legacy synthetic collections are always included 251 if self.collections: 252 # default append collection is core for collection-hosted roles, legacy for others 253 default_append_collection = 'ansible.builtin' if self._role_collection else 'ansible.legacy' 254 if 'ansible.builtin' not in self.collections and 'ansible.legacy' not in self.collections: 255 self.collections.append(default_append_collection) 256 257 task_data = self._load_role_yaml('tasks', main=self._from_files.get('tasks')) 258 259 if self._should_validate: 260 role_argspecs = self._get_role_argspecs() 261 task_data = self._prepend_validation_task(task_data, role_argspecs) 262 263 if task_data: 264 try: 265 self._task_blocks = load_list_of_blocks(task_data, play=self._play, role=self, loader=self._loader, variable_manager=self._variable_manager) 266 except AssertionError as e: 267 raise AnsibleParserError("The tasks/main.yml file for role '%s' must contain a list of tasks" % self._role_name, 268 obj=task_data, orig_exc=e) 269 270 handler_data = self._load_role_yaml('handlers', main=self._from_files.get('handlers')) 271 if handler_data: 272 try: 273 self._handler_blocks = load_list_of_blocks(handler_data, play=self._play, role=self, use_handlers=True, loader=self._loader, 274 variable_manager=self._variable_manager) 275 except AssertionError as e: 276 raise AnsibleParserError("The handlers/main.yml file for role '%s' must contain a list of tasks" % self._role_name, 277 obj=handler_data, orig_exc=e) 278 279 def _get_role_argspecs(self): 280 """Get the role argument spec data. 281 282 Role arg specs can be in one of two files in the role meta subdir: argument_specs.yml 283 or main.yml. The former has precedence over the latter. Data is not combined 284 between the files. 285 286 :returns: A dict of all data under the top-level ``argument_specs`` YAML key 287 in the argument spec file. An empty dict is returned if there is no 288 argspec data. 289 """ 290 base_argspec_path = os.path.join(self._role_path, 'meta', 'argument_specs') 291 292 for ext in C.YAML_FILENAME_EXTENSIONS: 293 full_path = base_argspec_path + ext 294 if self._loader.path_exists(full_path): 295 # Note: _load_role_yaml() takes care of rebuilding the path. 296 argument_specs = self._load_role_yaml('meta', main='argument_specs') 297 try: 298 return argument_specs.get('argument_specs') or {} 299 except AttributeError: 300 return {} 301 302 # We did not find the meta/argument_specs.[yml|yaml] file, so use the spec 303 # dict from the role meta data, if it exists. Ansible 2.11 and later will 304 # have the 'argument_specs' attribute, but earlier versions will not. 305 return getattr(self._metadata, 'argument_specs', {}) 306 307 def _prepend_validation_task(self, task_data, argspecs): 308 '''Insert a role validation task if we have a role argument spec. 309 310 This method will prepend a validation task to the front of the role task 311 list to perform argument spec validation before any other tasks, if an arg spec 312 exists for the entry point. Entry point defaults to `main`. 313 314 :param task_data: List of tasks loaded from the role. 315 :param argspecs: The role argument spec data dict. 316 317 :returns: The (possibly modified) task list. 318 ''' 319 if argspecs: 320 # Determine the role entry point so we can retrieve the correct argument spec. 321 # This comes from the `tasks_from` value to include_role or import_role. 322 entrypoint = self._from_files.get('tasks', 'main') 323 entrypoint_arg_spec = argspecs.get(entrypoint) 324 325 if entrypoint_arg_spec: 326 validation_task = self._create_validation_task(entrypoint_arg_spec, entrypoint) 327 328 # Prepend our validate_argument_spec action to happen before any tasks provided by the role. 329 # 'any tasks' can and does include 0 or None tasks, in which cases we create a list of tasks and add our 330 # validate_argument_spec task 331 if not task_data: 332 task_data = [] 333 task_data.insert(0, validation_task) 334 return task_data 335 336 def _create_validation_task(self, argument_spec, entrypoint_name): 337 '''Create a new task data structure that uses the validate_argument_spec action plugin. 338 339 :param argument_spec: The arg spec definition for a particular role entry point. 340 This will be the entire arg spec for the entry point as read from the input file. 341 :param entrypoint_name: The name of the role entry point associated with the 342 supplied `argument_spec`. 343 ''' 344 345 # If the arg spec provides a short description, use it to flesh out the validation task name 346 task_name = "Validating arguments against arg spec '%s'" % entrypoint_name 347 if 'short_description' in argument_spec: 348 task_name = task_name + ' - ' + argument_spec['short_description'] 349 350 return { 351 'action': { 352 'module': 'ansible.builtin.validate_argument_spec', 353 # Pass only the 'options' portion of the arg spec to the module. 354 'argument_spec': argument_spec.get('options', {}), 355 'provided_arguments': self._role_params, 356 'validate_args_context': { 357 'type': 'role', 358 'name': self._role_name, 359 'argument_spec_name': entrypoint_name, 360 'path': self._role_path 361 }, 362 }, 363 'name': task_name, 364 'tags': ['always'], 365 } 366 367 def _load_role_yaml(self, subdir, main=None, allow_dir=False): 368 ''' 369 Find and load role YAML files and return data found. 370 :param subdir: subdir of role to search (vars, files, tasks, handlers, defaults) 371 :type subdir: string 372 :param main: filename to match, will default to 'main.<ext>' if not provided. 373 :type main: string 374 :param allow_dir: If true we combine results of multiple matching files found. 375 If false, highlander rules. Only for vars(dicts) and not tasks(lists). 376 :type allow_dir: bool 377 378 :returns: data from the matched file(s), type can be dict or list depending on vars or tasks. 379 ''' 380 data = None 381 file_path = os.path.join(self._role_path, subdir) 382 if self._loader.path_exists(file_path) and self._loader.is_directory(file_path): 383 # Valid extensions and ordering for roles is hard-coded to maintain portability 384 extensions = ['.yml', '.yaml', '.json'] # same as default for YAML_FILENAME_EXTENSIONS 385 386 # look for files w/o extensions before/after bare name depending on it being set or not 387 # keep 'main' as original to figure out errors if no files found 388 if main is None: 389 _main = 'main' 390 extensions.append('') 391 else: 392 _main = main 393 extensions.insert(0, '') 394 395 # not really 'find_vars_files' but find_files_with_extensions_default_to_yaml_filename_extensions 396 found_files = self._loader.find_vars_files(file_path, _main, extensions, allow_dir) 397 if found_files: 398 for found in found_files: 399 new_data = self._loader.load_from_file(found) 400 if new_data: 401 if data is not None and isinstance(new_data, Mapping): 402 data = combine_vars(data, new_data) 403 else: 404 data = new_data 405 406 # found data so no need to continue unless we want to merge 407 if not allow_dir: 408 break 409 410 elif main is not None: 411 # this won't trigger with default only when <subdir>_from is specified 412 raise AnsibleParserError("Could not find specified file in role: %s/%s" % (subdir, main)) 413 414 return data 415 416 def _load_dependencies(self): 417 ''' 418 Recursively loads role dependencies from the metadata list of 419 dependencies, if it exists 420 ''' 421 422 deps = [] 423 if self._metadata: 424 for role_include in self._metadata.dependencies: 425 r = Role.load(role_include, play=self._play, parent_role=self) 426 deps.append(r) 427 428 return deps 429 430 # other functions 431 432 def add_parent(self, parent_role): 433 ''' adds a role to the list of this roles parents ''' 434 if not isinstance(parent_role, Role): 435 raise AnsibleAssertionError() 436 437 if parent_role not in self._parents: 438 self._parents.append(parent_role) 439 440 def get_parents(self): 441 return self._parents 442 443 def get_default_vars(self, dep_chain=None): 444 dep_chain = [] if dep_chain is None else dep_chain 445 446 default_vars = dict() 447 for dep in self.get_all_dependencies(): 448 default_vars = combine_vars(default_vars, dep.get_default_vars()) 449 if dep_chain: 450 for parent in dep_chain: 451 default_vars = combine_vars(default_vars, parent._default_vars) 452 default_vars = combine_vars(default_vars, self._default_vars) 453 return default_vars 454 455 def get_inherited_vars(self, dep_chain=None): 456 dep_chain = [] if dep_chain is None else dep_chain 457 458 inherited_vars = dict() 459 460 if dep_chain: 461 for parent in dep_chain: 462 inherited_vars = combine_vars(inherited_vars, parent._role_vars) 463 return inherited_vars 464 465 def get_role_params(self, dep_chain=None): 466 dep_chain = [] if dep_chain is None else dep_chain 467 468 params = {} 469 if dep_chain: 470 for parent in dep_chain: 471 params = combine_vars(params, parent._role_params) 472 params = combine_vars(params, self._role_params) 473 return params 474 475 def get_vars(self, dep_chain=None, include_params=True): 476 dep_chain = [] if dep_chain is None else dep_chain 477 478 all_vars = self.get_inherited_vars(dep_chain) 479 480 for dep in self.get_all_dependencies(): 481 all_vars = combine_vars(all_vars, dep.get_vars(include_params=include_params)) 482 483 all_vars = combine_vars(all_vars, self.vars) 484 all_vars = combine_vars(all_vars, self._role_vars) 485 if include_params: 486 all_vars = combine_vars(all_vars, self.get_role_params(dep_chain=dep_chain)) 487 488 return all_vars 489 490 def get_direct_dependencies(self): 491 return self._dependencies[:] 492 493 def get_all_dependencies(self): 494 ''' 495 Returns a list of all deps, built recursively from all child dependencies, 496 in the proper order in which they should be executed or evaluated. 497 ''' 498 499 child_deps = [] 500 501 for dep in self.get_direct_dependencies(): 502 for child_dep in dep.get_all_dependencies(): 503 child_deps.append(child_dep) 504 child_deps.append(dep) 505 506 return child_deps 507 508 def get_task_blocks(self): 509 return self._task_blocks[:] 510 511 def get_handler_blocks(self, play, dep_chain=None): 512 # Do not recreate this list each time ``get_handler_blocks`` is called. 513 # Cache the results so that we don't potentially overwrite with copied duplicates 514 # 515 # ``get_handler_blocks`` may be called when handling ``import_role`` during parsing 516 # as well as with ``Play.compile_roles_handlers`` from ``TaskExecutor`` 517 if self._compiled_handler_blocks: 518 return self._compiled_handler_blocks 519 520 self._compiled_handler_blocks = block_list = [] 521 522 # update the dependency chain here 523 if dep_chain is None: 524 dep_chain = [] 525 new_dep_chain = dep_chain + [self] 526 527 for dep in self.get_direct_dependencies(): 528 dep_blocks = dep.get_handler_blocks(play=play, dep_chain=new_dep_chain) 529 block_list.extend(dep_blocks) 530 531 for task_block in self._handler_blocks: 532 new_task_block = task_block.copy() 533 new_task_block._dep_chain = new_dep_chain 534 new_task_block._play = play 535 block_list.append(new_task_block) 536 537 return block_list 538 539 def has_run(self, host): 540 ''' 541 Returns true if this role has been iterated over completely and 542 at least one task was run 543 ''' 544 545 return host.name in self._completed and not self._metadata.allow_duplicates 546 547 def compile(self, play, dep_chain=None): 548 ''' 549 Returns the task list for this role, which is created by first 550 recursively compiling the tasks for all direct dependencies, and 551 then adding on the tasks for this role. 552 553 The role compile() also remembers and saves the dependency chain 554 with each task, so tasks know by which route they were found, and 555 can correctly take their parent's tags/conditionals into account. 556 ''' 557 from ansible.playbook.block import Block 558 from ansible.playbook.task import Task 559 560 block_list = [] 561 562 # update the dependency chain here 563 if dep_chain is None: 564 dep_chain = [] 565 new_dep_chain = dep_chain + [self] 566 567 deps = self.get_direct_dependencies() 568 for dep in deps: 569 dep_blocks = dep.compile(play=play, dep_chain=new_dep_chain) 570 block_list.extend(dep_blocks) 571 572 for task_block in self._task_blocks: 573 new_task_block = task_block.copy() 574 new_task_block._dep_chain = new_dep_chain 575 new_task_block._play = play 576 block_list.append(new_task_block) 577 578 eor_block = Block(play=play) 579 eor_block._loader = self._loader 580 eor_block._role = self 581 eor_block._variable_manager = self._variable_manager 582 eor_block.run_once = False 583 584 eor_task = Task(block=eor_block) 585 eor_task._role = self 586 eor_task.action = 'meta' 587 eor_task.args = {'_raw_params': 'role_complete'} 588 eor_task.implicit = True 589 eor_task.tags = ['always'] 590 eor_task.when = True 591 592 eor_block.block = [eor_task] 593 block_list.append(eor_block) 594 595 return block_list 596 597 def serialize(self, include_deps=True): 598 res = super(Role, self).serialize() 599 600 res['_role_name'] = self._role_name 601 res['_role_path'] = self._role_path 602 res['_role_vars'] = self._role_vars 603 res['_role_params'] = self._role_params 604 res['_default_vars'] = self._default_vars 605 res['_had_task_run'] = self._had_task_run.copy() 606 res['_completed'] = self._completed.copy() 607 608 if self._metadata: 609 res['_metadata'] = self._metadata.serialize() 610 611 if include_deps: 612 deps = [] 613 for role in self.get_direct_dependencies(): 614 deps.append(role.serialize()) 615 res['_dependencies'] = deps 616 617 parents = [] 618 for parent in self._parents: 619 parents.append(parent.serialize(include_deps=False)) 620 res['_parents'] = parents 621 622 return res 623 624 def deserialize(self, data, include_deps=True): 625 self._role_name = data.get('_role_name', '') 626 self._role_path = data.get('_role_path', '') 627 self._role_vars = data.get('_role_vars', dict()) 628 self._role_params = data.get('_role_params', dict()) 629 self._default_vars = data.get('_default_vars', dict()) 630 self._had_task_run = data.get('_had_task_run', dict()) 631 self._completed = data.get('_completed', dict()) 632 633 if include_deps: 634 deps = [] 635 for dep in data.get('_dependencies', []): 636 r = Role() 637 r.deserialize(dep) 638 deps.append(r) 639 setattr(self, '_dependencies', deps) 640 641 parent_data = data.get('_parents', []) 642 parents = [] 643 for parent in parent_data: 644 r = Role() 645 r.deserialize(parent, include_deps=False) 646 parents.append(r) 647 setattr(self, '_parents', parents) 648 649 metadata_data = data.get('_metadata') 650 if metadata_data: 651 m = RoleMetadata() 652 m.deserialize(metadata_data) 653 self._metadata = m 654 655 super(Role, self).deserialize(data) 656 657 def set_loader(self, loader): 658 self._loader = loader 659 for parent in self._parents: 660 parent.set_loader(loader) 661 for dep in self.get_direct_dependencies(): 662 dep.set_loader(loader) 663