1# Copyright (c) 2017 Ansible Project 2# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) 3 4from __future__ import (absolute_import, division, print_function) 5__metaclass__ = type 6 7DOCUMENTATION = ''' 8 name: aws_ec2 9 plugin_type: inventory 10 short_description: EC2 inventory source 11 requirements: 12 - boto3 13 - botocore 14 extends_documentation_fragment: 15 - inventory_cache 16 - constructed 17 description: 18 - Get inventory hosts from Amazon Web Services EC2. 19 - Uses a YAML configuration file that ends with C(aws_ec2.(yml|yaml)). 20 notes: 21 - If no credentials are provided and the control node has an associated IAM instance profile then the 22 role will be used for authentication. 23 author: 24 - Sloane Hertel (@s-hertel) 25 options: 26 aws_profile: 27 description: The AWS profile 28 type: str 29 aliases: [ boto_profile ] 30 env: 31 - name: AWS_DEFAULT_PROFILE 32 - name: AWS_PROFILE 33 aws_access_key: 34 description: The AWS access key to use. 35 type: str 36 aliases: [ aws_access_key_id ] 37 env: 38 - name: EC2_ACCESS_KEY 39 - name: AWS_ACCESS_KEY 40 - name: AWS_ACCESS_KEY_ID 41 aws_secret_key: 42 description: The AWS secret key that corresponds to the access key. 43 type: str 44 aliases: [ aws_secret_access_key ] 45 env: 46 - name: EC2_SECRET_KEY 47 - name: AWS_SECRET_KEY 48 - name: AWS_SECRET_ACCESS_KEY 49 aws_security_token: 50 description: The AWS security token if using temporary access and secret keys. 51 type: str 52 env: 53 - name: EC2_SECURITY_TOKEN 54 - name: AWS_SESSION_TOKEN 55 - name: AWS_SECURITY_TOKEN 56 plugin: 57 description: Token that ensures this is a source file for the plugin. 58 required: True 59 choices: ['aws_ec2'] 60 iam_role_arn: 61 description: The ARN of the IAM role to assume to perform the inventory lookup. You should still provide AWS 62 credentials with enough privilege to perform the AssumeRole action. 63 version_added: '2.9' 64 regions: 65 description: 66 - A list of regions in which to describe EC2 instances. 67 - If empty (the default) default this will include all regions, except possibly restricted ones like us-gov-west-1 and cn-north-1. 68 type: list 69 default: [] 70 hostnames: 71 description: 72 - A list in order of precedence for hostname variables. 73 - You can use the options specified in U(http://docs.aws.amazon.com/cli/latest/reference/ec2/describe-instances.html#options). 74 - To use tags as hostnames use the syntax tag:Name=Value to use the hostname Name_Value, or tag:Name to use the value of the Name tag. 75 type: list 76 default: [] 77 filters: 78 description: 79 - A dictionary of filter value pairs. 80 - Available filters are listed here U(http://docs.aws.amazon.com/cli/latest/reference/ec2/describe-instances.html#options). 81 type: dict 82 default: {} 83 include_extra_api_calls: 84 description: 85 - Add two additional API calls for every instance to include 'persistent' and 'events' host variables. 86 - Spot instances may be persistent and instances may have associated events. 87 type: bool 88 default: False 89 version_added: '2.8' 90 strict_permissions: 91 description: 92 - By default if a 403 (Forbidden) error code is encountered this plugin will fail. 93 - You can set this option to False in the inventory config file which will allow 403 errors to be gracefully skipped. 94 type: bool 95 default: True 96 use_contrib_script_compatible_sanitization: 97 description: 98 - By default this plugin is using a general group name sanitization to create safe and usable group names for use in Ansible. 99 This option allows you to override that, in efforts to allow migration from the old inventory script and 100 matches the sanitization of groups when the script's ``replace_dash_in_groups`` option is set to ``False``. 101 To replicate behavior of ``replace_dash_in_groups = True`` with constructed groups, 102 you will need to replace hyphens with underscores via the regex_replace filter for those entries. 103 - For this to work you should also turn off the TRANSFORM_INVALID_GROUP_CHARS setting, 104 otherwise the core engine will just use the standard sanitization on top. 105 - This is not the default as such names break certain functionality as not all characters are valid Python identifiers 106 which group names end up being used as. 107 type: bool 108 default: False 109 version_added: '2.8' 110''' 111 112EXAMPLES = ''' 113# Minimal example using environment vars or instance role credentials 114# Fetch all hosts in us-east-1, the hostname is the public DNS if it exists, otherwise the private IP address 115plugin: aws_ec2 116regions: 117 - us-east-1 118 119# Example using filters, ignoring permission errors, and specifying the hostname precedence 120plugin: aws_ec2 121boto_profile: aws_profile 122# Populate inventory with instances in these regions 123regions: 124 - us-east-1 125 - us-east-2 126filters: 127 # All instances with their `Environment` tag set to `dev` 128 tag:Environment: dev 129 # All dev and QA hosts 130 tag:Environment: 131 - dev 132 - qa 133 instance.group-id: sg-xxxxxxxx 134# Ignores 403 errors rather than failing 135strict_permissions: False 136# Note: I(hostnames) sets the inventory_hostname. To modify ansible_host without modifying 137# inventory_hostname use compose (see example below). 138hostnames: 139 - tag:Name=Tag1,Name=Tag2 # Return specific hosts only 140 - tag:CustomDNSName 141 - dns-name 142 - private-ip-address 143 144# Example using constructed features to create groups and set ansible_host 145plugin: aws_ec2 146regions: 147 - us-east-1 148 - us-west-1 149# keyed_groups may be used to create custom groups 150strict: False 151keyed_groups: 152 # Add e.g. x86_64 hosts to an arch_x86_64 group 153 - prefix: arch 154 key: 'architecture' 155 # Add hosts to tag_Name_Value groups for each Name/Value tag pair 156 - prefix: tag 157 key: tags 158 # Add hosts to e.g. instance_type_z3_tiny 159 - prefix: instance_type 160 key: instance_type 161 # Create security_groups_sg_abcd1234 group for each SG 162 - key: 'security_groups|json_query("[].group_id")' 163 prefix: 'security_groups' 164 # Create a group for each value of the Application tag 165 - key: tags.Application 166 separator: '' 167 # Create a group per region e.g. aws_region_us_east_2 168 - key: placement.region 169 prefix: aws_region 170 # Create a group (or groups) based on the value of a custom tag "Role" and add them to a metagroup called "project" 171 - key: tags['Role'] 172 prefix: foo 173 parent_group: "project" 174# Set individual variables with compose 175compose: 176 # Use the private IP address to connect to the host 177 # (note: this does not modify inventory_hostname, which is set via I(hostnames)) 178 ansible_host: private_ip_address 179''' 180 181import re 182 183from ansible.errors import AnsibleError 184from ansible.module_utils._text import to_native, to_text 185from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict 186from ansible.plugins.inventory import BaseInventoryPlugin, Constructable, Cacheable 187from ansible.utils.display import Display 188from ansible.module_utils.six import string_types 189 190try: 191 import boto3 192 import botocore 193except ImportError: 194 raise AnsibleError('The ec2 dynamic inventory plugin requires boto3 and botocore.') 195 196display = Display() 197 198# The mappings give an array of keys to get from the filter name to the value 199# returned by boto3's EC2 describe_instances method. 200 201instance_meta_filter_to_boto_attr = { 202 'group-id': ('Groups', 'GroupId'), 203 'group-name': ('Groups', 'GroupName'), 204 'network-interface.attachment.instance-owner-id': ('OwnerId',), 205 'owner-id': ('OwnerId',), 206 'requester-id': ('RequesterId',), 207 'reservation-id': ('ReservationId',), 208} 209 210instance_data_filter_to_boto_attr = { 211 'affinity': ('Placement', 'Affinity'), 212 'architecture': ('Architecture',), 213 'availability-zone': ('Placement', 'AvailabilityZone'), 214 'block-device-mapping.attach-time': ('BlockDeviceMappings', 'Ebs', 'AttachTime'), 215 'block-device-mapping.delete-on-termination': ('BlockDeviceMappings', 'Ebs', 'DeleteOnTermination'), 216 'block-device-mapping.device-name': ('BlockDeviceMappings', 'DeviceName'), 217 'block-device-mapping.status': ('BlockDeviceMappings', 'Ebs', 'Status'), 218 'block-device-mapping.volume-id': ('BlockDeviceMappings', 'Ebs', 'VolumeId'), 219 'client-token': ('ClientToken',), 220 'dns-name': ('PublicDnsName',), 221 'host-id': ('Placement', 'HostId'), 222 'hypervisor': ('Hypervisor',), 223 'iam-instance-profile.arn': ('IamInstanceProfile', 'Arn'), 224 'image-id': ('ImageId',), 225 'instance-id': ('InstanceId',), 226 'instance-lifecycle': ('InstanceLifecycle',), 227 'instance-state-code': ('State', 'Code'), 228 'instance-state-name': ('State', 'Name'), 229 'instance-type': ('InstanceType',), 230 'instance.group-id': ('SecurityGroups', 'GroupId'), 231 'instance.group-name': ('SecurityGroups', 'GroupName'), 232 'ip-address': ('PublicIpAddress',), 233 'kernel-id': ('KernelId',), 234 'key-name': ('KeyName',), 235 'launch-index': ('AmiLaunchIndex',), 236 'launch-time': ('LaunchTime',), 237 'monitoring-state': ('Monitoring', 'State'), 238 'network-interface.addresses.private-ip-address': ('NetworkInterfaces', 'PrivateIpAddress'), 239 'network-interface.addresses.primary': ('NetworkInterfaces', 'PrivateIpAddresses', 'Primary'), 240 'network-interface.addresses.association.public-ip': ('NetworkInterfaces', 'PrivateIpAddresses', 'Association', 'PublicIp'), 241 'network-interface.addresses.association.ip-owner-id': ('NetworkInterfaces', 'PrivateIpAddresses', 'Association', 'IpOwnerId'), 242 'network-interface.association.public-ip': ('NetworkInterfaces', 'Association', 'PublicIp'), 243 'network-interface.association.ip-owner-id': ('NetworkInterfaces', 'Association', 'IpOwnerId'), 244 'network-interface.association.allocation-id': ('ElasticGpuAssociations', 'ElasticGpuId'), 245 'network-interface.association.association-id': ('ElasticGpuAssociations', 'ElasticGpuAssociationId'), 246 'network-interface.attachment.attachment-id': ('NetworkInterfaces', 'Attachment', 'AttachmentId'), 247 'network-interface.attachment.instance-id': ('InstanceId',), 248 'network-interface.attachment.device-index': ('NetworkInterfaces', 'Attachment', 'DeviceIndex'), 249 'network-interface.attachment.status': ('NetworkInterfaces', 'Attachment', 'Status'), 250 'network-interface.attachment.attach-time': ('NetworkInterfaces', 'Attachment', 'AttachTime'), 251 'network-interface.attachment.delete-on-termination': ('NetworkInterfaces', 'Attachment', 'DeleteOnTermination'), 252 'network-interface.availability-zone': ('Placement', 'AvailabilityZone'), 253 'network-interface.description': ('NetworkInterfaces', 'Description'), 254 'network-interface.group-id': ('NetworkInterfaces', 'Groups', 'GroupId'), 255 'network-interface.group-name': ('NetworkInterfaces', 'Groups', 'GroupName'), 256 'network-interface.ipv6-addresses.ipv6-address': ('NetworkInterfaces', 'Ipv6Addresses', 'Ipv6Address'), 257 'network-interface.mac-address': ('NetworkInterfaces', 'MacAddress'), 258 'network-interface.network-interface-id': ('NetworkInterfaces', 'NetworkInterfaceId'), 259 'network-interface.owner-id': ('NetworkInterfaces', 'OwnerId'), 260 'network-interface.private-dns-name': ('NetworkInterfaces', 'PrivateDnsName'), 261 # 'network-interface.requester-id': (), 262 'network-interface.requester-managed': ('NetworkInterfaces', 'Association', 'IpOwnerId'), 263 'network-interface.status': ('NetworkInterfaces', 'Status'), 264 'network-interface.source-dest-check': ('NetworkInterfaces', 'SourceDestCheck'), 265 'network-interface.subnet-id': ('NetworkInterfaces', 'SubnetId'), 266 'network-interface.vpc-id': ('NetworkInterfaces', 'VpcId'), 267 'placement-group-name': ('Placement', 'GroupName'), 268 'platform': ('Platform',), 269 'private-dns-name': ('PrivateDnsName',), 270 'private-ip-address': ('PrivateIpAddress',), 271 'product-code': ('ProductCodes', 'ProductCodeId'), 272 'product-code.type': ('ProductCodes', 'ProductCodeType'), 273 'ramdisk-id': ('RamdiskId',), 274 'reason': ('StateTransitionReason',), 275 'root-device-name': ('RootDeviceName',), 276 'root-device-type': ('RootDeviceType',), 277 'source-dest-check': ('SourceDestCheck',), 278 'spot-instance-request-id': ('SpotInstanceRequestId',), 279 'state-reason-code': ('StateReason', 'Code'), 280 'state-reason-message': ('StateReason', 'Message'), 281 'subnet-id': ('SubnetId',), 282 'tag': ('Tags',), 283 'tag-key': ('Tags',), 284 'tag-value': ('Tags',), 285 'tenancy': ('Placement', 'Tenancy'), 286 'virtualization-type': ('VirtualizationType',), 287 'vpc-id': ('VpcId',), 288} 289 290 291class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable): 292 293 NAME = 'aws_ec2' 294 295 def __init__(self): 296 super(InventoryModule, self).__init__() 297 298 self.group_prefix = 'aws_ec2_' 299 300 # credentials 301 self.boto_profile = None 302 self.aws_secret_access_key = None 303 self.aws_access_key_id = None 304 self.aws_security_token = None 305 self.iam_role_arn = None 306 307 def _compile_values(self, obj, attr): 308 ''' 309 :param obj: A list or dict of instance attributes 310 :param attr: A key 311 :return The value(s) found via the attr 312 ''' 313 if obj is None: 314 return 315 316 temp_obj = [] 317 318 if isinstance(obj, list) or isinstance(obj, tuple): 319 for each in obj: 320 value = self._compile_values(each, attr) 321 if value: 322 temp_obj.append(value) 323 else: 324 temp_obj = obj.get(attr) 325 326 has_indexes = any([isinstance(temp_obj, list), isinstance(temp_obj, tuple)]) 327 if has_indexes and len(temp_obj) == 1: 328 return temp_obj[0] 329 330 return temp_obj 331 332 def _get_boto_attr_chain(self, filter_name, instance): 333 ''' 334 :param filter_name: The filter 335 :param instance: instance dict returned by boto3 ec2 describe_instances() 336 ''' 337 allowed_filters = sorted(list(instance_data_filter_to_boto_attr.keys()) + list(instance_meta_filter_to_boto_attr.keys())) 338 if filter_name not in allowed_filters: 339 raise AnsibleError("Invalid filter '%s' provided; filter must be one of %s." % (filter_name, 340 allowed_filters)) 341 if filter_name in instance_data_filter_to_boto_attr: 342 boto_attr_list = instance_data_filter_to_boto_attr[filter_name] 343 else: 344 boto_attr_list = instance_meta_filter_to_boto_attr[filter_name] 345 346 instance_value = instance 347 for attribute in boto_attr_list: 348 instance_value = self._compile_values(instance_value, attribute) 349 return instance_value 350 351 def _get_credentials(self): 352 ''' 353 :return A dictionary of boto client credentials 354 ''' 355 boto_params = {} 356 for credential in (('aws_access_key_id', self.aws_access_key_id), 357 ('aws_secret_access_key', self.aws_secret_access_key), 358 ('aws_session_token', self.aws_security_token)): 359 if credential[1]: 360 boto_params[credential[0]] = credential[1] 361 362 return boto_params 363 364 def _get_connection(self, credentials, region='us-east-1'): 365 try: 366 connection = boto3.session.Session(profile_name=self.boto_profile).client('ec2', region, **credentials) 367 except (botocore.exceptions.ProfileNotFound, botocore.exceptions.PartialCredentialsError) as e: 368 if self.boto_profile: 369 try: 370 connection = boto3.session.Session(profile_name=self.boto_profile).client('ec2', region) 371 except (botocore.exceptions.ProfileNotFound, botocore.exceptions.PartialCredentialsError) as e: 372 raise AnsibleError("Insufficient credentials found: %s" % to_native(e)) 373 else: 374 raise AnsibleError("Insufficient credentials found: %s" % to_native(e)) 375 return connection 376 377 def _boto3_assume_role(self, credentials, region): 378 """ 379 Assume an IAM role passed by iam_role_arn parameter 380 381 :return: a dict containing the credentials of the assumed role 382 """ 383 384 iam_role_arn = self.iam_role_arn 385 386 try: 387 sts_connection = boto3.session.Session(profile_name=self.boto_profile).client('sts', region, **credentials) 388 sts_session = sts_connection.assume_role(RoleArn=iam_role_arn, RoleSessionName='ansible_aws_ec2_dynamic_inventory') 389 return dict( 390 aws_access_key_id=sts_session['Credentials']['AccessKeyId'], 391 aws_secret_access_key=sts_session['Credentials']['SecretAccessKey'], 392 aws_session_token=sts_session['Credentials']['SessionToken'] 393 ) 394 except botocore.exceptions.ClientError as e: 395 raise AnsibleError("Unable to assume IAM role: %s" % to_native(e)) 396 397 def _boto3_conn(self, regions): 398 ''' 399 :param regions: A list of regions to create a boto3 client 400 401 Generator that yields a boto3 client and the region 402 ''' 403 404 credentials = self._get_credentials() 405 iam_role_arn = self.iam_role_arn 406 407 if not regions: 408 try: 409 # as per https://boto3.amazonaws.com/v1/documentation/api/latest/guide/ec2-example-regions-avail-zones.html 410 client = self._get_connection(credentials) 411 resp = client.describe_regions() 412 regions = [x['RegionName'] for x in resp.get('Regions', [])] 413 except botocore.exceptions.NoRegionError: 414 # above seems to fail depending on boto3 version, ignore and lets try something else 415 pass 416 417 # fallback to local list hardcoded in boto3 if still no regions 418 if not regions: 419 session = boto3.Session() 420 regions = session.get_available_regions('ec2') 421 422 # I give up, now you MUST give me regions 423 if not regions: 424 raise AnsibleError('Unable to get regions list from available methods, you must specify the "regions" option to continue.') 425 426 for region in regions: 427 connection = self._get_connection(credentials, region) 428 try: 429 if iam_role_arn is not None: 430 assumed_credentials = self._boto3_assume_role(credentials, region) 431 else: 432 assumed_credentials = credentials 433 connection = boto3.session.Session(profile_name=self.boto_profile).client('ec2', region, **assumed_credentials) 434 except (botocore.exceptions.ProfileNotFound, botocore.exceptions.PartialCredentialsError) as e: 435 if self.boto_profile: 436 try: 437 connection = boto3.session.Session(profile_name=self.boto_profile).client('ec2', region) 438 except (botocore.exceptions.ProfileNotFound, botocore.exceptions.PartialCredentialsError) as e: 439 raise AnsibleError("Insufficient credentials found: %s" % to_native(e)) 440 else: 441 raise AnsibleError("Insufficient credentials found: %s" % to_native(e)) 442 yield connection, region 443 444 def _get_instances_by_region(self, regions, filters, strict_permissions): 445 ''' 446 :param regions: a list of regions in which to describe instances 447 :param filters: a list of boto3 filter dictionaries 448 :param strict_permissions: a boolean determining whether to fail or ignore 403 error codes 449 :return A list of instance dictionaries 450 ''' 451 all_instances = [] 452 453 for connection, region in self._boto3_conn(regions): 454 try: 455 # By default find non-terminated/terminating instances 456 if not any([f['Name'] == 'instance-state-name' for f in filters]): 457 filters.append({'Name': 'instance-state-name', 'Values': ['running', 'pending', 'stopping', 'stopped']}) 458 paginator = connection.get_paginator('describe_instances') 459 reservations = paginator.paginate(Filters=filters).build_full_result().get('Reservations') 460 instances = [] 461 for r in reservations: 462 new_instances = r['Instances'] 463 for instance in new_instances: 464 instance.update(self._get_reservation_details(r)) 465 if self.get_option('include_extra_api_calls'): 466 instance.update(self._get_event_set_and_persistence(connection, instance['InstanceId'], instance.get('SpotInstanceRequestId'))) 467 instances.extend(new_instances) 468 except botocore.exceptions.ClientError as e: 469 if e.response['ResponseMetadata']['HTTPStatusCode'] == 403 and not strict_permissions: 470 instances = [] 471 else: 472 raise AnsibleError("Failed to describe instances: %s" % to_native(e)) 473 except botocore.exceptions.BotoCoreError as e: 474 raise AnsibleError("Failed to describe instances: %s" % to_native(e)) 475 476 all_instances.extend(instances) 477 478 return sorted(all_instances, key=lambda x: x['InstanceId']) 479 480 def _get_reservation_details(self, reservation): 481 return { 482 'OwnerId': reservation['OwnerId'], 483 'RequesterId': reservation.get('RequesterId', ''), 484 'ReservationId': reservation['ReservationId'] 485 } 486 487 def _get_event_set_and_persistence(self, connection, instance_id, spot_instance): 488 host_vars = {'Events': '', 'Persistent': False} 489 try: 490 kwargs = {'InstanceIds': [instance_id]} 491 host_vars['Events'] = connection.describe_instance_status(**kwargs)['InstanceStatuses'][0].get('Events', '') 492 except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: 493 if not self.get_option('strict_permissions'): 494 pass 495 else: 496 raise AnsibleError("Failed to describe instance status: %s" % to_native(e)) 497 if spot_instance: 498 try: 499 kwargs = {'SpotInstanceRequestIds': [spot_instance]} 500 host_vars['Persistent'] = bool( 501 connection.describe_spot_instance_requests(**kwargs)['SpotInstanceRequests'][0].get('Type') == 'persistent' 502 ) 503 except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: 504 if not self.get_option('strict_permissions'): 505 pass 506 else: 507 raise AnsibleError("Failed to describe spot instance requests: %s" % to_native(e)) 508 return host_vars 509 510 def _get_tag_hostname(self, preference, instance): 511 tag_hostnames = preference.split('tag:', 1)[1] 512 if ',' in tag_hostnames: 513 tag_hostnames = tag_hostnames.split(',') 514 else: 515 tag_hostnames = [tag_hostnames] 516 tags = boto3_tag_list_to_ansible_dict(instance.get('Tags', [])) 517 for v in tag_hostnames: 518 if '=' in v: 519 tag_name, tag_value = v.split('=') 520 if tags.get(tag_name) == tag_value: 521 return to_text(tag_name) + "_" + to_text(tag_value) 522 else: 523 tag_value = tags.get(v) 524 if tag_value: 525 return to_text(tag_value) 526 return None 527 528 def _get_hostname(self, instance, hostnames): 529 ''' 530 :param instance: an instance dict returned by boto3 ec2 describe_instances() 531 :param hostnames: a list of hostname destination variables in order of preference 532 :return the preferred identifer for the host 533 ''' 534 if not hostnames: 535 hostnames = ['dns-name', 'private-dns-name'] 536 537 hostname = None 538 for preference in hostnames: 539 if 'tag' in preference: 540 if not preference.startswith('tag:'): 541 raise AnsibleError("To name a host by tags name_value, use 'tag:name=value'.") 542 hostname = self._get_tag_hostname(preference, instance) 543 else: 544 hostname = self._get_boto_attr_chain(preference, instance) 545 if hostname: 546 break 547 if hostname: 548 if ':' in to_text(hostname): 549 return self._sanitize_group_name((to_text(hostname))) 550 else: 551 return to_text(hostname) 552 553 def _query(self, regions, filters, strict_permissions): 554 ''' 555 :param regions: a list of regions to query 556 :param filters: a list of boto3 filter dictionaries 557 :param hostnames: a list of hostname destination variables in order of preference 558 :param strict_permissions: a boolean determining whether to fail or ignore 403 error codes 559 ''' 560 return {'aws_ec2': self._get_instances_by_region(regions, filters, strict_permissions)} 561 562 def _populate(self, groups, hostnames): 563 for group in groups: 564 group = self.inventory.add_group(group) 565 self._add_hosts(hosts=groups[group], group=group, hostnames=hostnames) 566 self.inventory.add_child('all', group) 567 568 def _add_hosts(self, hosts, group, hostnames): 569 ''' 570 :param hosts: a list of hosts to be added to a group 571 :param group: the name of the group to which the hosts belong 572 :param hostnames: a list of hostname destination variables in order of preference 573 ''' 574 for host in hosts: 575 hostname = self._get_hostname(host, hostnames) 576 577 host = camel_dict_to_snake_dict(host, ignore_list=['Tags']) 578 host['tags'] = boto3_tag_list_to_ansible_dict(host.get('tags', [])) 579 580 # Allow easier grouping by region 581 host['placement']['region'] = host['placement']['availability_zone'][:-1] 582 583 if not hostname: 584 continue 585 self.inventory.add_host(hostname, group=group) 586 for hostvar, hostval in host.items(): 587 self.inventory.set_variable(hostname, hostvar, hostval) 588 589 # Use constructed if applicable 590 591 strict = self.get_option('strict') 592 593 # Composed variables 594 self._set_composite_vars(self.get_option('compose'), host, hostname, strict=strict) 595 596 # Complex groups based on jinja2 conditionals, hosts that meet the conditional are added to group 597 self._add_host_to_composed_groups(self.get_option('groups'), host, hostname, strict=strict) 598 599 # Create groups based on variable values and add the corresponding hosts to it 600 self._add_host_to_keyed_groups(self.get_option('keyed_groups'), host, hostname, strict=strict) 601 602 def _set_credentials(self): 603 ''' 604 :param config_data: contents of the inventory config file 605 ''' 606 607 self.boto_profile = self.get_option('aws_profile') 608 self.aws_access_key_id = self.get_option('aws_access_key') 609 self.aws_secret_access_key = self.get_option('aws_secret_key') 610 self.aws_security_token = self.get_option('aws_security_token') 611 self.iam_role_arn = self.get_option('iam_role_arn') 612 613 if not self.boto_profile and not (self.aws_access_key_id and self.aws_secret_access_key): 614 session = botocore.session.get_session() 615 try: 616 credentials = session.get_credentials().get_frozen_credentials() 617 except AttributeError: 618 pass 619 else: 620 self.aws_access_key_id = credentials.access_key 621 self.aws_secret_access_key = credentials.secret_key 622 self.aws_security_token = credentials.token 623 624 if not self.boto_profile and not (self.aws_access_key_id and self.aws_secret_access_key): 625 raise AnsibleError("Insufficient boto credentials found. Please provide them in your " 626 "inventory configuration file or set them as environment variables.") 627 628 def verify_file(self, path): 629 ''' 630 :param loader: an ansible.parsing.dataloader.DataLoader object 631 :param path: the path to the inventory config file 632 :return the contents of the config file 633 ''' 634 if super(InventoryModule, self).verify_file(path): 635 if path.endswith(('aws_ec2.yml', 'aws_ec2.yaml')): 636 return True 637 display.debug("aws_ec2 inventory filename must end with 'aws_ec2.yml' or 'aws_ec2.yaml'") 638 return False 639 640 def parse(self, inventory, loader, path, cache=True): 641 642 super(InventoryModule, self).parse(inventory, loader, path) 643 644 self._read_config_data(path) 645 646 if self.get_option('use_contrib_script_compatible_sanitization'): 647 self._sanitize_group_name = self._legacy_script_compatible_group_sanitization 648 649 self._set_credentials() 650 651 # get user specifications 652 regions = self.get_option('regions') 653 filters = ansible_dict_to_boto3_filter_list(self.get_option('filters')) 654 hostnames = self.get_option('hostnames') 655 strict_permissions = self.get_option('strict_permissions') 656 657 cache_key = self.get_cache_key(path) 658 # false when refresh_cache or --flush-cache is used 659 if cache: 660 # get the user-specified directive 661 cache = self.get_option('cache') 662 663 # Generate inventory 664 cache_needs_update = False 665 if cache: 666 try: 667 results = self._cache[cache_key] 668 except KeyError: 669 # if cache expires or cache file doesn't exist 670 cache_needs_update = True 671 672 if not cache or cache_needs_update: 673 results = self._query(regions, filters, strict_permissions) 674 675 self._populate(results, hostnames) 676 677 # If the cache has expired/doesn't exist or if refresh_inventory/flush cache is used 678 # when the user is using caching, update the cached inventory 679 if cache_needs_update or (not cache and self.get_option('cache')): 680 self._cache[cache_key] = results 681 682 @staticmethod 683 def _legacy_script_compatible_group_sanitization(name): 684 685 # note that while this mirrors what the script used to do, it has many issues with unicode and usability in python 686 regex = re.compile(r"[^A-Za-z0-9\_\-]") 687 688 return regex.sub('_', name) 689 690 691def ansible_dict_to_boto3_filter_list(filters_dict): 692 693 """ Convert an Ansible dict of filters to list of dicts that boto3 can use 694 Args: 695 filters_dict (dict): Dict of AWS filters. 696 Basic Usage: 697 >>> filters = {'some-aws-id': 'i-01234567'} 698 >>> ansible_dict_to_boto3_filter_list(filters) 699 { 700 'some-aws-id': 'i-01234567' 701 } 702 Returns: 703 List: List of AWS filters and their values 704 [ 705 { 706 'Name': 'some-aws-id', 707 'Values': [ 708 'i-01234567', 709 ] 710 } 711 ] 712 """ 713 714 filters_list = [] 715 for k, v in filters_dict.items(): 716 filter_dict = {'Name': k} 717 if isinstance(v, string_types): 718 filter_dict['Values'] = [v] 719 else: 720 filter_dict['Values'] = v 721 722 filters_list.append(filter_dict) 723 724 return filters_list 725 726 727def boto3_tag_list_to_ansible_dict(tags_list, tag_name_key_name=None, tag_value_key_name=None): 728 729 """ Convert a boto3 list of resource tags to a flat dict of key:value pairs 730 Args: 731 tags_list (list): List of dicts representing AWS tags. 732 tag_name_key_name (str): Value to use as the key for all tag keys (useful because boto3 doesn't always use "Key") 733 tag_value_key_name (str): Value to use as the key for all tag values (useful because boto3 doesn't always use "Value") 734 Basic Usage: 735 >>> tags_list = [{'Key': 'MyTagKey', 'Value': 'MyTagValue'}] 736 >>> boto3_tag_list_to_ansible_dict(tags_list) 737 [ 738 { 739 'Key': 'MyTagKey', 740 'Value': 'MyTagValue' 741 } 742 ] 743 Returns: 744 Dict: Dict of key:value pairs representing AWS tags 745 { 746 'MyTagKey': 'MyTagValue', 747 } 748 """ 749 750 if tag_name_key_name and tag_value_key_name: 751 tag_candidates = {tag_name_key_name: tag_value_key_name} 752 else: 753 tag_candidates = {'key': 'value', 'Key': 'Value'} 754 755 if not tags_list: 756 return {} 757 for k, v in tag_candidates.items(): 758 if k in tags_list[0] and v in tags_list[0]: 759 return dict((tag[k], tag[v]) for tag in tags_list) 760 raise ValueError("Couldn't find tag key (candidates %s) in tag list %s" % (str(tag_candidates), str(tags_list))) 761