1# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import ipaddress 16import socket 17 18import munch 19 20from openstack import _log 21from openstack.cloud import exc 22from openstack import utils 23 24 25NON_CALLABLES = (str, bool, dict, int, float, list, type(None)) 26 27 28def find_nova_interfaces(addresses, ext_tag=None, key_name=None, version=4, 29 mac_addr=None): 30 ret = [] 31 for (k, v) in iter(addresses.items()): 32 if key_name is not None and k != key_name: 33 # key_name is specified and it doesn't match the current network. 34 # Continue with the next one 35 continue 36 37 for interface_spec in v: 38 if ext_tag is not None: 39 if 'OS-EXT-IPS:type' not in interface_spec: 40 # ext_tag is specified, but this interface has no tag 41 # We could actually return right away as this means that 42 # this cloud doesn't support OS-EXT-IPS. Nevertheless, 43 # it would be better to perform an explicit check. e.g.: 44 # cloud._has_nova_extension('OS-EXT-IPS') 45 # But this needs cloud to be passed to this function. 46 continue 47 elif interface_spec['OS-EXT-IPS:type'] != ext_tag: 48 # Type doesn't match, continue with next one 49 continue 50 51 if mac_addr is not None: 52 if 'OS-EXT-IPS-MAC:mac_addr' not in interface_spec: 53 # mac_addr is specified, but this interface has no mac_addr 54 # We could actually return right away as this means that 55 # this cloud doesn't support OS-EXT-IPS-MAC. Nevertheless, 56 # it would be better to perform an explicit check. e.g.: 57 # cloud._has_nova_extension('OS-EXT-IPS-MAC') 58 # But this needs cloud to be passed to this function. 59 continue 60 elif interface_spec['OS-EXT-IPS-MAC:mac_addr'] != mac_addr: 61 # MAC doesn't match, continue with next one 62 continue 63 64 if interface_spec['version'] == version: 65 ret.append(interface_spec) 66 return ret 67 68 69def find_nova_addresses(addresses, ext_tag=None, key_name=None, version=4, 70 mac_addr=None): 71 interfaces = find_nova_interfaces(addresses, ext_tag, key_name, version, 72 mac_addr) 73 floating_addrs = [] 74 fixed_addrs = [] 75 for i in interfaces: 76 if i.get('OS-EXT-IPS:type') == 'floating': 77 floating_addrs.append(i['addr']) 78 else: 79 fixed_addrs.append(i['addr']) 80 return floating_addrs + fixed_addrs 81 82 83def get_server_ip(server, public=False, cloud_public=True, **kwargs): 84 """Get an IP from the Nova addresses dict 85 86 :param server: The server to pull the address from 87 :param public: Whether the address we're looking for should be considered 88 'public' and therefore reachabiliity tests should be 89 used. (defaults to False) 90 :param cloud_public: Whether the cloud has been configured to use private 91 IPs from servers as the interface_ip. This inverts the 92 public reachability logic, as in this case it's the 93 private ip we expect shade to be able to reach 94 """ 95 addrs = find_nova_addresses(server['addresses'], **kwargs) 96 return find_best_address( 97 addrs, public=public, cloud_public=cloud_public) 98 99 100def get_server_private_ip(server, cloud=None): 101 """Find the private IP address 102 103 If Neutron is available, search for a port on a network where 104 `router:external` is False and `shared` is False. This combination 105 indicates a private network with private IP addresses. This port should 106 have the private IP. 107 108 If Neutron is not available, or something goes wrong communicating with it, 109 as a fallback, try the list of addresses associated with the server dict, 110 looking for an IP type tagged as 'fixed' in the network named 'private'. 111 112 Last resort, ignore the IP type and just look for an IP on the 'private' 113 network (e.g., Rackspace). 114 """ 115 if cloud and not cloud.use_internal_network(): 116 return None 117 118 # Try to get a floating IP interface. If we have one then return the 119 # private IP address associated with that floating IP for consistency. 120 fip_ints = find_nova_interfaces(server['addresses'], ext_tag='floating') 121 fip_mac = None 122 if fip_ints: 123 fip_mac = fip_ints[0].get('OS-EXT-IPS-MAC:mac_addr') 124 125 # Short circuit the ports/networks search below with a heavily cached 126 # and possibly pre-configured network name 127 if cloud: 128 int_nets = cloud.get_internal_ipv4_networks() 129 for int_net in int_nets: 130 int_ip = get_server_ip( 131 server, key_name=int_net['name'], 132 ext_tag='fixed', 133 cloud_public=not cloud.private, 134 mac_addr=fip_mac) 135 if int_ip is not None: 136 return int_ip 137 # Try a second time without the fixed tag. This is for old nova-network 138 # results that do not have the fixed/floating tag. 139 for int_net in int_nets: 140 int_ip = get_server_ip( 141 server, key_name=int_net['name'], 142 cloud_public=not cloud.private, 143 mac_addr=fip_mac) 144 if int_ip is not None: 145 return int_ip 146 147 ip = get_server_ip( 148 server, ext_tag='fixed', key_name='private', mac_addr=fip_mac) 149 if ip: 150 return ip 151 152 # Last resort, and Rackspace 153 return get_server_ip( 154 server, key_name='private') 155 156 157def get_server_external_ipv4(cloud, server): 158 """Find an externally routable IP for the server. 159 160 There are 5 different scenarios we have to account for: 161 162 * Cloud has externally routable IP from neutron but neutron APIs don't 163 work (only info available is in nova server record) (rackspace) 164 * Cloud has externally routable IP from neutron (runabove, ovh) 165 * Cloud has externally routable IP from neutron AND supports optional 166 private tenant networks (vexxhost, unitedstack) 167 * Cloud only has private tenant network provided by neutron and requires 168 floating-ip for external routing (dreamhost, hp) 169 * Cloud only has private tenant network provided by nova-network and 170 requires floating-ip for external routing (auro) 171 172 :param cloud: the cloud we're working with 173 :param server: the server dict from which we want to get an IPv4 address 174 :return: a string containing the IPv4 address or None 175 """ 176 177 if not cloud.use_external_network(): 178 return None 179 180 if server['accessIPv4']: 181 return server['accessIPv4'] 182 183 # Short circuit the ports/networks search below with a heavily cached 184 # and possibly pre-configured network name 185 ext_nets = cloud.get_external_ipv4_networks() 186 for ext_net in ext_nets: 187 ext_ip = get_server_ip( 188 server, key_name=ext_net['name'], public=True, 189 cloud_public=not cloud.private) 190 if ext_ip is not None: 191 return ext_ip 192 193 # Try to get a floating IP address 194 # Much as I might find floating IPs annoying, if it has one, that's 195 # almost certainly the one that wants to be used 196 ext_ip = get_server_ip( 197 server, ext_tag='floating', public=True, 198 cloud_public=not cloud.private) 199 if ext_ip is not None: 200 return ext_ip 201 202 # The cloud doesn't support Neutron or Neutron can't be contacted. The 203 # server might have fixed addresses that are reachable from outside the 204 # cloud (e.g. Rax) or have plain ol' floating IPs 205 206 # Try to get an address from a network named 'public' 207 ext_ip = get_server_ip( 208 server, key_name='public', public=True, 209 cloud_public=not cloud.private) 210 if ext_ip is not None: 211 return ext_ip 212 213 # Nothing else works, try to find a globally routable IP address 214 for interfaces in server['addresses'].values(): 215 for interface in interfaces: 216 try: 217 ip = ipaddress.ip_address(interface['addr']) 218 except Exception: 219 # Skip any error, we're looking for a working ip - if the 220 # cloud returns garbage, it wouldn't be the first weird thing 221 # but it still doesn't meet the requirement of "be a working 222 # ip address" 223 continue 224 if ip.version == 4 and not ip.is_private: 225 return str(ip) 226 227 return None 228 229 230def find_best_address(addresses, public=False, cloud_public=True): 231 do_check = public == cloud_public 232 if not addresses: 233 return None 234 if len(addresses) == 1: 235 return addresses[0] 236 if len(addresses) > 1 and do_check: 237 # We only want to do this check if the address is supposed to be 238 # reachable. Otherwise we're just debug log spamming on every listing 239 # of private ip addresses 240 for address in addresses: 241 try: 242 for count in utils.iterate_timeout( 243 5, "Timeout waiting for %s" % address, wait=0.1): 244 # Return the first one that is reachable 245 try: 246 for res in socket.getaddrinfo( 247 address, 22, socket.AF_UNSPEC, 248 socket.SOCK_STREAM, 0): 249 family, socktype, proto, _, sa = res 250 connect_socket = socket.socket( 251 family, socktype, proto) 252 connect_socket.settimeout(1) 253 connect_socket.connect(sa) 254 return address 255 except socket.error: 256 # Sometimes a "no route to address" type error 257 # will fail fast, but can often come alive 258 # when retried. 259 continue 260 except Exception: 261 pass 262 263 # Give up and return the first - none work as far as we can tell 264 if do_check: 265 log = _log.setup_logging('openstack') 266 log.debug( 267 "The cloud returned multiple addresses %s:, and we could not " 268 "connect to port 22 on either. That might be what you wanted, " 269 "but we have no clue what's going on, so we picked the first one " 270 "%s" % (addresses, addresses[0])) 271 return addresses[0] 272 273 274def get_server_external_ipv6(server): 275 """ Get an IPv6 address reachable from outside the cloud. 276 277 This function assumes that if a server has an IPv6 address, that address 278 is reachable from outside the cloud. 279 280 :param server: the server from which we want to get an IPv6 address 281 :return: a string containing the IPv6 address or None 282 """ 283 # Don't return ipv6 interfaces if forcing IPv4 284 if server['accessIPv6']: 285 return server['accessIPv6'] 286 addresses = find_nova_addresses(addresses=server['addresses'], version=6) 287 return find_best_address(addresses, public=True) 288 289 290def get_server_default_ip(cloud, server): 291 """ Get the configured 'default' address 292 293 It is possible in clouds.yaml to configure for a cloud a network that 294 is the 'default_interface'. This is the network that should be used 295 to talk to instances on the network. 296 297 :param cloud: the cloud we're working with 298 :param server: the server dict from which we want to get the default 299 IPv4 address 300 :return: a string containing the IPv4 address or None 301 """ 302 ext_net = cloud.get_default_network() 303 if ext_net: 304 if (cloud._local_ipv6 and not cloud.force_ipv4): 305 # try 6 first, fall back to four 306 versions = [6, 4] 307 else: 308 versions = [4] 309 for version in versions: 310 ext_ip = get_server_ip( 311 server, key_name=ext_net['name'], version=version, public=True, 312 cloud_public=not cloud.private) 313 if ext_ip is not None: 314 return ext_ip 315 return None 316 317 318def _get_interface_ip(cloud, server): 319 """ Get the interface IP for the server 320 321 Interface IP is the IP that should be used for communicating with the 322 server. It is: 323 - the IP on the configured default_interface network 324 - if cloud.private, the private ip if it exists 325 - if the server has a public ip, the public ip 326 """ 327 default_ip = get_server_default_ip(cloud, server) 328 if default_ip: 329 return default_ip 330 331 if cloud.private and server['private_v4']: 332 return server['private_v4'] 333 334 if (server['public_v6'] and cloud._local_ipv6 and not cloud.force_ipv4): 335 return server['public_v6'] 336 else: 337 return server['public_v4'] 338 339 340def get_groups_from_server(cloud, server, server_vars): 341 groups = [] 342 343 # NOTE(efried): This is hardcoded to 'compute' because this method is only 344 # used from ComputeCloudMixin. 345 region = cloud.config.get_region_name('compute') 346 cloud_name = cloud.name 347 348 # Create a group for the cloud 349 groups.append(cloud_name) 350 351 # Create a group on region 352 groups.append(region) 353 354 # And one by cloud_region 355 groups.append("%s_%s" % (cloud_name, region)) 356 357 # Check if group metadata key in servers' metadata 358 group = server['metadata'].get('group') 359 if group: 360 groups.append(group) 361 362 for extra_group in server['metadata'].get('groups', '').split(','): 363 if extra_group: 364 groups.append(extra_group) 365 366 groups.append('instance-%s' % server['id']) 367 368 for key in ('flavor', 'image'): 369 if 'name' in server_vars[key]: 370 groups.append('%s-%s' % (key, server_vars[key]['name'])) 371 372 for key, value in iter(server['metadata'].items()): 373 groups.append('meta-%s_%s' % (key, value)) 374 375 az = server_vars.get('az', None) 376 if az: 377 # Make groups for az, region_az and cloud_region_az 378 groups.append(az) 379 groups.append('%s_%s' % (region, az)) 380 groups.append('%s_%s_%s' % (cloud.name, region, az)) 381 return groups 382 383 384def expand_server_vars(cloud, server): 385 """Backwards compatibility function.""" 386 return add_server_interfaces(cloud, server) 387 388 389def _make_address_dict(fip, port): 390 address = dict(version=4, addr=fip['floating_ip_address']) 391 address['OS-EXT-IPS:type'] = 'floating' 392 address['OS-EXT-IPS-MAC:mac_addr'] = port['mac_address'] 393 return address 394 395 396def _get_supplemental_addresses(cloud, server): 397 fixed_ip_mapping = {} 398 for name, network in server['addresses'].items(): 399 for address in network: 400 if address['version'] == 6: 401 continue 402 if address.get('OS-EXT-IPS:type') == 'floating': 403 # We have a floating IP that nova knows about, do nothing 404 return server['addresses'] 405 fixed_ip_mapping[address['addr']] = name 406 try: 407 # Don't bother doing this before the server is active, it's a waste 408 # of an API call while polling for a server to come up 409 if (cloud.has_service('network') 410 and cloud._has_floating_ips() 411 and server['status'] == 'ACTIVE'): 412 for port in cloud.search_ports( 413 filters=dict(device_id=server['id'])): 414 # This SHOULD return one and only one FIP - but doing it as a 415 # search/list lets the logic work regardless 416 for fip in cloud.search_floating_ips( 417 filters=dict(port_id=port['id'])): 418 fixed_net = fixed_ip_mapping.get(fip['fixed_ip_address']) 419 if fixed_net is None: 420 log = _log.setup_logging('openstack') 421 log.debug( 422 "The cloud returned floating ip %(fip)s attached" 423 " to server %(server)s but the fixed ip associated" 424 " with the floating ip in the neutron listing" 425 " does not exist in the nova listing. Something" 426 " is exceptionally broken.", 427 dict(fip=fip['id'], server=server['id'])) 428 else: 429 server['addresses'][fixed_net].append( 430 _make_address_dict(fip, port)) 431 except exc.OpenStackCloudException: 432 # If something goes wrong with a cloud call, that's cool - this is 433 # an attempt to provide additional data and should not block forward 434 # progress 435 pass 436 return server['addresses'] 437 438 439def add_server_interfaces(cloud, server): 440 """Add network interface information to server. 441 442 Query the cloud as necessary to add information to the server record 443 about the network information needed to interface with the server. 444 445 Ensures that public_v4, public_v6, private_v4, private_v6, interface_ip, 446 accessIPv4 and accessIPv6 are always set. 447 """ 448 # First, add an IP address. Set it to '' rather than None if it does 449 # not exist to remain consistent with the pre-existing missing values 450 server['addresses'] = _get_supplemental_addresses(cloud, server) 451 server['public_v4'] = get_server_external_ipv4(cloud, server) or '' 452 # If we're forcing IPv4, then don't report IPv6 interfaces which 453 # are likely to be unconfigured. 454 if cloud.force_ipv4: 455 server['public_v6'] = '' 456 else: 457 server['public_v6'] = get_server_external_ipv6(server) or '' 458 server['private_v4'] = get_server_private_ip(server, cloud) or '' 459 server['interface_ip'] = _get_interface_ip(cloud, server) or '' 460 461 # Some clouds do not set these, but they're a regular part of the Nova 462 # server record. Since we know them, go ahead and set them. In the case 463 # where they were set previous, we use the values, so this will not break 464 # clouds that provide the information 465 if cloud.private and server['private_v4']: 466 server['accessIPv4'] = server['private_v4'] 467 else: 468 server['accessIPv4'] = server['public_v4'] 469 server['accessIPv6'] = server['public_v6'] 470 471 return server 472 473 474def expand_server_security_groups(cloud, server): 475 try: 476 groups = cloud.list_server_security_groups(server) 477 except exc.OpenStackCloudException: 478 groups = [] 479 server['security_groups'] = groups or [] 480 481 482def get_hostvars_from_server(cloud, server, mounts=None): 483 """Expand additional server information useful for ansible inventory. 484 485 Variables in this function may make additional cloud queries to flesh out 486 possibly interesting info, making it more expensive to call than 487 expand_server_vars if caching is not set up. If caching is set up, 488 the extra cost should be minimal. 489 """ 490 server_vars = add_server_interfaces(cloud, server) 491 492 flavor_id = server['flavor'].get('id') 493 if flavor_id: 494 # In newer nova, the flavor record can be kept around for flavors 495 # that no longer exist. The id and name are not there. 496 flavor_name = cloud.get_flavor_name(flavor_id) 497 if flavor_name: 498 server_vars['flavor']['name'] = flavor_name 499 elif 'original_name' in server['flavor']: 500 # Users might be have code still expecting name. That name is in 501 # original_name. 502 server_vars['flavor']['name'] = server['flavor']['original_name'] 503 504 expand_server_security_groups(cloud, server) 505 506 # OpenStack can return image as a string when you've booted from volume 507 if str(server['image']) == server['image']: 508 image_id = server['image'] 509 server_vars['image'] = dict(id=image_id) 510 else: 511 image_id = server['image'].get('id', None) 512 if image_id: 513 image_name = cloud.get_image_name(image_id) 514 if image_name: 515 server_vars['image']['name'] = image_name 516 517 volumes = [] 518 if cloud.has_service('volume'): 519 try: 520 for volume in cloud.get_volumes(server): 521 # Make things easier to consume elsewhere 522 volume['device'] = volume['attachments'][0]['device'] 523 volumes.append(volume) 524 except exc.OpenStackCloudException: 525 pass 526 server_vars['volumes'] = volumes 527 if mounts: 528 for mount in mounts: 529 for vol in server_vars['volumes']: 530 if vol['display_name'] == mount['display_name']: 531 if 'mount' in mount: 532 vol['mount'] = mount['mount'] 533 534 return server_vars 535 536 537def obj_to_munch(obj): 538 """ Turn an object with attributes into a dict suitable for serializing. 539 540 Some of the things that are returned in OpenStack are objects with 541 attributes. That's awesome - except when you want to expose them as JSON 542 structures. We use this as the basis of get_hostvars_from_server above so 543 that we can just have a plain dict of all of the values that exist in the 544 nova metadata for a server. 545 """ 546 if obj is None: 547 return None 548 elif isinstance(obj, munch.Munch) or hasattr(obj, 'mock_add_spec'): 549 # If we obj_to_munch twice, don't fail, just return the munch 550 # Also, don't try to modify Mock objects - that way lies madness 551 return obj 552 elif isinstance(obj, dict): 553 # The new request-id tracking spec: 554 # https://specs.openstack.org/openstack/nova-specs/specs/juno/approved/log-request-id-mappings.html 555 # adds a request-ids attribute to returned objects. It does this even 556 # with dicts, which now become dict subclasses. So we want to convert 557 # the dict we get, but we also want it to fall through to object 558 # attribute processing so that we can also get the request_ids 559 # data into our resulting object. 560 instance = munch.Munch(obj) 561 else: 562 instance = munch.Munch() 563 564 for key in dir(obj): 565 try: 566 value = getattr(obj, key) 567 # some attributes can be defined as a @propierty, so we can't assure 568 # to have a valid value 569 # e.g. id in python-novaclient/tree/novaclient/v2/quotas.py 570 except AttributeError: 571 continue 572 if isinstance(value, NON_CALLABLES) and not key.startswith('_'): 573 instance[key] = value 574 return instance 575 576 577obj_to_dict = obj_to_munch 578 579 580def obj_list_to_munch(obj_list): 581 """Enumerate through lists of objects and return lists of dictonaries. 582 583 Some of the objects returned in OpenStack are actually lists of objects, 584 and in order to expose the data structures as JSON, we need to facilitate 585 the conversion to lists of dictonaries. 586 """ 587 return [obj_to_munch(obj) for obj in obj_list] 588 589 590obj_list_to_dict = obj_list_to_munch 591 592 593def get_and_munchify(key, data): 594 """Get the value associated to key and convert it. 595 596 The value will be converted in a Munch object or a list of Munch objects 597 based on the type 598 """ 599 result = data.get(key, []) if key else data 600 if isinstance(result, list): 601 return obj_list_to_munch(result) 602 elif isinstance(result, dict): 603 return obj_to_munch(result) 604 return result 605