1# Copyright (c) 2018 Remy Leone 2# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) 3 4from __future__ import absolute_import, division, print_function 5 6__metaclass__ = type 7 8DOCUMENTATION = """ 9 name: nb_inventory 10 plugin_type: inventory 11 author: 12 - Remy Leone (@sieben) 13 - Anthony Ruhier (@Anthony25) 14 - Nikhil Singh Baliyan (@nikkytub) 15 - Sander Steffann (@steffann) 16 - Douglas Heriot (@DouglasHeriot) 17 short_description: NetBox inventory source 18 description: 19 - Get inventory hosts from NetBox 20 extends_documentation_fragment: 21 - constructed 22 - inventory_cache 23 options: 24 plugin: 25 description: token that ensures this is a source file for the 'netbox' plugin. 26 required: True 27 choices: ['netbox.netbox.nb_inventory'] 28 api_endpoint: 29 description: Endpoint of the NetBox API 30 required: True 31 env: 32 - name: NETBOX_API 33 validate_certs: 34 description: 35 - Allows connection when SSL certificates are not valid. Set to C(false) when certificates are not trusted. 36 default: True 37 type: boolean 38 cert: 39 description: 40 - Certificate path 41 default: False 42 key: 43 description: 44 - Certificate key path 45 default: False 46 ca_path: 47 description: 48 - CA path 49 default: False 50 follow_redirects: 51 description: 52 - Determine how redirects are followed. 53 - By default, I(follow_redirects) is set to uses urllib2 default behavior. 54 default: urllib2 55 choices: ['urllib2', 'all', 'yes', 'safe', 'none'] 56 config_context: 57 description: 58 - If True, it adds config_context in host vars. 59 - Config-context enables the association of arbitrary data to devices and virtual machines grouped by 60 region, site, role, platform, and/or tenant. Please check official netbox docs for more info. 61 default: False 62 type: boolean 63 flatten_config_context: 64 description: 65 - If I(config_context) is enabled, by default it's added as a host var named config_context. 66 - If flatten_config_context is set to True, the config context variables will be added directly to the host instead. 67 default: False 68 type: boolean 69 version_added: "0.2.1" 70 flatten_local_context_data: 71 description: 72 - If I(local_context_data) is enabled, by default it's added as a host var named local_context_data. 73 - If flatten_local_context_data is set to True, the config context variables will be added directly to the host instead. 74 default: False 75 type: boolean 76 version_added: "0.3.0" 77 flatten_custom_fields: 78 description: 79 - By default, host custom fields are added as a dictionary host var named custom_fields. 80 - If flatten_custom_fields is set to True, the fields will be added directly to the host instead. 81 default: False 82 type: boolean 83 version_added: "0.2.1" 84 token: 85 required: False 86 description: 87 - NetBox API token to be able to read against NetBox. 88 - This may not be required depending on the NetBox setup. 89 env: 90 # in order of precedence 91 - name: NETBOX_TOKEN 92 - name: NETBOX_API_KEY 93 plurals: 94 description: 95 - If True, all host vars are contained inside single-element arrays for legacy compatibility with old versions of this plugin. 96 - Group names will be plural (ie. "sites_mysite" instead of "site_mysite") 97 - The choices of I(group_by) will be changed by this option. 98 default: True 99 type: boolean 100 version_added: "0.2.1" 101 interfaces: 102 description: 103 - If True, it adds the device or virtual machine interface information in host vars. 104 default: False 105 type: boolean 106 version_added: "0.1.7" 107 services: 108 description: 109 - If True, it adds the device or virtual machine services information in host vars. 110 default: True 111 type: boolean 112 version_added: "0.2.0" 113 fetch_all: 114 description: 115 - By default, fetching interfaces and services will get all of the contents of NetBox regardless of query_filters applied to devices and VMs. 116 - When set to False, separate requests will be made fetching interfaces, services, and IP addresses for each device_id and virtual_machine_id. 117 - If you are using the various query_filters options to reduce the number of devices, you may find querying Netbox faster with fetch_all set to False. 118 - For efficiency, when False, these requests will be batched, for example /api/dcim/interfaces?limit=0&device_id=1&device_id=2&device_id=3 119 - These GET request URIs can become quite large for a large number of devices. If you run into HTTP 414 errors, you can adjust the max_uri_length option to suit your web server. 120 default: True 121 type: boolean 122 version_added: "0.2.1" 123 group_by: 124 description: 125 - Keys used to create groups. The I(plurals) option controls which of these are valid. 126 - I(rack_group) is supported on NetBox versions 2.10 or lower only 127 - I(location) is supported on NetBox versions 2.11 or higher only 128 type: list 129 choices: 130 - sites 131 - site 132 - location 133 - tenants 134 - tenant 135 - racks 136 - rack 137 - rack_group 138 - rack_role 139 - tags 140 - tag 141 - device_roles 142 - role 143 - device_types 144 - device_type 145 - manufacturers 146 - manufacturer 147 - platforms 148 - platform 149 - region 150 - cluster 151 - cluster_type 152 - cluster_group 153 - is_virtual 154 - services 155 - status 156 default: [] 157 group_names_raw: 158 description: Will not add the group_by choice name to the group names 159 default: False 160 type: boolean 161 version_added: "0.2.0" 162 query_filters: 163 description: List of parameters passed to the query string for both devices and VMs (Multiple values may be separated by commas) 164 type: list 165 default: [] 166 device_query_filters: 167 description: List of parameters passed to the query string for devices (Multiple values may be separated by commas) 168 type: list 169 default: [] 170 vm_query_filters: 171 description: List of parameters passed to the query string for VMs (Multiple values may be separated by commas) 172 type: list 173 default: [] 174 timeout: 175 description: Timeout for Netbox requests in seconds 176 type: int 177 default: 60 178 max_uri_length: 179 description: 180 - When fetch_all is False, GET requests to NetBox may become quite long and return a HTTP 414 (URI Too Long). 181 - You can adjust this option to be smaller to avoid 414 errors, or larger for a reduced number of requests. 182 type: int 183 default: 4000 184 version_added: "0.2.1" 185 virtual_chassis_name: 186 description: 187 - When a device is part of a virtual chassis, use the virtual chassis name as the Ansible inventory hostname. 188 - The host var values will be from the virtual chassis master. 189 type: boolean 190 default: False 191 dns_name: 192 description: 193 - Force IP Addresses to be fetched so that the dns_name for the primary_ip of each device or VM is set as a host_var. 194 - Setting interfaces will also fetch IP addresses and the dns_name host_var will be set. 195 type: boolean 196 default: False 197 ansible_host_dns_name: 198 description: 199 - If True, sets DNS Name (fetched from primary_ip) to be used in ansible_host variable, instead of IP Address. 200 type: boolean 201 default: False 202 compose: 203 description: List of custom ansible host vars to create from the device object fetched from NetBox 204 default: {} 205 type: dict 206""" 207 208EXAMPLES = """ 209# netbox_inventory.yml file in YAML format 210# Example command line: ansible-inventory -v --list -i netbox_inventory.yml 211 212plugin: netbox.netbox.nb_inventory 213api_endpoint: http://localhost:8000 214validate_certs: True 215config_context: False 216group_by: 217 - device_roles 218query_filters: 219 - role: network-edge-router 220device_query_filters: 221 - has_primary_ip: 'true' 222 223# has_primary_ip is a useful way to filter out patch panels and other passive devices 224 225# Query filters are passed directly as an argument to the fetching queries. 226# You can repeat tags in the query string. 227 228query_filters: 229 - role: server 230 - tag: web 231 - tag: production 232 233# See the NetBox documentation at https://netbox.readthedocs.io/en/stable/rest-api/overview/ 234# the query_filters work as a logical **OR** 235# 236# Prefix any custom fields with cf_ and pass the field value with the regular NetBox query string 237 238query_filters: 239 - cf_foo: bar 240 241# NetBox inventory plugin also supports Constructable semantics 242# You can fill your hosts vars using the compose option: 243 244plugin: netbox.netbox.nb_inventory 245compose: 246 foo: last_updated 247 bar: display_name 248 nested_variable: rack.display_name 249 250# You can use keyed_groups to group on properties of devices or VMs. 251# NOTE: It's only possible to key off direct items on the device/VM objects. 252plugin: netbox.netbox.nb_inventory 253keyed_groups: 254 - prefix: status 255 key: status.value 256""" 257 258import json 259import uuid 260import math 261from functools import partial 262from sys import version as python_version 263from threading import Thread 264from typing import Iterable 265from itertools import chain 266from collections import defaultdict 267from ipaddress import ip_interface 268from packaging import specifiers, version 269 270from ansible.plugins.inventory import BaseInventoryPlugin, Constructable, Cacheable 271from ansible.module_utils.ansible_release import __version__ as ansible_version 272from ansible.errors import AnsibleError 273from ansible.module_utils._text import to_text, to_native 274from ansible.module_utils.urls import open_url 275from ansible.module_utils.six.moves.urllib import error as urllib_error 276from ansible.module_utils.six.moves.urllib.parse import urlencode 277 278 279class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable): 280 NAME = "netbox.netbox.nb_inventory" 281 282 def _fetch_information(self, url): 283 results = None 284 cache_key = self.get_cache_key(url) 285 286 # get the user's cache option to see if we should save the cache if it is changing 287 user_cache_setting = self.get_option("cache") 288 289 # read if the user has caching enabled and the cache isn't being refreshed 290 attempt_to_read_cache = user_cache_setting and self.use_cache 291 292 # attempt to read the cache if inventory isn't being refreshed and the user has caching enabled 293 if attempt_to_read_cache: 294 try: 295 results = self._cache[cache_key] 296 need_to_fetch = False 297 except KeyError: 298 # occurs if the cache_key is not in the cache or if the cache_key expired 299 # we need to fetch the URL now 300 need_to_fetch = True 301 else: 302 # not reading from cache so do fetch 303 need_to_fetch = True 304 305 if need_to_fetch: 306 self.display.v("Fetching: " + url) 307 try: 308 response = open_url( 309 url, 310 headers=self.headers, 311 timeout=self.timeout, 312 validate_certs=self.validate_certs, 313 follow_redirects=self.follow_redirects, 314 client_cert=self.cert, 315 client_key=self.key, 316 ca_path=self.ca_path, 317 ) 318 except urllib_error.HTTPError as e: 319 """This will return the response body when we encounter an error. 320 This is to help determine what might be the issue when encountering an error. 321 Please check issue #294 for more info. 322 """ 323 # Prevent inventory from failing completely if the token does not have the proper permissions for specific URLs 324 if e.code == 403: 325 self.display.display( 326 "Permission denied: {0}. This may impair functionality of the inventory plugin.".format( 327 url 328 ), 329 color="red", 330 ) 331 # Need to return mock response data that is empty to prevent any failures downstream 332 return {"results": [], "next": None} 333 334 raise AnsibleError(to_native(e.fp.read())) 335 336 try: 337 raw_data = to_text(response.read(), errors="surrogate_or_strict") 338 except UnicodeError: 339 raise AnsibleError( 340 "Incorrect encoding of fetched payload from NetBox API." 341 ) 342 343 try: 344 results = json.loads(raw_data) 345 except ValueError: 346 raise AnsibleError("Incorrect JSON payload: %s" % raw_data) 347 348 # put result in cache if enabled 349 if user_cache_setting: 350 self._cache[cache_key] = results 351 352 return results 353 354 def get_resource_list(self, api_url): 355 """Retrieves resource list from netbox API. 356 Returns: 357 A list of all resource from netbox API. 358 """ 359 if not api_url: 360 raise AnsibleError("Please check API URL in script configuration file.") 361 362 resources = [] 363 364 # Handle pagination 365 while api_url: 366 api_output = self._fetch_information(api_url) 367 resources.extend(api_output["results"]) 368 api_url = api_output["next"] 369 370 return resources 371 372 def get_resource_list_chunked(self, api_url, query_key, query_values): 373 # Make an API call for multiple specific IDs, like /api/ipam/ip-addresses?limit=0&device_id=1&device_id=2&device_id=3 374 # Drastically cuts down HTTP requests comnpared to 1 request per host, in the case where we don't want to fetch_all 375 376 # Make sure query_values is subscriptable 377 if not isinstance(query_values, list): 378 query_values = list(query_values) 379 380 def query_string(value, separator="&"): 381 return separator + query_key + "=" + str(value) 382 383 # Calculate how many queries we can do per API call to stay within max_url_length 384 largest_value = str(max(query_values, default=0)) # values are always id ints 385 length_per_value = len(query_string(largest_value)) 386 chunk_size = math.floor((self.max_uri_length - len(api_url)) / length_per_value) 387 388 # Sanity check, for case where max_uri_length < (api_url + length_per_value) 389 if chunk_size < 1: 390 chunk_size = 1 391 392 if self.api_version in specifiers.SpecifierSet("~=2.6.0"): 393 # Issue netbox-community/netbox#3507 was fixed in v2.7.5 394 # If using NetBox v2.7.0-v2.7.4 will have to manually set max_uri_length to 0, 395 # but it's probably faster to keep fetch_all: True 396 # (You should really just upgrade your Netbox install) 397 chunk_size = 1 398 399 resources = [] 400 401 for i in range(0, len(query_values), chunk_size): 402 chunk = query_values[i : i + chunk_size] 403 # process chunk of size <= chunk_size 404 url = api_url 405 for value in chunk: 406 url += query_string(value, "&" if "?" in url else "?") 407 408 resources.extend(self.get_resource_list(url)) 409 410 return resources 411 412 @property 413 def group_extractors(self): 414 415 # List of group_by options and hostvars to extract 416 417 # Some keys are different depending on plurals option 418 extractors = { 419 "disk": self.extract_disk, 420 "memory": self.extract_memory, 421 "vcpus": self.extract_vcpus, 422 "status": self.extract_status, 423 "config_context": self.extract_config_context, 424 "local_context_data": self.extract_local_context_data, 425 "custom_fields": self.extract_custom_fields, 426 "region": self.extract_regions, 427 "cluster": self.extract_cluster, 428 "cluster_group": self.extract_cluster_group, 429 "cluster_type": self.extract_cluster_type, 430 "is_virtual": self.extract_is_virtual, 431 self._pluralize_group_by("site"): self.extract_site, 432 self._pluralize_group_by("tenant"): self.extract_tenant, 433 self._pluralize_group_by("rack"): self.extract_rack, 434 "rack_role": self.extract_rack_role, 435 self._pluralize_group_by("tag"): self.extract_tags, 436 self._pluralize_group_by("role"): self.extract_device_role, 437 self._pluralize_group_by("platform"): self.extract_platform, 438 self._pluralize_group_by("device_type"): self.extract_device_type, 439 self._pluralize_group_by("manufacturer"): self.extract_manufacturer, 440 } 441 442 # Locations were added in 2.11 replacing rack-groups. 443 if self.api_version >= version.parse("2.11"): 444 extractors.update( 445 {"location": self.extract_location,} 446 ) 447 else: 448 extractors.update( 449 {"rack_group": self.extract_rack_group,} 450 ) 451 452 if self.services: 453 extractors.update( 454 {"services": self.extract_services,} 455 ) 456 457 if self.interfaces: 458 extractors.update( 459 {"interfaces": self.extract_interfaces,} 460 ) 461 462 if self.interfaces or self.dns_name or self.ansible_host_dns_name: 463 extractors.update( 464 {"dns_name": self.extract_dns_name,} 465 ) 466 467 return extractors 468 469 def _pluralize_group_by(self, group_by): 470 mapping = { 471 "site": "sites", 472 "tenant": "tenants", 473 "rack": "racks", 474 "tag": "tags", 475 "role": "device_roles", 476 "platform": "platforms", 477 "device_type": "device_types", 478 "manufacturer": "manufacturers", 479 } 480 481 if self.plurals: 482 mapped = mapping.get(group_by) 483 return mapped or group_by 484 else: 485 return group_by 486 487 def _pluralize(self, extracted_value): 488 # If plurals is enabled, wrap in a single-element list for backwards compatibility 489 if self.plurals: 490 return [extracted_value] 491 else: 492 return extracted_value 493 494 def _objects_array_following_parents( 495 self, initial_object_id, object_lookup, object_parent_lookup 496 ): 497 objects = [] 498 499 object_id = initial_object_id 500 501 # Keep looping until the object has no parent 502 while object_id is not None: 503 object_slug = object_lookup[object_id] 504 505 if object_slug in objects: 506 # Won't ever happen - defensively guard against infinite loop 507 break 508 509 objects.append(object_slug) 510 511 # Get the parent of this object 512 object_id = object_parent_lookup[object_id] 513 514 return objects 515 516 def extract_disk(self, host): 517 return host.get("disk") 518 519 def extract_vcpus(self, host): 520 return host.get("vcpus") 521 522 def extract_status(self, host): 523 return host["status"] 524 525 def extract_memory(self, host): 526 return host.get("memory") 527 528 def extract_platform(self, host): 529 try: 530 return self._pluralize(self.platforms_lookup[host["platform"]["id"]]) 531 except Exception: 532 return 533 534 def extract_services(self, host): 535 try: 536 services_lookup = ( 537 self.vm_services_lookup 538 if host["is_virtual"] 539 else self.device_services_lookup 540 ) 541 542 return list(services_lookup[host["id"]].values()) 543 544 except Exception: 545 return 546 547 def extract_device_type(self, host): 548 try: 549 return self._pluralize(self.device_types_lookup[host["device_type"]["id"]]) 550 except Exception: 551 return 552 553 def extract_rack(self, host): 554 try: 555 return self._pluralize(self.racks_lookup[host["rack"]["id"]]) 556 except Exception: 557 return 558 559 def extract_rack_group(self, host): 560 # A host may have a rack. A rack may have a rack_group. A rack_group may have a parent rack_group. 561 # Produce a list of rack_groups: 562 # - it will be empty if the device has no rack, or the rack has no rack_group 563 # - it will have 1 element if the rack's group has no parent 564 # - it will have multiple elements if the rack's group has a parent group 565 566 rack = host.get("rack", None) 567 if not isinstance(rack, dict): 568 # Device has no rack 569 return None 570 571 rack_id = rack.get("id", None) 572 if rack_id is None: 573 # Device has no rack 574 return None 575 576 return self._objects_array_following_parents( 577 initial_object_id=self.racks_group_lookup[rack_id], 578 object_lookup=self.rack_groups_lookup, 579 object_parent_lookup=self.rack_group_parent_lookup, 580 ) 581 582 def extract_rack_role(self, host): 583 try: 584 return self.racks_role_lookup[host["rack"]["id"]] 585 except Exception: 586 return 587 588 def extract_site(self, host): 589 try: 590 return self._pluralize(self.sites_lookup[host["site"]["id"]]) 591 except Exception: 592 return 593 594 def extract_tenant(self, host): 595 try: 596 return self._pluralize(self.tenants_lookup[host["tenant"]["id"]]) 597 except Exception: 598 return 599 600 def extract_device_role(self, host): 601 try: 602 if "device_role" in host: 603 return self._pluralize( 604 self.device_roles_lookup[host["device_role"]["id"]] 605 ) 606 elif "role" in host: 607 return self._pluralize(self.device_roles_lookup[host["role"]["id"]]) 608 except Exception: 609 return 610 611 def extract_config_context(self, host): 612 try: 613 if self.flatten_config_context: 614 # Don't wrap in an array if we're about to flatten it to separate host vars 615 return host["config_context"] 616 else: 617 return self._pluralize(host["config_context"]) 618 except Exception: 619 return 620 621 def extract_local_context_data(self, host): 622 try: 623 if self.flatten_local_context_data: 624 # Don't wrap in an array if we're about to flatten it to separate host vars 625 return host["local_context_data"] 626 else: 627 return self._pluralize(host["local_context_data"]) 628 except Exception: 629 return 630 631 def extract_manufacturer(self, host): 632 try: 633 return self._pluralize( 634 self.manufacturers_lookup[host["device_type"]["manufacturer"]["id"]] 635 ) 636 except Exception: 637 return 638 639 def extract_primary_ip(self, host): 640 try: 641 address = host["primary_ip"]["address"] 642 return str(ip_interface(address).ip) 643 except Exception: 644 return 645 646 def extract_primary_ip4(self, host): 647 try: 648 address = host["primary_ip4"]["address"] 649 return str(ip_interface(address).ip) 650 except Exception: 651 return 652 653 def extract_primary_ip6(self, host): 654 try: 655 address = host["primary_ip6"]["address"] 656 return str(ip_interface(address).ip) 657 except Exception: 658 return 659 660 def extract_tags(self, host): 661 try: 662 tag_zero = host["tags"][0] 663 # Check the type of the first element in the "tags" array. 664 # If a dictionary (Netbox >= 2.9), return an array of tags' slugs. 665 if isinstance(tag_zero, dict): 666 return list(sub["slug"] for sub in host["tags"]) 667 # If a string (Netbox <= 2.8), return the original "tags" array. 668 elif isinstance(tag_zero, str): 669 return host["tags"] 670 # If tag_zero fails definition (no tags), return the empty array. 671 except Exception: 672 return host["tags"] 673 674 def extract_interfaces(self, host): 675 try: 676 677 interfaces_lookup = ( 678 self.vm_interfaces_lookup 679 if host["is_virtual"] 680 else self.device_interfaces_lookup 681 ) 682 683 interfaces = list(interfaces_lookup[host["id"]].values()) 684 685 before_netbox_v29 = bool(self.ipaddresses_intf_lookup) 686 # Attach IP Addresses to their interface 687 for interface in interfaces: 688 if before_netbox_v29: 689 interface["ip_addresses"] = list( 690 self.ipaddresses_intf_lookup[interface["id"]].values() 691 ) 692 else: 693 interface["ip_addresses"] = list( 694 self.vm_ipaddresses_intf_lookup[interface["id"]].values() 695 if host["is_virtual"] 696 else self.device_ipaddresses_intf_lookup[ 697 interface["id"] 698 ].values() 699 ) 700 interface["tags"] = list(sub["slug"] for sub in interface["tags"]) 701 702 return interfaces 703 except Exception: 704 return 705 706 def extract_custom_fields(self, host): 707 try: 708 return host["custom_fields"] 709 except Exception: 710 return 711 712 def extract_regions(self, host): 713 # A host may have a site. A site may have a region. A region may have a parent region. 714 # Produce a list of regions: 715 # - it will be empty if the device has no site, or the site has no region set 716 # - it will have 1 element if the site's region has no parent 717 # - it will have multiple elements if the site's region has a parent region 718 719 site = host.get("site", None) 720 if not isinstance(site, dict): 721 # Device has no site 722 return [] 723 724 site_id = site.get("id", None) 725 if site_id is None: 726 # Device has no site 727 return [] 728 729 return self._objects_array_following_parents( 730 initial_object_id=self.sites_region_lookup[site_id], 731 object_lookup=self.regions_lookup, 732 object_parent_lookup=self.regions_parent_lookup, 733 ) 734 735 def extract_location(self, host): 736 # A host may have a location. A location may have a parent location. 737 # Produce a list of locations: 738 # - it will be empty if the device has no location 739 # - it will have 1 element if the device's location has no parent 740 # - it will have multiple elements if the location has a parent location 741 742 try: 743 location_id = host["location"]["id"] 744 except (KeyError, TypeError): 745 # Device has no location 746 return [] 747 748 return self._objects_array_following_parents( 749 initial_object_id=location_id, 750 object_lookup=self.locations_lookup, 751 object_parent_lookup=self.locations_parent_lookup, 752 ) 753 754 def extract_cluster(self, host): 755 try: 756 # cluster does not have a slug 757 return host["cluster"]["name"] 758 except Exception: 759 return 760 761 def extract_cluster_group(self, host): 762 try: 763 return self.clusters_group_lookup[host["cluster"]["id"]] 764 except Exception: 765 return 766 767 def extract_cluster_type(self, host): 768 try: 769 return self.clusters_type_lookup[host["cluster"]["id"]] 770 except Exception: 771 return 772 773 def extract_is_virtual(self, host): 774 return host.get("is_virtual") 775 776 def extract_dns_name(self, host): 777 # No primary IP assigned 778 if not host.get("primary_ip"): 779 return None 780 781 before_netbox_v29 = bool(self.ipaddresses_lookup) 782 if before_netbox_v29: 783 ip_address = self.ipaddresses_lookup.get(host["primary_ip"]["id"]) 784 else: 785 if host["is_virtual"]: 786 ip_address = self.vm_ipaddresses_lookup.get(host["primary_ip"]["id"]) 787 else: 788 ip_address = self.device_ipaddresses_lookup.get( 789 host["primary_ip"]["id"] 790 ) 791 792 # Don"t assign a host_var for empty dns_name 793 if ip_address.get("dns_name") == "": 794 return None 795 796 return ip_address.get("dns_name") 797 798 def refresh_platforms_lookup(self): 799 url = self.api_endpoint + "/api/dcim/platforms/?limit=0" 800 platforms = self.get_resource_list(api_url=url) 801 self.platforms_lookup = dict( 802 (platform["id"], platform["slug"]) for platform in platforms 803 ) 804 805 def refresh_sites_lookup(self): 806 url = self.api_endpoint + "/api/dcim/sites/?limit=0" 807 sites = self.get_resource_list(api_url=url) 808 self.sites_lookup = dict((site["id"], site["slug"]) for site in sites) 809 810 def get_region_for_site(site): 811 # Will fail if site does not have a region defined in Netbox 812 try: 813 return (site["id"], site["region"]["id"]) 814 except Exception: 815 return (site["id"], None) 816 817 # Dictionary of site id to region id 818 self.sites_region_lookup = dict(map(get_region_for_site, sites)) 819 820 def refresh_regions_lookup(self): 821 url = self.api_endpoint + "/api/dcim/regions/?limit=0" 822 regions = self.get_resource_list(api_url=url) 823 self.regions_lookup = dict((region["id"], region["slug"]) for region in regions) 824 825 def get_region_parent(region): 826 # Will fail if region does not have a parent region 827 try: 828 return (region["id"], region["parent"]["id"]) 829 except Exception: 830 return (region["id"], None) 831 832 # Dictionary of region id to parent region id 833 self.regions_parent_lookup = dict( 834 filter(lambda x: x is not None, map(get_region_parent, regions)) 835 ) 836 837 def refresh_locations_lookup(self): 838 # Locations were added in v2.11. Return empty lookups for previous versions. 839 if self.api_version < version.parse("2.11"): 840 return 841 842 url = self.api_endpoint + "/api/dcim/locations/?limit=0" 843 locations = self.get_resource_list(api_url=url) 844 self.locations_lookup = dict( 845 (location["id"], location["slug"]) for location in locations 846 ) 847 848 def get_location_parent(location): 849 # Will fail if location does not have a parent location 850 try: 851 return (location["id"], location["parent"]["id"]) 852 except Exception: 853 return (location["id"], None) 854 855 def get_location_site(location): 856 # Locations MUST be assigned to a site 857 return (location["id"], location["site"]["id"]) 858 859 # Dictionary of location id to parent location id 860 self.locations_parent_lookup = dict( 861 filter(None, map(get_location_parent, locations)) 862 ) 863 # Location to site lookup 864 self.locations_site_lookup = dict(map(get_location_site, locations)) 865 866 def refresh_tenants_lookup(self): 867 url = self.api_endpoint + "/api/tenancy/tenants/?limit=0" 868 tenants = self.get_resource_list(api_url=url) 869 self.tenants_lookup = dict((tenant["id"], tenant["slug"]) for tenant in tenants) 870 871 def refresh_racks_lookup(self): 872 url = self.api_endpoint + "/api/dcim/racks/?limit=0" 873 racks = self.get_resource_list(api_url=url) 874 self.racks_lookup = dict((rack["id"], rack["name"]) for rack in racks) 875 876 def get_group_for_rack(rack): 877 try: 878 return (rack["id"], rack["group"]["id"]) 879 except Exception: 880 return (rack["id"], None) 881 882 def get_role_for_rack(rack): 883 try: 884 return (rack["id"], rack["role"]["slug"]) 885 except Exception: 886 return (rack["id"], None) 887 888 self.racks_group_lookup = dict(map(get_group_for_rack, racks)) 889 self.racks_role_lookup = dict(map(get_role_for_rack, racks)) 890 891 def refresh_rack_groups_lookup(self): 892 # Locations were added in v2.11 replacing rack groups. Do nothing for 2.11+ 893 if self.api_version >= version.parse("2.11"): 894 return 895 896 url = self.api_endpoint + "/api/dcim/rack-groups/?limit=0" 897 rack_groups = self.get_resource_list(api_url=url) 898 self.rack_groups_lookup = dict( 899 (rack_group["id"], rack_group["slug"]) for rack_group in rack_groups 900 ) 901 902 def get_rack_group_parent(rack_group): 903 try: 904 return (rack_group["id"], rack_group["parent"]["id"]) 905 except Exception: 906 return (rack_group["id"], None) 907 908 # Dictionary of rack group id to parent rack group id 909 self.rack_group_parent_lookup = dict(map(get_rack_group_parent, rack_groups)) 910 911 def refresh_device_roles_lookup(self): 912 url = self.api_endpoint + "/api/dcim/device-roles/?limit=0" 913 device_roles = self.get_resource_list(api_url=url) 914 self.device_roles_lookup = dict( 915 (device_role["id"], device_role["slug"]) for device_role in device_roles 916 ) 917 918 def refresh_device_types_lookup(self): 919 url = self.api_endpoint + "/api/dcim/device-types/?limit=0" 920 device_types = self.get_resource_list(api_url=url) 921 self.device_types_lookup = dict( 922 (device_type["id"], device_type["slug"]) for device_type in device_types 923 ) 924 925 def refresh_manufacturers_lookup(self): 926 url = self.api_endpoint + "/api/dcim/manufacturers/?limit=0" 927 manufacturers = self.get_resource_list(api_url=url) 928 self.manufacturers_lookup = dict( 929 (manufacturer["id"], manufacturer["slug"]) for manufacturer in manufacturers 930 ) 931 932 def refresh_clusters_lookup(self): 933 url = self.api_endpoint + "/api/virtualization/clusters/?limit=0" 934 clusters = self.get_resource_list(api_url=url) 935 936 def get_cluster_type(cluster): 937 # Will fail if cluster does not have a type (required property so should always be true) 938 try: 939 return (cluster["id"], cluster["type"]["slug"]) 940 except Exception: 941 return (cluster["id"], None) 942 943 def get_cluster_group(cluster): 944 # Will fail if cluster does not have a group (group is optional) 945 try: 946 return (cluster["id"], cluster["group"]["slug"]) 947 except Exception: 948 return (cluster["id"], None) 949 950 self.clusters_type_lookup = dict(map(get_cluster_type, clusters)) 951 self.clusters_group_lookup = dict(map(get_cluster_group, clusters)) 952 953 def refresh_services(self): 954 url = self.api_endpoint + "/api/ipam/services/?limit=0" 955 services = [] 956 957 if self.fetch_all: 958 services = self.get_resource_list(url) 959 else: 960 device_services = self.get_resource_list_chunked( 961 api_url=url, 962 query_key="device_id", 963 query_values=self.devices_lookup.keys(), 964 ) 965 vm_services = self.get_resource_list_chunked( 966 api_url=url, 967 query_key="virtual_machine_id", 968 query_values=self.vms_lookup.keys(), 969 ) 970 services = chain(device_services, vm_services) 971 972 # Construct a dictionary of dictionaries, separately for devices and vms. 973 # Allows looking up services by device id or vm id 974 self.device_services_lookup = defaultdict(dict) 975 self.vm_services_lookup = defaultdict(dict) 976 977 for service in services: 978 service_id = service["id"] 979 980 if service.get("device"): 981 self.device_services_lookup[service["device"]["id"]][ 982 service_id 983 ] = service 984 985 if service.get("virtual_machine"): 986 self.vm_services_lookup[service["virtual_machine"]["id"]][ 987 service_id 988 ] = service 989 990 def refresh_interfaces(self): 991 992 url_device_interfaces = self.api_endpoint + "/api/dcim/interfaces/?limit=0" 993 url_vm_interfaces = ( 994 self.api_endpoint + "/api/virtualization/interfaces/?limit=0" 995 ) 996 997 device_interfaces = [] 998 vm_interfaces = [] 999 1000 if self.fetch_all: 1001 device_interfaces = self.get_resource_list(url_device_interfaces) 1002 vm_interfaces = self.get_resource_list(url_vm_interfaces) 1003 else: 1004 device_interfaces = self.get_resource_list_chunked( 1005 api_url=url_device_interfaces, 1006 query_key="device_id", 1007 query_values=self.devices_lookup.keys(), 1008 ) 1009 vm_interfaces = self.get_resource_list_chunked( 1010 api_url=url_vm_interfaces, 1011 query_key="virtual_machine_id", 1012 query_values=self.vms_lookup.keys(), 1013 ) 1014 1015 # Construct a dictionary of dictionaries, separately for devices and vms. 1016 # For a given device id or vm id, get a lookup of interface id to interface 1017 # This is because interfaces may be returned multiple times when querying for virtual chassis parent and child in separate queries 1018 self.device_interfaces_lookup = defaultdict(dict) 1019 self.vm_interfaces_lookup = defaultdict(dict) 1020 1021 # /dcim/interfaces gives count_ipaddresses per interface. /virtualization/interfaces does not 1022 self.devices_with_ips = set() 1023 1024 for interface in device_interfaces: 1025 interface_id = interface["id"] 1026 device_id = interface["device"]["id"] 1027 1028 # Check if device_id is actually a device we've fetched, and was not filtered out by query_filters 1029 if device_id not in self.devices_lookup: 1030 continue 1031 1032 # Check if device_id is part of a virtual chasis 1033 # If so, treat its interfaces as actually part of the master 1034 device = self.devices_lookup[device_id] 1035 virtual_chassis_master = self._get_host_virtual_chassis_master(device) 1036 if virtual_chassis_master is not None: 1037 device_id = virtual_chassis_master 1038 1039 self.device_interfaces_lookup[device_id][interface_id] = interface 1040 1041 # Keep track of what devices have interfaces with IPs, so if fetch_all is False we can avoid unnecessary queries 1042 if interface["count_ipaddresses"] > 0: 1043 self.devices_with_ips.add(device_id) 1044 1045 for interface in vm_interfaces: 1046 interface_id = interface["id"] 1047 vm_id = interface["virtual_machine"]["id"] 1048 1049 self.vm_interfaces_lookup[vm_id][interface_id] = interface 1050 1051 # Note: depends on the result of refresh_interfaces for self.devices_with_ips 1052 def refresh_ipaddresses(self): 1053 url = ( 1054 self.api_endpoint 1055 + "/api/ipam/ip-addresses/?limit=0&assigned_to_interface=true" 1056 ) 1057 ipaddresses = [] 1058 1059 if self.fetch_all: 1060 ipaddresses = self.get_resource_list(url) 1061 else: 1062 device_ips = self.get_resource_list_chunked( 1063 api_url=url, 1064 query_key="device_id", 1065 query_values=list(self.devices_with_ips), 1066 ) 1067 vm_ips = self.get_resource_list_chunked( 1068 api_url=url, 1069 query_key="virtual_machine_id", 1070 query_values=self.vms_lookup.keys(), 1071 ) 1072 1073 ipaddresses = chain(device_ips, vm_ips) 1074 1075 # Construct a dictionary of lists, to allow looking up ip addresses by interface id 1076 # Note that interface ids share the same namespace for both devices and vms so this is a single dictionary 1077 self.ipaddresses_intf_lookup = defaultdict(dict) 1078 # Construct a dictionary of the IP addresses themselves 1079 self.ipaddresses_lookup = defaultdict(dict) 1080 # NetBox v2.9 and onwards 1081 self.vm_ipaddresses_intf_lookup = defaultdict(dict) 1082 self.vm_ipaddresses_lookup = defaultdict(dict) 1083 self.device_ipaddresses_intf_lookup = defaultdict(dict) 1084 self.device_ipaddresses_lookup = defaultdict(dict) 1085 1086 for ipaddress in ipaddresses: 1087 # As of NetBox v2.9 "assigned_object_x" replaces "interface" 1088 if ipaddress.get("assigned_object_id"): 1089 interface_id = ipaddress["assigned_object_id"] 1090 ip_id = ipaddress["id"] 1091 # We need to copy the ipaddress entry to preserve the original in case caching is used. 1092 ipaddress_copy = ipaddress.copy() 1093 1094 if ipaddress["assigned_object_type"] == "virtualization.vminterface": 1095 self.vm_ipaddresses_lookup[ip_id] = ipaddress_copy 1096 self.vm_ipaddresses_intf_lookup[interface_id][ 1097 ip_id 1098 ] = ipaddress_copy 1099 else: 1100 self.device_ipaddresses_lookup[ip_id] = ipaddress_copy 1101 self.device_ipaddresses_intf_lookup[interface_id][ 1102 ip_id 1103 ] = ipaddress_copy # Remove "assigned_object_X" attributes, as that's redundant when ipaddress is added to an interface 1104 1105 del ipaddress_copy["assigned_object_id"] 1106 del ipaddress_copy["assigned_object_type"] 1107 del ipaddress_copy["assigned_object"] 1108 continue 1109 1110 if not ipaddress.get("interface"): 1111 continue 1112 interface_id = ipaddress["interface"]["id"] 1113 ip_id = ipaddress["id"] 1114 1115 # We need to copy the ipaddress entry to preserve the original in case caching is used. 1116 ipaddress_copy = ipaddress.copy() 1117 1118 self.ipaddresses_intf_lookup[interface_id][ip_id] = ipaddress_copy 1119 self.ipaddresses_lookup[ip_id] = ipaddress_copy 1120 # Remove "interface" attribute, as that's redundant when ipaddress is added to an interface 1121 del ipaddress_copy["interface"] 1122 1123 @property 1124 def lookup_processes(self): 1125 lookups = [ 1126 self.refresh_sites_lookup, 1127 self.refresh_regions_lookup, 1128 self.refresh_locations_lookup, 1129 self.refresh_tenants_lookup, 1130 self.refresh_racks_lookup, 1131 self.refresh_rack_groups_lookup, 1132 self.refresh_device_roles_lookup, 1133 self.refresh_platforms_lookup, 1134 self.refresh_device_types_lookup, 1135 self.refresh_manufacturers_lookup, 1136 self.refresh_clusters_lookup, 1137 ] 1138 1139 if self.interfaces: 1140 lookups.append(self.refresh_interfaces) 1141 1142 if self.services: 1143 lookups.append(self.refresh_services) 1144 1145 return lookups 1146 1147 @property 1148 def lookup_processes_secondary(self): 1149 lookups = [] 1150 1151 # IP addresses are needed for either interfaces or dns_name options 1152 if self.interfaces or self.dns_name or self.ansible_host_dns_name: 1153 lookups.append(self.refresh_ipaddresses) 1154 1155 return lookups 1156 1157 def refresh_lookups(self, lookups): 1158 1159 # Exceptions that occur in threads by default are printed to stderr, and ignored by the main thread 1160 # They need to be caught, and raised in the main thread to prevent further execution of this plugin 1161 1162 thread_exceptions = [] 1163 1164 def handle_thread_exceptions(lookup): 1165 def wrapper(): 1166 try: 1167 lookup() 1168 except Exception as e: 1169 # Save for the main-thread to re-raise 1170 # Also continue to raise on this thread, so the default handler can run to print to stderr 1171 thread_exceptions.append(e) 1172 raise e 1173 1174 return wrapper 1175 1176 thread_list = [] 1177 1178 try: 1179 for lookup in lookups: 1180 thread = Thread(target=handle_thread_exceptions(lookup)) 1181 thread_list.append(thread) 1182 thread.start() 1183 1184 for thread in thread_list: 1185 thread.join() 1186 1187 # Wait till we've joined all threads before raising any exceptions 1188 for exception in thread_exceptions: 1189 raise exception 1190 1191 finally: 1192 # Avoid retain cycles 1193 thread_exceptions = None 1194 1195 def fetch_api_docs(self): 1196 openapi = self._fetch_information( 1197 self.api_endpoint + "/api/docs/?format=openapi" 1198 ) 1199 1200 self.api_version = version.parse(openapi["info"]["version"]) 1201 self.allowed_device_query_parameters = [ 1202 p["name"] for p in openapi["paths"]["/dcim/devices/"]["get"]["parameters"] 1203 ] 1204 self.allowed_vm_query_parameters = [ 1205 p["name"] 1206 for p in openapi["paths"]["/virtualization/virtual-machines/"]["get"][ 1207 "parameters" 1208 ] 1209 ] 1210 1211 def validate_query_parameter(self, parameter, allowed_query_parameters): 1212 if not (isinstance(parameter, dict) and len(parameter) == 1): 1213 self.display.warning( 1214 "Warning query parameters %s not a dict with a single key." % parameter 1215 ) 1216 return None 1217 1218 k = tuple(parameter.keys())[0] 1219 v = tuple(parameter.values())[0] 1220 1221 if not (k in allowed_query_parameters or k.startswith("cf_")): 1222 msg = "Warning: %s not in %s or starting with cf (Custom field)" % ( 1223 k, 1224 allowed_query_parameters, 1225 ) 1226 self.display.warning(msg=msg) 1227 return None 1228 return k, v 1229 1230 def filter_query_parameters(self, parameters, allowed_query_parameters): 1231 return filter( 1232 lambda parameter: parameter is not None, 1233 # For each element of query_filters, test if it's allowed 1234 map( 1235 # Create a partial function with the device-specific list of query parameters 1236 partial( 1237 self.validate_query_parameter, 1238 allowed_query_parameters=allowed_query_parameters, 1239 ), 1240 parameters, 1241 ), 1242 ) 1243 1244 def refresh_url(self): 1245 device_query_parameters = [("limit", 0)] 1246 vm_query_parameters = [("limit", 0)] 1247 device_url = self.api_endpoint + "/api/dcim/devices/?" 1248 vm_url = self.api_endpoint + "/api/virtualization/virtual-machines/?" 1249 1250 # Add query_filtes to both devices and vms query, if they're valid 1251 if isinstance(self.query_filters, Iterable): 1252 device_query_parameters.extend( 1253 self.filter_query_parameters( 1254 self.query_filters, self.allowed_device_query_parameters 1255 ) 1256 ) 1257 1258 vm_query_parameters.extend( 1259 self.filter_query_parameters( 1260 self.query_filters, self.allowed_vm_query_parameters 1261 ) 1262 ) 1263 1264 if isinstance(self.device_query_filters, Iterable): 1265 device_query_parameters.extend( 1266 self.filter_query_parameters( 1267 self.device_query_filters, self.allowed_device_query_parameters 1268 ) 1269 ) 1270 1271 if isinstance(self.vm_query_filters, Iterable): 1272 vm_query_parameters.extend( 1273 self.filter_query_parameters( 1274 self.vm_query_filters, self.allowed_vm_query_parameters 1275 ) 1276 ) 1277 1278 # When query_filters is Iterable, and is not empty: 1279 # - If none of the filters are valid for devices, do not fetch any devices 1280 # - If none of the filters are valid for VMs, do not fetch any VMs 1281 # If either device_query_filters or vm_query_filters are set, 1282 # device_query_parameters and vm_query_parameters will have > 1 element so will continue to be requested 1283 if self.query_filters and isinstance(self.query_filters, Iterable): 1284 if len(device_query_parameters) <= 1: 1285 device_url = None 1286 1287 if len(vm_query_parameters) <= 1: 1288 vm_url = None 1289 1290 # Append the parameters to the URLs 1291 if device_url: 1292 device_url = device_url + urlencode(device_query_parameters) 1293 if vm_url: 1294 vm_url = vm_url + urlencode(vm_query_parameters) 1295 1296 # Exclude config_context if not required 1297 if not self.config_context: 1298 if device_url: 1299 device_url = device_url + "&exclude=config_context" 1300 if vm_url: 1301 vm_url = vm_url + "&exclude=config_context" 1302 1303 return device_url, vm_url 1304 1305 def fetch_hosts(self): 1306 device_url, vm_url = self.refresh_url() 1307 1308 self.devices_list = [] 1309 self.vms_list = [] 1310 1311 if device_url: 1312 self.devices_list = self.get_resource_list(device_url) 1313 1314 if vm_url: 1315 self.vms_list = self.get_resource_list(vm_url) 1316 1317 # Allow looking up devices/vms by their ids 1318 self.devices_lookup = {device["id"]: device for device in self.devices_list} 1319 self.vms_lookup = {vm["id"]: vm for vm in self.vms_list} 1320 1321 # There's nothing that explicitly says if a host is virtual or not - add in a new field 1322 for host in self.devices_list: 1323 host["is_virtual"] = False 1324 1325 for host in self.vms_list: 1326 host["is_virtual"] = True 1327 1328 def extract_name(self, host): 1329 # An host in an Ansible inventory requires an hostname. 1330 # name is an unique but not required attribute for a device in NetBox 1331 # We default to an UUID for hostname in case the name is not set in NetBox 1332 # Use virtual chassis name if set by the user. 1333 if self.virtual_chassis_name and self._get_host_virtual_chassis_master(host): 1334 return host["virtual_chassis"]["name"] or str(uuid.uuid4()) 1335 else: 1336 return host["name"] or str(uuid.uuid4()) 1337 1338 def generate_group_name(self, grouping, group): 1339 1340 # Check for special case - if group is a boolean, just return grouping name instead 1341 # eg. "is_virtual" - returns true for VMs, should put them in a group named "is_virtual", not "is_virtual_True" 1342 if isinstance(group, bool): 1343 if group: 1344 return grouping 1345 else: 1346 # Don't create the inverse group 1347 return None 1348 1349 # Special case. Extract name from service, which is a hash. 1350 if grouping == "services": 1351 group = group["name"] 1352 grouping = "service" 1353 1354 if grouping == "status": 1355 group = group["value"] 1356 1357 if self.group_names_raw: 1358 return group 1359 else: 1360 return "_".join([grouping, group]) 1361 1362 def add_host_to_groups(self, host, hostname): 1363 site_group_by = self._pluralize_group_by("site") 1364 1365 for grouping in self.group_by: 1366 1367 # Don't handle regions here since no hosts are ever added to region groups 1368 # Sites and locations are also specially handled in the main() 1369 if grouping in ["region", site_group_by, "location"]: 1370 continue 1371 1372 if grouping not in self.group_extractors: 1373 raise AnsibleError( 1374 'group_by option "%s" is not valid. Check group_by documentation or check the plurals option. It can determine what group_by options are valid.' 1375 % grouping 1376 ) 1377 1378 groups_for_host = self.group_extractors[grouping](host) 1379 1380 if not groups_for_host: 1381 continue 1382 1383 # Make groups_for_host a list if it isn't already 1384 if not isinstance(groups_for_host, list): 1385 groups_for_host = [groups_for_host] 1386 1387 for group_for_host in groups_for_host: 1388 group_name = self.generate_group_name(grouping, group_for_host) 1389 1390 if not group_name: 1391 continue 1392 1393 # Group names may be transformed by the ansible TRANSFORM_INVALID_GROUP_CHARS setting 1394 # add_group returns the actual group name used 1395 transformed_group_name = self.inventory.add_group(group=group_name) 1396 self.inventory.add_host(group=transformed_group_name, host=hostname) 1397 1398 def _add_site_groups(self): 1399 # Map site id to transformed group names 1400 self.site_group_names = dict() 1401 1402 for site_id, site_name in self.sites_lookup.items(): 1403 site_group_name = self.generate_group_name( 1404 self._pluralize_group_by("site"), site_name 1405 ) 1406 # Add the site group to get its transformed name 1407 site_transformed_group_name = self.inventory.add_group( 1408 group=site_group_name 1409 ) 1410 self.site_group_names[site_id] = site_transformed_group_name 1411 1412 def _add_region_groups(self): 1413 # Mapping of region id to group name 1414 region_transformed_group_names = self._setup_nested_groups( 1415 "region", self.regions_lookup, self.regions_parent_lookup 1416 ) 1417 1418 # Add site groups as children of region groups 1419 for site_id in self.sites_lookup: 1420 region_id = self.sites_region_lookup.get(site_id, None) 1421 if region_id is None: 1422 continue 1423 1424 self.inventory.add_child( 1425 region_transformed_group_names[region_id], 1426 self.site_group_names[site_id], 1427 ) 1428 1429 def _add_location_groups(self): 1430 # Mapping of location id to group name 1431 self.location_group_names = self._setup_nested_groups( 1432 "location", self.locations_lookup, self.locations_parent_lookup 1433 ) 1434 1435 # Add location to site groups as children 1436 for location_id, location_slug in self.locations_lookup.items(): 1437 if self.locations_parent_lookup.get(location_id, None): 1438 # Only top level locations should be children of sites 1439 continue 1440 1441 site_transformed_group_name = self.site_group_names[ 1442 self.locations_site_lookup[location_id] 1443 ] 1444 1445 self.inventory.add_child( 1446 site_transformed_group_name, self.location_group_names[location_id] 1447 ) 1448 1449 def _setup_nested_groups(self, group, lookup, parent_lookup): 1450 # Mapping of id to group name 1451 transformed_group_names = dict() 1452 1453 # Create groups for each object 1454 for obj_id in lookup: 1455 group_name = self.generate_group_name(group, lookup[obj_id]) 1456 transformed_group_names[obj_id] = self.inventory.add_group(group=group_name) 1457 1458 # Now that all groups exist, add relationships between them 1459 for obj_id in lookup: 1460 group_name = transformed_group_names[obj_id] 1461 parent_id = parent_lookup.get(obj_id, None) 1462 if parent_id is not None and parent_id in transformed_group_names: 1463 parent_name = transformed_group_names[parent_id] 1464 self.inventory.add_child(parent_name, group_name) 1465 1466 return transformed_group_names 1467 1468 def _fill_host_variables(self, host, hostname): 1469 extracted_primary_ip = self.extract_primary_ip(host=host) 1470 if extracted_primary_ip: 1471 self.inventory.set_variable(hostname, "ansible_host", extracted_primary_ip) 1472 1473 if self.ansible_host_dns_name: 1474 extracted_dns_name = self.extract_dns_name(host=host) 1475 if extracted_dns_name: 1476 self.inventory.set_variable( 1477 hostname, "ansible_host", extracted_dns_name 1478 ) 1479 1480 extracted_primary_ip4 = self.extract_primary_ip4(host=host) 1481 if extracted_primary_ip4: 1482 self.inventory.set_variable(hostname, "primary_ip4", extracted_primary_ip4) 1483 1484 extracted_primary_ip6 = self.extract_primary_ip6(host=host) 1485 if extracted_primary_ip6: 1486 self.inventory.set_variable(hostname, "primary_ip6", extracted_primary_ip6) 1487 1488 for attribute, extractor in self.group_extractors.items(): 1489 extracted_value = extractor(host) 1490 1491 # Compare with None, not just check for a truth comparison - allow empty arrays, etc to be host vars 1492 if extracted_value is None: 1493 continue 1494 1495 # Special case - all group_by options are single strings, but tag is a list of tags 1496 # Keep the groups named singular "tag_sometag", but host attribute should be "tags":["sometag", "someothertag"] 1497 if attribute == "tag": 1498 attribute = "tags" 1499 1500 if attribute == "region": 1501 attribute = "regions" 1502 1503 if attribute == "location": 1504 attribute = "locations" 1505 1506 if attribute == "rack_group": 1507 attribute = "rack_groups" 1508 1509 # Flatten the dict into separate host vars, if enabled 1510 if isinstance(extracted_value, dict) and ( 1511 (attribute == "config_context" and self.flatten_config_context) 1512 or (attribute == "custom_fields" and self.flatten_custom_fields) 1513 or ( 1514 attribute == "local_context_data" 1515 and self.flatten_local_context_data 1516 ) 1517 ): 1518 for key, value in extracted_value.items(): 1519 self.inventory.set_variable(hostname, key, value) 1520 else: 1521 self.inventory.set_variable(hostname, attribute, extracted_value) 1522 1523 def _get_host_virtual_chassis_master(self, host): 1524 virtual_chassis = host.get("virtual_chassis", None) 1525 1526 if not virtual_chassis: 1527 return None 1528 1529 master = virtual_chassis.get("master", None) 1530 1531 if not master: 1532 return None 1533 1534 return master.get("id", None) 1535 1536 def main(self): 1537 # Get info about the API - version, allowed query parameters 1538 self.fetch_api_docs() 1539 1540 self.fetch_hosts() 1541 1542 # Interface, and Service lookup will depend on hosts, if option fetch_all is false 1543 self.refresh_lookups(self.lookup_processes) 1544 1545 # Looking up IP Addresses depends on the result of interfaces count_ipaddresses field 1546 # - can skip any device/vm without any IPs 1547 self.refresh_lookups(self.lookup_processes_secondary) 1548 1549 # If we're grouping by regions, hosts are not added to region groups 1550 # If we're grouping by locations, hosts may be added to the site or location 1551 # - the site groups are added as sub-groups of regions 1552 # - the location groups are added as sub-groups of sites 1553 # So, we need to make sure we're also grouping by sites if regions or locations are enabled 1554 site_group_by = self._pluralize_group_by("site") 1555 if ( 1556 site_group_by in self.group_by 1557 or "location" in self.group_by 1558 or "region" in self.group_by 1559 ): 1560 self._add_site_groups() 1561 1562 # Create groups for locations. Will be a part of site groups. 1563 if "location" in self.group_by and self.api_version >= version.parse("2.11"): 1564 self._add_location_groups() 1565 1566 # Create groups for regions, containing the site groups 1567 if "region" in self.group_by: 1568 self._add_region_groups() 1569 1570 for host in chain(self.devices_list, self.vms_list): 1571 1572 virtual_chassis_master = self._get_host_virtual_chassis_master(host) 1573 if ( 1574 virtual_chassis_master is not None 1575 and virtual_chassis_master != host["id"] 1576 ): 1577 # Device is part of a virtual chassis, but is not the master 1578 continue 1579 1580 hostname = self.extract_name(host=host) 1581 self.inventory.add_host(host=hostname) 1582 self._fill_host_variables(host=host, hostname=hostname) 1583 1584 strict = self.get_option("strict") 1585 1586 # Composed variables 1587 self._set_composite_vars( 1588 self.get_option("compose"), host, hostname, strict=strict 1589 ) 1590 1591 # Complex groups based on jinja2 conditionals, hosts that meet the conditional are added to group 1592 self._add_host_to_composed_groups( 1593 self.get_option("groups"), host, hostname, strict=strict 1594 ) 1595 1596 # Create groups based on variable values and add the corresponding hosts to it 1597 self._add_host_to_keyed_groups( 1598 self.get_option("keyed_groups"), host, hostname, strict=strict 1599 ) 1600 self.add_host_to_groups(host=host, hostname=hostname) 1601 1602 # Special processing for sites and locations as those groups were already created 1603 if getattr(self, "location_group_names", None) and host.get("location"): 1604 # Add host to location group when host is assigned to the location 1605 self.inventory.add_host( 1606 group=self.location_group_names[host["location"]["id"]], 1607 host=hostname, 1608 ) 1609 elif getattr(self, "site_group_names", None) and host.get("site"): 1610 # Add host to site group when host is NOT assigned to a location 1611 self.inventory.add_host( 1612 group=self.site_group_names[host["site"]["id"]], host=hostname, 1613 ) 1614 1615 def parse(self, inventory, loader, path, cache=True): 1616 super(InventoryModule, self).parse(inventory, loader, path) 1617 self._read_config_data(path=path) 1618 self.use_cache = cache 1619 1620 # Netbox access 1621 token = self.get_option("token") 1622 # Handle extra "/" from api_endpoint configuration and trim if necessary, see PR#49943 1623 self.api_endpoint = self.get_option("api_endpoint").strip("/") 1624 self.timeout = self.get_option("timeout") 1625 self.max_uri_length = self.get_option("max_uri_length") 1626 self.validate_certs = self.get_option("validate_certs") 1627 self.follow_redirects = self.get_option("follow_redirects") 1628 self.config_context = self.get_option("config_context") 1629 self.flatten_config_context = self.get_option("flatten_config_context") 1630 self.flatten_local_context_data = self.get_option("flatten_local_context_data") 1631 self.flatten_custom_fields = self.get_option("flatten_custom_fields") 1632 self.plurals = self.get_option("plurals") 1633 self.interfaces = self.get_option("interfaces") 1634 self.services = self.get_option("services") 1635 self.fetch_all = self.get_option("fetch_all") 1636 self.headers = { 1637 "User-Agent": "ansible %s Python %s" 1638 % (ansible_version, python_version.split(" ")[0]), 1639 "Content-type": "application/json", 1640 } 1641 self.cert = self.get_option("cert") 1642 self.key = self.get_option("key") 1643 self.ca_path = self.get_option("ca_path") 1644 if token: 1645 self.headers.update({"Authorization": "Token %s" % token}) 1646 1647 # Filter and group_by options 1648 self.group_by = self.get_option("group_by") 1649 self.group_names_raw = self.get_option("group_names_raw") 1650 self.query_filters = self.get_option("query_filters") 1651 self.device_query_filters = self.get_option("device_query_filters") 1652 self.vm_query_filters = self.get_option("vm_query_filters") 1653 self.virtual_chassis_name = self.get_option("virtual_chassis_name") 1654 self.dns_name = self.get_option("dns_name") 1655 self.ansible_host_dns_name = self.get_option("ansible_host_dns_name") 1656 1657 self.main() 1658