1#!/usr/local/bin/python3.8 2 3# 4# (c) 2015, Steve Gargan <steve.gargan@gmail.com> 5# 6# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) 7 8from __future__ import (absolute_import, division, print_function) 9__metaclass__ = type 10 11###################################################################### 12 13''' 14Consul.io inventory script (http://consul.io) 15====================================== 16 17Generates Ansible inventory from nodes in a Consul cluster. This script will 18group nodes by: 19 - datacenter, 20 - registered service 21 - service tags 22 - service status 23 - values from the k/v store 24 25This script can be run with the switches 26--list as expected groups all the nodes in all datacenters 27--datacenter, to restrict the nodes to a single datacenter 28--host to restrict the inventory to a single named node. (requires datacenter config) 29 30The configuration for this plugin is read from a consul_io.ini file located in the 31same directory as this inventory script or via environment variables. All config options in the config file 32are optional except the host and port, which must point to a valid agent or 33server running the http api. For more information on enabling the endpoint see. 34 35http://www.consul.io/docs/agent/options.html 36 37Other options include: 38 39'bulk_load' 40 41boolean flag. Load all possible data before building inventory JSON 42If true, script processes in-memory data. JSON generation reduces drastically 43This can also be set with the environmental variable CONSUL_BULK_LOAD. 44 45'datacenter': 46 47which restricts the included nodes to those from the given datacenter 48This can also be set with the environmental variable CONSUL_DATACENTER. 49 50'url': 51 52the URL of the Consul cluster. host, port and scheme are derived from the 53URL. If not specified, connection configuration defaults to http requests 54to localhost on port 8500. 55This can also be set with the environmental variable CONSUL_URL. 56 57'domain': 58 59if specified then the inventory will generate domain names that will resolve 60via Consul's inbuilt DNS. The name is derived from the node name, datacenter 61and domain <node_name>.node.<datacenter>.<domain>. Note that you will need to 62have consul hooked into your DNS server for these to resolve. See the consul 63DNS docs for more info. 64 65which restricts the included nodes to those from the given datacenter 66This can also be set with the environmental variable CONSUL_DOMAIN. 67 68'suffixes': 69 70boolean flag. By default, final JSON is built based on all available info in consul. 71Suffixes means that services groups will be added in addition to basic information. See servers_suffix for additional info 72There are cases when speed is preferable than having services groups 73False value will reduce script execution time drastically. 74This can also be set with the environmental variable CONSUL_SUFFIXES. 75 76'servers_suffix': 77 78defining the a suffix to add to the service name when creating the service 79group. e.g Service name of 'redis' and a suffix of '_servers' will add 80each nodes address to the group name 'redis_servers'. No suffix is added 81if this is not set 82This can also be set with the environmental variable CONSUL_SERVERS_SUFFIX. 83 84'tags': 85 86boolean flag defining if service tags should be used to create Inventory 87groups e.g. an nginx service with the tags ['master', 'v1'] will create 88groups nginx_master and nginx_v1 to which the node running the service 89will be added. No tag groups are created if this is missing. 90This can also be set with the environmental variable CONSUL_TAGS. 91 92'token': 93 94ACL token to use to authorize access to the key value store. May be required 95to retrieve the kv_groups and kv_metadata based on your consul configuration. 96This can also be set with the environmental variable CONSUL_TOKEN. 97 98'kv_groups': 99 100This is used to lookup groups for a node in the key value store. It specifies a 101path to which each discovered node's name will be added to create a key to query 102the key/value store. There it expects to find a comma separated list of group 103names to which the node should be added e.g. if the inventory contains node 104'nyc-web-1' in datacenter 'nyc-dc1' and kv_groups = 'ansible/groups' then the key 105'ansible/groups/nyc-dc1/nyc-web-1' will be queried for a group list. If this query 106 returned 'test,honeypot' then the node address to both groups. 107This can also be set with the environmental variable CONSUL_KV_GROUPS. 108 109'kv_metadata': 110 111kv_metadata is used to lookup metadata for each discovered node. Like kv_groups 112above it is used to build a path to lookup in the kv store where it expects to 113find a json dictionary of metadata entries. If found, each key/value pair in the 114dictionary is added to the metadata for the node. eg node 'nyc-web-1' in datacenter 115'nyc-dc1' and kv_metadata = 'ansible/metadata', then the key 116'ansible/metadata/nyc-dc1/nyc-web-1' should contain '{"databse": "postgres"}' 117This can also be set with the environmental variable CONSUL_KV_METADATA. 118 119'availability': 120 121if true then availability groups will be created for each service. The node will 122be added to one of the groups based on the health status of the service. The 123group name is derived from the service name and the configurable availability 124suffixes 125This can also be set with the environmental variable CONSUL_AVAILABILITY. 126 127'available_suffix': 128 129suffix that should be appended to the service availability groups for available 130services e.g. if the suffix is '_up' and the service is nginx, then nodes with 131healthy nginx services will be added to the nginix_up group. Defaults to 132'_available' 133This can also be set with the environmental variable CONSUL_AVAILABLE_SUFFIX. 134 135'unavailable_suffix': 136 137as above but for unhealthy services, defaults to '_unavailable' 138This can also be set with the environmental variable CONSUL_UNAVAILABLE_SUFFIX. 139 140Note that if the inventory discovers an 'ssh' service running on a node it will 141register the port as ansible_ssh_port in the node's metadata and this port will 142be used to access the machine. 143``` 144 145''' 146 147import os 148import re 149import argparse 150import sys 151 152from ansible.module_utils.six.moves import configparser 153 154 155def get_log_filename(): 156 tty_filename = '/dev/tty' 157 stdout_filename = '/dev/stdout' 158 159 if not os.path.exists(tty_filename): 160 return stdout_filename 161 if not os.access(tty_filename, os.W_OK): 162 return stdout_filename 163 if os.getenv('TEAMCITY_VERSION'): 164 return stdout_filename 165 166 return tty_filename 167 168 169def setup_logging(): 170 filename = get_log_filename() 171 172 import logging.config 173 logging.config.dictConfig({ 174 'version': 1, 175 'formatters': { 176 'simple': { 177 'format': '%(asctime)s - %(name)s - %(levelname)s - %(message)s', 178 }, 179 }, 180 'root': { 181 'level': os.getenv('ANSIBLE_INVENTORY_CONSUL_IO_LOG_LEVEL', 'WARN'), 182 'handlers': ['console'], 183 }, 184 'handlers': { 185 'console': { 186 'class': 'logging.FileHandler', 187 'filename': filename, 188 'formatter': 'simple', 189 }, 190 }, 191 'loggers': { 192 'iso8601': { 193 'qualname': 'iso8601', 194 'level': 'INFO', 195 }, 196 }, 197 }) 198 logger = logging.getLogger('consul_io.py') 199 logger.debug('Invoked with %r', sys.argv) 200 201 202if os.getenv('ANSIBLE_INVENTORY_CONSUL_IO_LOG_ENABLED'): 203 setup_logging() 204 205 206import json 207 208try: 209 import consul 210except ImportError as e: 211 sys.exit("""failed=True msg='python-consul required for this module. 212See https://python-consul.readthedocs.io/en/latest/#installation'""") 213 214from ansible.module_utils.six import iteritems 215 216 217class ConsulInventory(object): 218 219 def __init__(self): 220 ''' Create an inventory based on the catalog of nodes and services 221 registered in a consul cluster''' 222 self.node_metadata = {} 223 self.nodes = {} 224 self.nodes_by_service = {} 225 self.nodes_by_tag = {} 226 self.nodes_by_datacenter = {} 227 self.nodes_by_kv = {} 228 self.nodes_by_availability = {} 229 self.current_dc = None 230 self.inmemory_kv = [] 231 self.inmemory_nodes = [] 232 233 config = ConsulConfig() 234 self.config = config 235 236 self.consul_api = config.get_consul_api() 237 238 if config.has_config('datacenter'): 239 if config.has_config('host'): 240 self.load_data_for_node(config.host, config.datacenter) 241 else: 242 self.load_data_for_datacenter(config.datacenter) 243 else: 244 self.load_all_data_consul() 245 246 self.combine_all_results() 247 print(json.dumps(self.inventory, sort_keys=True, indent=2)) 248 249 def bulk_load(self, datacenter): 250 index, groups_list = self.consul_api.kv.get(self.config.kv_groups, recurse=True, dc=datacenter) 251 index, metadata_list = self.consul_api.kv.get(self.config.kv_metadata, recurse=True, dc=datacenter) 252 index, nodes = self.consul_api.catalog.nodes(dc=datacenter) 253 self.inmemory_kv += groups_list 254 self.inmemory_kv += metadata_list 255 self.inmemory_nodes += nodes 256 257 def load_all_data_consul(self): 258 ''' cycle through each of the datacenters in the consul catalog and process 259 the nodes in each ''' 260 self.datacenters = self.consul_api.catalog.datacenters() 261 for datacenter in self.datacenters: 262 self.current_dc = datacenter 263 self.bulk_load(datacenter) 264 self.load_data_for_datacenter(datacenter) 265 266 def load_availability_groups(self, node, datacenter): 267 '''check the health of each service on a node and add the node to either 268 an 'available' or 'unavailable' grouping. The suffix for each group can be 269 controlled from the config''' 270 if self.config.has_config('availability'): 271 for service_name, service in iteritems(node['Services']): 272 for node in self.consul_api.health.service(service_name)[1]: 273 if self.is_service_available(node, service_name): 274 suffix = self.config.get_availability_suffix( 275 'available_suffix', '_available') 276 else: 277 suffix = self.config.get_availability_suffix( 278 'unavailable_suffix', '_unavailable') 279 self.add_node_to_map(self.nodes_by_availability, 280 service_name + suffix, node['Node']) 281 282 def is_service_available(self, node, service_name): 283 '''check the availability of the service on the node beside ensuring the 284 availability of the node itself''' 285 consul_ok = service_ok = False 286 for check in node['Checks']: 287 if check['CheckID'] == 'serfHealth': 288 consul_ok = check['Status'] == 'passing' 289 elif check['ServiceName'] == service_name: 290 service_ok = check['Status'] == 'passing' 291 return consul_ok and service_ok 292 293 def consul_get_kv_inmemory(self, key): 294 result = filter(lambda x: x['Key'] == key, self.inmemory_kv) 295 return result.pop() if result else None 296 297 def consul_get_node_inmemory(self, node): 298 result = filter(lambda x: x['Node'] == node, self.inmemory_nodes) 299 return {"Node": result.pop(), "Services": {}} if result else None 300 301 def load_data_for_datacenter(self, datacenter): 302 '''processes all the nodes in a particular datacenter''' 303 if self.config.bulk_load == 'true': 304 nodes = self.inmemory_nodes 305 else: 306 index, nodes = self.consul_api.catalog.nodes(dc=datacenter) 307 for node in nodes: 308 self.add_node_to_map(self.nodes_by_datacenter, datacenter, node) 309 self.load_data_for_node(node['Node'], datacenter) 310 311 def load_data_for_node(self, node, datacenter): 312 '''loads the data for a single node adding it to various groups based on 313 metadata retrieved from the kv store and service availability''' 314 315 if self.config.suffixes == 'true': 316 index, node_data = self.consul_api.catalog.node(node, dc=datacenter) 317 else: 318 node_data = self.consul_get_node_inmemory(node) 319 node = node_data['Node'] 320 321 self.add_node_to_map(self.nodes, 'all', node) 322 self.add_metadata(node_data, "consul_datacenter", datacenter) 323 self.add_metadata(node_data, "consul_nodename", node['Node']) 324 325 self.load_groups_from_kv(node_data) 326 self.load_node_metadata_from_kv(node_data) 327 if self.config.suffixes == 'true': 328 self.load_availability_groups(node_data, datacenter) 329 for name, service in node_data['Services'].items(): 330 self.load_data_from_service(name, service, node_data) 331 332 def load_node_metadata_from_kv(self, node_data): 333 ''' load the json dict at the metadata path defined by the kv_metadata value 334 and the node name add each entry in the dictionary to the node's 335 metadata ''' 336 node = node_data['Node'] 337 if self.config.has_config('kv_metadata'): 338 key = "%s/%s/%s" % (self.config.kv_metadata, self.current_dc, node['Node']) 339 if self.config.bulk_load == 'true': 340 metadata = self.consul_get_kv_inmemory(key) 341 else: 342 index, metadata = self.consul_api.kv.get(key) 343 if metadata and metadata['Value']: 344 try: 345 metadata = json.loads(metadata['Value']) 346 for k, v in metadata.items(): 347 self.add_metadata(node_data, k, v) 348 except Exception: 349 pass 350 351 def load_groups_from_kv(self, node_data): 352 ''' load the comma separated list of groups at the path defined by the 353 kv_groups config value and the node name add the node address to each 354 group found ''' 355 node = node_data['Node'] 356 if self.config.has_config('kv_groups'): 357 key = "%s/%s/%s" % (self.config.kv_groups, self.current_dc, node['Node']) 358 if self.config.bulk_load == 'true': 359 groups = self.consul_get_kv_inmemory(key) 360 else: 361 index, groups = self.consul_api.kv.get(key) 362 if groups and groups['Value']: 363 for group in groups['Value'].decode().split(','): 364 self.add_node_to_map(self.nodes_by_kv, group.strip(), node) 365 366 def load_data_from_service(self, service_name, service, node_data): 367 '''process a service registered on a node, adding the node to a group with 368 the service name. Each service tag is extracted and the node is added to a 369 tag grouping also''' 370 self.add_metadata(node_data, "consul_services", service_name, True) 371 372 if self.is_service("ssh", service_name): 373 self.add_metadata(node_data, "ansible_ssh_port", service['Port']) 374 375 if self.config.has_config('servers_suffix'): 376 service_name = service_name + self.config.servers_suffix 377 378 self.add_node_to_map(self.nodes_by_service, service_name, node_data['Node']) 379 self.extract_groups_from_tags(service_name, service, node_data) 380 381 def is_service(self, target, name): 382 return name and (name.lower() == target.lower()) 383 384 def extract_groups_from_tags(self, service_name, service, node_data): 385 '''iterates each service tag and adds the node to groups derived from the 386 service and tag names e.g. nginx_master''' 387 if self.config.has_config('tags') and service['Tags']: 388 tags = service['Tags'] 389 self.add_metadata(node_data, "consul_%s_tags" % service_name, tags) 390 for tag in service['Tags']: 391 tagname = service_name + '_' + tag 392 self.add_node_to_map(self.nodes_by_tag, tagname, node_data['Node']) 393 394 def combine_all_results(self): 395 '''prunes and sorts all groupings for combination into the final map''' 396 self.inventory = {"_meta": {"hostvars": self.node_metadata}} 397 groupings = [self.nodes, self.nodes_by_datacenter, self.nodes_by_service, 398 self.nodes_by_tag, self.nodes_by_kv, self.nodes_by_availability] 399 for grouping in groupings: 400 for name, addresses in grouping.items(): 401 self.inventory[name] = sorted(list(set(addresses))) 402 403 def add_metadata(self, node_data, key, value, is_list=False): 404 ''' Pushed an element onto a metadata dict for the node, creating 405 the dict if it doesn't exist ''' 406 key = self.to_safe(key) 407 node = self.get_inventory_name(node_data['Node']) 408 409 if node in self.node_metadata: 410 metadata = self.node_metadata[node] 411 else: 412 metadata = {} 413 self.node_metadata[node] = metadata 414 if is_list: 415 self.push(metadata, key, value) 416 else: 417 metadata[key] = value 418 419 def get_inventory_name(self, node_data): 420 '''return the ip or a node name that can be looked up in consul's dns''' 421 domain = self.config.domain 422 if domain: 423 node_name = node_data['Node'] 424 if self.current_dc: 425 return '%s.node.%s.%s' % (node_name, self.current_dc, domain) 426 else: 427 return '%s.node.%s' % (node_name, domain) 428 else: 429 return node_data['Address'] 430 431 def add_node_to_map(self, map, name, node): 432 self.push(map, name, self.get_inventory_name(node)) 433 434 def push(self, my_dict, key, element): 435 ''' Pushed an element onto an array that may not have been defined in the 436 dict ''' 437 key = self.to_safe(key) 438 if key in my_dict: 439 my_dict[key].append(element) 440 else: 441 my_dict[key] = [element] 442 443 def to_safe(self, word): 444 ''' Converts 'bad' characters in a string to underscores so they can be used 445 as Ansible groups ''' 446 return re.sub(r'[^A-Za-z0-9\-\.]', '_', word) 447 448 def sanitize_dict(self, d): 449 450 new_dict = {} 451 for k, v in d.items(): 452 if v is not None: 453 new_dict[self.to_safe(str(k))] = self.to_safe(str(v)) 454 return new_dict 455 456 def sanitize_list(self, seq): 457 new_seq = [] 458 for d in seq: 459 new_seq.append(self.sanitize_dict(d)) 460 return new_seq 461 462 463class ConsulConfig(dict): 464 465 def __init__(self): 466 self.read_settings() 467 self.read_cli_args() 468 self.read_env_vars() 469 470 def has_config(self, name): 471 if hasattr(self, name): 472 return getattr(self, name) 473 else: 474 return False 475 476 def read_settings(self): 477 ''' Reads the settings from the consul_io.ini file (or consul.ini for backwards compatibility)''' 478 config = configparser.SafeConfigParser() 479 if os.path.isfile(os.path.dirname(os.path.realpath(__file__)) + '/consul_io.ini'): 480 config.read(os.path.dirname(os.path.realpath(__file__)) + '/consul_io.ini') 481 else: 482 config.read(os.path.dirname(os.path.realpath(__file__)) + '/consul.ini') 483 484 config_options = ['host', 'token', 'datacenter', 'servers_suffix', 485 'tags', 'kv_metadata', 'kv_groups', 'availability', 486 'unavailable_suffix', 'available_suffix', 'url', 487 'domain', 'suffixes', 'bulk_load'] 488 for option in config_options: 489 value = None 490 if config.has_option('consul', option): 491 value = config.get('consul', option).lower() 492 setattr(self, option, value) 493 494 def read_cli_args(self): 495 ''' Command line argument processing ''' 496 parser = argparse.ArgumentParser(description='Produce an Ansible Inventory file based nodes in a Consul cluster') 497 498 parser.add_argument('--list', action='store_true', 499 help='Get all inventory variables from all nodes in the consul cluster') 500 parser.add_argument('--host', action='store', 501 help='Get all inventory variables about a specific consul node,' 502 'requires datacenter set in consul.ini.') 503 parser.add_argument('--datacenter', action='store', 504 help='Get all inventory about a specific consul datacenter') 505 506 args = parser.parse_args() 507 arg_names = ['host', 'datacenter'] 508 509 for arg in arg_names: 510 if getattr(args, arg): 511 setattr(self, arg, getattr(args, arg)) 512 513 def read_env_vars(self): 514 env_var_options = ['host', 'token', 'datacenter', 'servers_suffix', 515 'tags', 'kv_metadata', 'kv_groups', 'availability', 516 'unavailable_suffix', 'available_suffix', 'url', 517 'domain', 'suffixes', 'bulk_load'] 518 for option in env_var_options: 519 value = None 520 env_var = 'CONSUL_' + option.upper() 521 if os.environ.get(env_var): 522 setattr(self, option, os.environ.get(env_var)) 523 524 def get_availability_suffix(self, suffix, default): 525 if self.has_config(suffix): 526 return self.has_config(suffix) 527 return default 528 529 def get_consul_api(self): 530 '''get an instance of the api based on the supplied configuration''' 531 host = 'localhost' 532 port = 8500 533 token = None 534 scheme = 'http' 535 536 if hasattr(self, 'url'): 537 from ansible.module_utils.six.moves.urllib.parse import urlparse 538 o = urlparse(self.url) 539 if o.hostname: 540 host = o.hostname 541 if o.port: 542 port = o.port 543 if o.scheme: 544 scheme = o.scheme 545 546 if hasattr(self, 'token'): 547 token = self.token 548 if not token: 549 token = 'anonymous' 550 return consul.Consul(host=host, port=port, token=token, scheme=scheme) 551 552 553ConsulInventory() 554