1# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); you may 4# not use this file except in compliance with the License. You may obtain 5# a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12# License for the specific language governing permissions and limitations 13# under the License. 14 15import copy 16import os.path 17import urllib 18import warnings 19 20try: 21 import keyring 22except ImportError: 23 keyring = None 24 25from keystoneauth1 import discover 26import keystoneauth1.exceptions.catalog 27from keystoneauth1.loading import adapter as ks_load_adap 28from keystoneauth1 import session as ks_session 29import os_service_types 30import requestsexceptions 31try: 32 import statsd 33except ImportError: 34 statsd = None 35try: 36 import prometheus_client 37except ImportError: 38 prometheus_client = None 39try: 40 import influxdb 41except ImportError: 42 influxdb = None 43 44from openstack import _log 45from openstack.config import _util 46from openstack.config import defaults as config_defaults 47from openstack import exceptions 48from openstack import proxy 49from openstack import version as openstack_version 50 51 52_logger = _log.setup_logging('openstack') 53 54SCOPE_KEYS = { 55 'domain_id', 'domain_name', 56 'project_id', 'project_name', 57 'system_scope' 58} 59 60# Sentinel for nonexistence 61_ENOENT = object() 62 63 64def _make_key(key, service_type): 65 if not service_type: 66 return key 67 else: 68 service_type = service_type.lower().replace('-', '_') 69 return "_".join([service_type, key]) 70 71 72def _disable_service(config, service_type, reason=None): 73 service_type = service_type.lower().replace('-', '_') 74 key = 'has_{service_type}'.format(service_type=service_type) 75 config[key] = False 76 if reason: 77 d_key = _make_key('disabled_reason', service_type) 78 config[d_key] = reason 79 80 81def _get_implied_microversion(version): 82 if not version: 83 return 84 if '.' in version: 85 # Some services historically had a .0 in their normal api version. 86 # Neutron springs to mind with version "2.0". If a user has "2.0" 87 # set in a variable or config file just because history, we don't 88 # need to send any microversion headers. 89 if version.split('.')[1] != "0": 90 return version 91 92 93def from_session(session, name=None, region_name=None, 94 force_ipv4=False, 95 app_name=None, app_version=None, **kwargs): 96 """Construct a CloudRegion from an existing `keystoneauth1.session.Session` 97 98 When a Session already exists, we don't actually even need to go through 99 the OpenStackConfig.get_one_cloud dance. We have a Session with Auth info. 100 The only parameters that are really needed are adapter/catalog related. 101 102 :param keystoneauth1.session.session session: 103 An existing authenticated Session to use. 104 :param str name: 105 A name to use for this cloud region in logging. If left empty, the 106 hostname of the auth_url found in the Session will be used. 107 :param str region_name: 108 The region name to connect to. 109 :param bool force_ipv4: 110 Whether or not to disable IPv6 support. Defaults to False. 111 :param str app_name: 112 Name of the application to be added to User Agent. 113 :param str app_version: 114 Version of the application to be added to User Agent. 115 :param kwargs: 116 Config settings for this cloud region. 117 """ 118 config_dict = config_defaults.get_defaults() 119 config_dict.update(**kwargs) 120 return CloudRegion( 121 name=name, session=session, config=config_dict, 122 region_name=region_name, force_ipv4=force_ipv4, 123 app_name=app_name, app_version=app_version) 124 125 126def from_conf(conf, session=None, service_types=None, **kwargs): 127 """Create a CloudRegion from oslo.config ConfigOpts. 128 129 :param oslo_config.cfg.ConfigOpts conf: 130 An oslo.config ConfigOpts containing keystoneauth1.Adapter options in 131 sections named according to project (e.g. [nova], not [compute]). 132 TODO: Current behavior is to use defaults if no such section exists, 133 which may not be what we want long term. 134 :param keystoneauth1.session.Session session: 135 An existing authenticated Session to use. This is currently required. 136 TODO: Load this (and auth) from the conf. 137 :param service_types: 138 A list/set of service types for which to look for and process config 139 opts. If None, all known service types are processed. Note that we will 140 not error if a supplied service type can not be processed successfully 141 (unless you try to use the proxy, of course). This tolerates uses where 142 the consuming code has paths for a given service, but those paths are 143 not exercised for given end user setups, and we do not want to generate 144 errors for e.g. missing/invalid conf sections in those cases. We also 145 don't check to make sure your service types are spelled correctly - 146 caveat implementor. 147 :param kwargs: 148 Additional keyword arguments to be passed directly to the CloudRegion 149 constructor. 150 :raise openstack.exceptions.ConfigException: 151 If session is not specified. 152 :return: 153 An openstack.config.cloud_region.CloudRegion. 154 """ 155 if not session: 156 # TODO(mordred) Fill this in - not needed for first stab with nova 157 raise exceptions.ConfigException("A Session must be supplied.") 158 config_dict = kwargs.pop('config', config_defaults.get_defaults()) 159 stm = os_service_types.ServiceTypes() 160 for st in stm.all_types_by_service_type: 161 if service_types is not None and st not in service_types: 162 _disable_service( 163 config_dict, st, 164 reason="Not in the list of requested service_types.") 165 continue 166 project_name = stm.get_project_name(st) 167 if project_name not in conf: 168 if '-' in project_name: 169 project_name = project_name.replace('-', '_') 170 171 if project_name not in conf: 172 _disable_service( 173 config_dict, st, 174 reason="No section for project '{project}' (service type " 175 "'{service_type}') was present in the config." 176 .format(project=project_name, service_type=st)) 177 continue 178 opt_dict = {} 179 # Populate opt_dict with (appropriately processed) Adapter conf opts 180 try: 181 ks_load_adap.process_conf_options(conf[project_name], opt_dict) 182 except Exception as e: 183 # NOTE(efried): This is for (at least) a couple of scenarios: 184 # (1) oslo_config.cfg.NoSuchOptError when ksa adapter opts are not 185 # registered in this section. 186 # (2) TypeError, when opts are registered but bogus (e.g. 187 # 'interface' and 'valid_interfaces' are both present). 188 # We may want to consider (providing a kwarg giving the caller the 189 # option of) blowing up right away for (2) rather than letting them 190 # get all the way to the point of trying the service and having 191 # *that* blow up. 192 reason = ("Encountered an exception attempting to process config " 193 "for project '{project}' (service type " 194 "'{service_type}'): {exception}".format( 195 project=project_name, service_type=st, exception=e)) 196 _logger.warning("Disabling service '{service_type}': " 197 "{reason}".format(service_type=st, reason=reason)) 198 _disable_service(config_dict, st, reason=reason) 199 continue 200 # Load them into config_dict under keys prefixed by ${service_type}_ 201 for raw_name, opt_val in opt_dict.items(): 202 config_name = _make_key(raw_name, st) 203 config_dict[config_name] = opt_val 204 return CloudRegion( 205 session=session, config=config_dict, **kwargs) 206 207 208class CloudRegion: 209 # TODO(efried): Doc the rest of the kwargs 210 """The configuration for a Region of an OpenStack Cloud. 211 212 A CloudRegion encapsulates the config information needed for connections 213 to all of the services in a Region of a Cloud. 214 215 :param str region_name: 216 The default region name for all services in this CloudRegion. If 217 both ``region_name`` and ``config['region_name'] are specified, the 218 kwarg takes precedence. May be overridden for a given ${service} 219 via a ${service}_region_name key in the ``config`` dict. 220 :param dict config: 221 A dict of configuration values for the CloudRegion and its 222 services. The key for a ${config_option} for a specific ${service} 223 should be ${service}_${config_option}. For example, to configure 224 the endpoint_override for the block_storage service, the ``config`` 225 dict should contain:: 226 227 'block_storage_endpoint_override': 'http://...' 228 229 To provide a default to be used if no service-specific override is 230 present, just use the unprefixed ${config_option} as the service 231 key, e.g.:: 232 233 'interface': 'public' 234 """ 235 def __init__(self, name=None, region_name=None, config=None, 236 force_ipv4=False, auth_plugin=None, 237 openstack_config=None, session_constructor=None, 238 app_name=None, app_version=None, session=None, 239 discovery_cache=None, extra_config=None, 240 cache_expiration_time=0, cache_expirations=None, 241 cache_path=None, cache_class='dogpile.cache.null', 242 cache_arguments=None, password_callback=None, 243 statsd_host=None, statsd_port=None, statsd_prefix=None, 244 influxdb_config=None, 245 collector_registry=None, 246 cache_auth=False): 247 self._name = name 248 self.config = _util.normalize_keys(config) 249 # NOTE(efried): For backward compatibility: a) continue to accept the 250 # region_name kwarg; b) make it take precedence over (non-service_type- 251 # specific) region_name set in the config dict. 252 if region_name is not None: 253 self.config['region_name'] = region_name 254 self._extra_config = extra_config or {} 255 self.log = _log.setup_logging('openstack.config') 256 self._force_ipv4 = force_ipv4 257 self._auth = auth_plugin 258 self._cache_auth = cache_auth 259 self.load_auth_from_cache() 260 self._openstack_config = openstack_config 261 self._keystone_session = session 262 self._session_constructor = session_constructor or ks_session.Session 263 self._app_name = app_name 264 self._app_version = app_version 265 self._discovery_cache = discovery_cache or None 266 self._cache_expiration_time = cache_expiration_time 267 self._cache_expirations = cache_expirations or {} 268 self._cache_path = cache_path 269 self._cache_class = cache_class 270 self._cache_arguments = cache_arguments 271 self._password_callback = password_callback 272 self._statsd_host = statsd_host 273 self._statsd_port = statsd_port 274 self._statsd_prefix = statsd_prefix 275 self._statsd_client = None 276 self._influxdb_config = influxdb_config 277 self._influxdb_client = None 278 self._collector_registry = collector_registry 279 280 self._service_type_manager = os_service_types.ServiceTypes() 281 282 def __getattr__(self, key): 283 """Return arbitrary attributes.""" 284 285 if key.startswith('os_'): 286 key = key[3:] 287 288 if key in [attr.replace('-', '_') for attr in self.config]: 289 return self.config[key] 290 else: 291 return None 292 293 def __iter__(self): 294 return self.config.__iter__() 295 296 def __eq__(self, other): 297 return ( 298 self.name == other.name 299 and self.config == other.config) 300 301 def __ne__(self, other): 302 return not self == other 303 304 @property 305 def name(self): 306 if self._name is None: 307 try: 308 self._name = urllib.parse.urlparse( 309 self.get_session().auth.auth_url).hostname 310 except Exception: 311 self._name = self._app_name or '' 312 return self._name 313 314 @property 315 def full_name(self): 316 """Return a string that can be used as an identifier. 317 318 Always returns a valid string. It will have name and region_name 319 or just one of the two if only one is set, or else 'unknown'. 320 """ 321 region_name = self.get_region_name() 322 if self.name and region_name: 323 return ":".join([self.name, region_name]) 324 elif self.name and not region_name: 325 return self.name 326 elif not self.name and region_name: 327 return region_name 328 else: 329 return 'unknown' 330 331 def set_service_value(self, key, service_type, value): 332 key = _make_key(key, service_type) 333 self.config[key] = value 334 335 def set_session_constructor(self, session_constructor): 336 """Sets the Session constructor.""" 337 self._session_constructor = session_constructor 338 339 def get_requests_verify_args(self): 340 """Return the verify and cert values for the requests library.""" 341 insecure = self.config.get('insecure', False) 342 verify = self.config.get('verify', True) 343 cacert = self.config.get('cacert') 344 # Insecure is the most aggressive setting, so it wins 345 if insecure: 346 verify = False 347 if verify and cacert: 348 verify = os.path.expanduser(cacert) 349 else: 350 if cacert: 351 warnings.warn( 352 "You are specifying a cacert for the cloud {full_name}" 353 " but also to ignore the host verification. The host SSL" 354 " cert will not be verified.".format( 355 full_name=self.full_name)) 356 357 cert = self.config.get('cert') 358 if cert: 359 cert = os.path.expanduser(cert) 360 if self.config.get('key'): 361 cert = (cert, os.path.expanduser(self.config.get('key'))) 362 return (verify, cert) 363 364 def get_services(self): 365 """Return a list of service types we know something about.""" 366 services = [] 367 for key, val in self.config.items(): 368 if (key.endswith('api_version') 369 or key.endswith('service_type') 370 or key.endswith('service_name')): 371 services.append("_".join(key.split('_')[:-2])) 372 return list(set(services)) 373 374 def get_enabled_services(self): 375 services = set() 376 377 all_services = [k['service_type'] for k in 378 self._service_type_manager.services] 379 all_services.extend(k[4:] for k in 380 self.config.keys() if k.startswith('has_')) 381 382 for srv in all_services: 383 ep = self.get_endpoint_from_catalog(srv) 384 if ep: 385 services.add(srv.replace('-', '_')) 386 387 return services 388 389 def get_auth_args(self): 390 return self.config.get('auth', {}) 391 392 def _get_config( 393 self, key, service_type, 394 default=None, 395 fallback_to_unprefixed=False, 396 converter=None): 397 '''Get a config value for a service_type. 398 399 Finds the config value for a key, looking first for it prefixed by 400 the given service_type, then by any known aliases of that service_type. 401 Finally, if fallback_to_unprefixed is True, a value will be looked 402 for without a prefix to support the config values where a global 403 default makes sense. 404 405 For instance, ``_get_config('example', 'block-storage', True)`` would 406 first look for ``block_storage_example``, then ``volumev3_example``, 407 ``volumev2_example`` and ``volume_example``. If no value was found, it 408 would look for ``example``. 409 410 If none of that works, it returns the value in ``default``. 411 ''' 412 if service_type is None: 413 return self.config.get(key) 414 415 for st in self._service_type_manager.get_all_types(service_type): 416 value = self.config.get(_make_key(key, st)) 417 if value is not None: 418 break 419 else: 420 if fallback_to_unprefixed: 421 value = self.config.get(key) 422 423 if value is None: 424 return default 425 else: 426 if converter is not None: 427 value = converter(value) 428 return value 429 430 def _get_service_config(self, key, service_type): 431 config_dict = self.config.get(key) 432 if not config_dict: 433 return None 434 if not isinstance(config_dict, dict): 435 return config_dict 436 437 for st in self._service_type_manager.get_all_types(service_type): 438 if st in config_dict: 439 return config_dict[st] 440 441 def get_region_name(self, service_type=None): 442 # If a region_name for the specific service_type is configured, use it; 443 # else use the one configured for the CloudRegion as a whole. 444 return self._get_config( 445 'region_name', service_type, fallback_to_unprefixed=True) 446 447 def get_interface(self, service_type=None): 448 return self._get_config( 449 'interface', service_type, fallback_to_unprefixed=True) 450 451 def get_api_version(self, service_type): 452 version = self._get_config('api_version', service_type) 453 if version: 454 try: 455 float(version) 456 except ValueError: 457 if 'latest' in version: 458 warnings.warn( 459 "You have a configured API_VERSION with 'latest' in" 460 " it. In the context of openstacksdk this doesn't make" 461 " any sense.") 462 return None 463 return version 464 465 def get_default_microversion(self, service_type): 466 return self._get_config('default_microversion', service_type) 467 468 def get_service_type(self, service_type): 469 # People requesting 'volume' are doing so because os-client-config 470 # let them. What they want is block-storage, not explicitly the 471 # v1 of cinder. If someone actually wants v1, they'll have api_version 472 # set to 1, in which case block-storage will still work properly. 473 # Use service-types-manager to grab the official type name. _get_config 474 # will still look for config by alias, but starting with the official 475 # type will get us things in the right order. 476 if self._service_type_manager.is_known(service_type): 477 service_type = self._service_type_manager.get_service_type( 478 service_type) 479 return self._get_config( 480 'service_type', service_type, default=service_type) 481 482 def get_service_name(self, service_type): 483 return self._get_config('service_name', service_type) 484 485 def get_endpoint(self, service_type): 486 auth = self.config.get('auth', {}) 487 value = self._get_config('endpoint_override', service_type) 488 if not value: 489 value = self._get_config('endpoint', service_type) 490 if not value and self.config.get('auth_type') == 'none': 491 # If endpoint is given and we're using the none auth type, 492 # then the endpoint value is the endpoint_override for every 493 # service. 494 value = auth.get('endpoint') 495 if (not value and service_type == 'identity' 496 and SCOPE_KEYS.isdisjoint(set(auth.keys()))): 497 # There are a small number of unscoped identity operations. 498 # Specifically, looking up a list of projects/domains/system to 499 # scope to. 500 value = auth.get('auth_url') 501 # Because of course. Seriously. 502 # We have to override the Rackspace block-storage endpoint because 503 # only v1 is in the catalog but the service actually does support 504 # v2. But the endpoint needs the project_id. 505 service_type = self._service_type_manager.get_service_type( 506 service_type) 507 if ( 508 value 509 and self.config.get('profile') == 'rackspace' 510 and service_type == 'block-storage' 511 ): 512 value = value + auth.get('project_id') 513 return value 514 515 def get_endpoint_from_catalog( 516 self, service_type, interface=None, region_name=None): 517 """Return the endpoint for a given service as found in the catalog. 518 519 For values respecting endpoint overrides, see 520 :meth:`~openstack.connection.Connection.endpoint_for` 521 522 :param service_type: Service Type of the endpoint to search for. 523 :param interface: 524 Interface of the endpoint to search for. Optional, defaults to 525 the configured value for interface for this Connection. 526 :param region_name: 527 Region Name of the endpoint to search for. Optional, defaults to 528 the configured value for region_name for this Connection. 529 530 :returns: The endpoint of the service, or None if not found. 531 """ 532 interface = interface or self.get_interface(service_type) 533 region_name = region_name or self.get_region_name(service_type) 534 session = self.get_session() 535 catalog = session.auth.get_access(session).service_catalog 536 try: 537 return catalog.url_for( 538 service_type=service_type, 539 interface=interface, 540 region_name=region_name) 541 except keystoneauth1.exceptions.catalog.EndpointNotFound: 542 return None 543 544 def get_connect_retries(self, service_type): 545 return self._get_config('connect_retries', service_type, 546 fallback_to_unprefixed=True, 547 converter=int) 548 549 def get_status_code_retries(self, service_type): 550 return self._get_config('status_code_retries', service_type, 551 fallback_to_unprefixed=True, 552 converter=int) 553 554 @property 555 def prefer_ipv6(self): 556 return not self._force_ipv4 557 558 @property 559 def force_ipv4(self): 560 return self._force_ipv4 561 562 def get_auth(self): 563 """Return a keystoneauth plugin from the auth credentials.""" 564 return self._auth 565 566 def skip_auth_cache(self): 567 return not keyring or not self._auth or not self._cache_auth 568 569 def load_auth_from_cache(self): 570 if self.skip_auth_cache(): 571 return 572 573 cache_id = self._auth.get_cache_id() 574 575 # skip if the plugin does not support caching 576 if not cache_id: 577 return 578 579 try: 580 state = keyring.get_password('openstacksdk', cache_id) 581 except RuntimeError: # the fail backend raises this 582 self.log.debug('Failed to fetch auth from keyring') 583 return 584 585 self.log.debug('Reusing authentication from keyring') 586 self._auth.set_auth_state(state) 587 588 def set_auth_cache(self): 589 if self.skip_auth_cache(): 590 return 591 592 cache_id = self._auth.get_cache_id() 593 state = self._auth.get_auth_state() 594 595 try: 596 keyring.set_password('openstacksdk', cache_id, state) 597 except RuntimeError: # the fail backend raises this 598 self.log.debug('Failed to set auth into keyring') 599 600 def insert_user_agent(self): 601 """Set sdk information into the user agent of the Session. 602 603 .. warning:: 604 This method is here to be used by os-client-config. It exists 605 as a hook point so that os-client-config can provice backwards 606 compatibility and still be in the User Agent for people using 607 os-client-config directly. 608 609 Normal consumers of SDK should use app_name and app_version. However, 610 if someone else writes a subclass of 611 :class:`~openstack.config.cloud_region.CloudRegion` it may be 612 desirable. 613 """ 614 self._keystone_session.additional_user_agent.append( 615 ('openstacksdk', openstack_version.__version__)) 616 617 def get_session(self): 618 """Return a keystoneauth session based on the auth credentials.""" 619 if self._keystone_session is None: 620 if not self._auth: 621 raise exceptions.ConfigException( 622 "Problem with auth parameters") 623 (verify, cert) = self.get_requests_verify_args() 624 # Turn off urllib3 warnings about insecure certs if we have 625 # explicitly configured requests to tell it we do not want 626 # cert verification 627 if not verify: 628 self.log.debug( 629 "Turning off SSL warnings for {full_name}" 630 " since verify=False".format(full_name=self.full_name)) 631 requestsexceptions.squelch_warnings(insecure_requests=not verify) 632 self._keystone_session = self._session_constructor( 633 auth=self._auth, 634 verify=verify, 635 cert=cert, 636 timeout=self.config.get('api_timeout'), 637 collect_timing=self.config.get('timing'), 638 discovery_cache=self._discovery_cache) 639 self.insert_user_agent() 640 # Using old keystoneauth with new os-client-config fails if 641 # we pass in app_name and app_version. Those are not essential, 642 # nor a reason to bump our minimum, so just test for the session 643 # having the attribute post creation and set them then. 644 if hasattr(self._keystone_session, 'app_name'): 645 self._keystone_session.app_name = self._app_name 646 if hasattr(self._keystone_session, 'app_version'): 647 self._keystone_session.app_version = self._app_version 648 return self._keystone_session 649 650 def get_service_catalog(self): 651 """Helper method to grab the service catalog.""" 652 return self._auth.get_access(self.get_session()).service_catalog 653 654 def _get_version_request(self, service_type, version): 655 """Translate OCC version args to those needed by ksa adapter. 656 657 If no version is requested explicitly and we have a configured version, 658 set the version parameter and let ksa deal with expanding that to 659 min=ver.0, max=ver.latest. 660 661 If version is set, pass it through. 662 663 If version is not set and we don't have a configured version, default 664 to latest. 665 666 If version is set, contains a '.', and default_microversion is not 667 set, also pass it as a default microversion. 668 """ 669 version_request = _util.VersionRequest() 670 if version == 'latest': 671 version_request.max_api_version = 'latest' 672 return version_request 673 674 if not version: 675 version = self.get_api_version(service_type) 676 677 # Octavia doens't have a version discovery document. Hard-code an 678 # exception to this logic for now. 679 if not version and service_type not in ('load-balancer',): 680 version_request.max_api_version = 'latest' 681 else: 682 version_request.version = version 683 684 default_microversion = self.get_default_microversion(service_type) 685 implied_microversion = _get_implied_microversion(version) 686 if (implied_microversion and default_microversion 687 and implied_microversion != default_microversion): 688 raise exceptions.ConfigException( 689 "default_microversion of {default_microversion} was given" 690 " for {service_type}, but api_version looks like a" 691 " microversion as well. Please set api_version to just the" 692 " desired major version, or omit default_microversion".format( 693 default_microversion=default_microversion, 694 service_type=service_type)) 695 if implied_microversion: 696 default_microversion = implied_microversion 697 # If we're inferring a microversion, don't pass the whole 698 # string in as api_version, since that tells keystoneauth 699 # we're looking for a major api version. 700 version_request.version = version[0] 701 702 version_request.default_microversion = default_microversion 703 704 return version_request 705 706 def get_all_version_data(self, service_type): 707 # Seriously. Don't think about the existential crisis 708 # that is the next line. You'll wind up in cthulhu's lair. 709 service_type = self.get_service_type(service_type) 710 region_name = self.get_region_name(service_type) 711 versions = self.get_session().get_all_version_data( 712 service_type=service_type, 713 interface=self.get_interface(service_type), 714 region_name=region_name, 715 ) 716 region_versions = versions.get(region_name, {}) 717 interface_versions = region_versions.get( 718 self.get_interface(service_type), {}) 719 return interface_versions.get(service_type, []) 720 721 def _get_endpoint_from_catalog(self, service_type, constructor): 722 adapter = constructor( 723 session=self.get_session(), 724 service_type=self.get_service_type(service_type), 725 service_name=self.get_service_name(service_type), 726 interface=self.get_interface(service_type), 727 region_name=self.get_region_name(service_type), 728 ) 729 return adapter.get_endpoint() 730 731 def _get_hardcoded_endpoint(self, service_type, constructor): 732 endpoint = self._get_endpoint_from_catalog( 733 service_type, constructor) 734 if not endpoint.rstrip().rsplit('/')[-1] == 'v2.0': 735 if not endpoint.endswith('/'): 736 endpoint += '/' 737 endpoint = urllib.parse.urljoin(endpoint, 'v2.0') 738 return endpoint 739 740 def get_session_client( 741 self, service_type, version=None, 742 constructor=proxy.Proxy, 743 **kwargs): 744 """Return a prepped keystoneauth Adapter for a given service. 745 746 This is useful for making direct requests calls against a 747 'mounted' endpoint. That is, if you do: 748 749 client = get_session_client('compute') 750 751 then you can do: 752 753 client.get('/flavors') 754 755 and it will work like you think. 756 """ 757 version_request = self._get_version_request(service_type, version) 758 759 kwargs.setdefault('region_name', self.get_region_name(service_type)) 760 kwargs.setdefault('connect_retries', 761 self.get_connect_retries(service_type)) 762 kwargs.setdefault('status_code_retries', 763 self.get_status_code_retries(service_type)) 764 kwargs.setdefault('statsd_prefix', self.get_statsd_prefix()) 765 kwargs.setdefault('statsd_client', self.get_statsd_client()) 766 kwargs.setdefault('prometheus_counter', self.get_prometheus_counter()) 767 kwargs.setdefault( 768 'prometheus_histogram', self.get_prometheus_histogram()) 769 kwargs.setdefault('influxdb_config', self._influxdb_config) 770 kwargs.setdefault('influxdb_client', self.get_influxdb_client()) 771 endpoint_override = self.get_endpoint(service_type) 772 version = version_request.version 773 min_api_version = ( 774 kwargs.pop('min_version', None) or version_request.min_api_version) 775 max_api_version = ( 776 kwargs.pop('max_version', None) or version_request.max_api_version) 777 778 # Older neutron has inaccessible discovery document. Nobody noticed 779 # because neutronclient hard-codes an append of v2.0. YAY! 780 # Also, older octavia has a similar issue. 781 if service_type in ('network', 'load-balancer'): 782 version = None 783 min_api_version = None 784 max_api_version = None 785 if endpoint_override is None: 786 endpoint_override = self._get_hardcoded_endpoint( 787 service_type, constructor) 788 789 client = constructor( 790 session=self.get_session(), 791 service_type=self.get_service_type(service_type), 792 service_name=self.get_service_name(service_type), 793 interface=self.get_interface(service_type), 794 version=version, 795 min_version=min_api_version, 796 max_version=max_api_version, 797 endpoint_override=endpoint_override, 798 default_microversion=version_request.default_microversion, 799 rate_limit=self.get_rate_limit(service_type), 800 concurrency=self.get_concurrency(service_type), 801 **kwargs) 802 if version_request.default_microversion: 803 default_microversion = version_request.default_microversion 804 info = client.get_endpoint_data() 805 if not discover.version_between( 806 info.min_microversion, 807 info.max_microversion, 808 default_microversion 809 ): 810 if self.get_default_microversion(service_type): 811 raise exceptions.ConfigException( 812 "A default microversion for service {service_type} of" 813 " {default_microversion} was requested, but the cloud" 814 " only supports a minimum of {min_microversion} and" 815 " a maximum of {max_microversion}.".format( 816 service_type=service_type, 817 default_microversion=default_microversion, 818 min_microversion=discover.version_to_string( 819 info.min_microversion), 820 max_microversion=discover.version_to_string( 821 info.max_microversion))) 822 else: 823 raise exceptions.ConfigException( 824 "A default microversion for service {service_type} of" 825 " {default_microversion} was requested, but the cloud" 826 " only supports a minimum of {min_microversion} and" 827 " a maximum of {max_microversion}. The default" 828 " microversion was set because a microversion" 829 " formatted version string, '{api_version}', was" 830 " passed for the api_version of the service. If it" 831 " was not intended to set a default microversion" 832 " please remove anything other than an integer major" 833 " version from the version setting for" 834 " the service.".format( 835 service_type=service_type, 836 api_version=self.get_api_version(service_type), 837 default_microversion=default_microversion, 838 min_microversion=discover.version_to_string( 839 info.min_microversion), 840 max_microversion=discover.version_to_string( 841 info.max_microversion))) 842 return client 843 844 def get_session_endpoint( 845 self, service_type, min_version=None, max_version=None): 846 """Return the endpoint from config or the catalog. 847 848 If a configuration lists an explicit endpoint for a service, 849 return that. Otherwise, fetch the service catalog from the 850 keystone session and return the appropriate endpoint. 851 852 :param service_type: Official service type of service 853 """ 854 855 override_endpoint = self.get_endpoint(service_type) 856 if override_endpoint: 857 return override_endpoint 858 859 region_name = self.get_region_name(service_type) 860 service_name = self.get_service_name(service_type) 861 interface = self.get_interface(service_type) 862 session = self.get_session() 863 # Do this as kwargs because of os-client-config unittest mocking 864 version_kwargs = {} 865 if min_version: 866 version_kwargs['min_version'] = min_version 867 if max_version: 868 version_kwargs['max_version'] = max_version 869 try: 870 # Return the highest version we find that matches 871 # the request 872 endpoint = session.get_endpoint( 873 service_type=service_type, 874 region_name=region_name, 875 interface=interface, 876 service_name=service_name, 877 **version_kwargs 878 ) 879 except keystoneauth1.exceptions.catalog.EndpointNotFound: 880 endpoint = None 881 if not endpoint: 882 self.log.warning( 883 "Keystone catalog entry not found (" 884 "service_type=%s,service_name=%s," 885 "interface=%s,region_name=%s)", 886 service_type, 887 service_name, 888 interface, 889 region_name, 890 ) 891 return endpoint 892 893 def get_cache_expiration_time(self): 894 # TODO(mordred) We should be validating/transforming this on input 895 return int(self._cache_expiration_time) 896 897 def get_cache_path(self): 898 return self._cache_path 899 900 def get_cache_class(self): 901 return self._cache_class 902 903 def get_cache_arguments(self): 904 return copy.deepcopy(self._cache_arguments) 905 906 def get_cache_expirations(self): 907 return copy.deepcopy(self._cache_expirations) 908 909 def get_cache_resource_expiration(self, resource, default=None): 910 """Get expiration time for a resource 911 912 :param resource: Name of the resource type 913 :param default: Default value to return if not found (optional, 914 defaults to None) 915 916 :returns: Expiration time for the resource type as float or default 917 """ 918 if resource not in self._cache_expirations: 919 return default 920 return float(self._cache_expirations[resource]) 921 922 def requires_floating_ip(self): 923 """Return whether or not this cloud requires floating ips. 924 925 926 :returns: True of False if know, None if discovery is needed. 927 If requires_floating_ip is not configured but the cloud is 928 known to not provide floating ips, will return False. 929 """ 930 if self.config['floating_ip_source'] == "None": 931 return False 932 return self.config.get('requires_floating_ip') 933 934 def get_external_networks(self): 935 """Get list of network names for external networks.""" 936 return [ 937 net['name'] for net in self.config.get('networks', []) 938 if net['routes_externally']] 939 940 def get_external_ipv4_networks(self): 941 """Get list of network names for external IPv4 networks.""" 942 return [ 943 net['name'] for net in self.config.get('networks', []) 944 if net['routes_ipv4_externally']] 945 946 def get_external_ipv6_networks(self): 947 """Get list of network names for external IPv6 networks.""" 948 return [ 949 net['name'] for net in self.config.get('networks', []) 950 if net['routes_ipv6_externally']] 951 952 def get_internal_networks(self): 953 """Get list of network names for internal networks.""" 954 return [ 955 net['name'] for net in self.config.get('networks', []) 956 if not net['routes_externally']] 957 958 def get_internal_ipv4_networks(self): 959 """Get list of network names for internal IPv4 networks.""" 960 return [ 961 net['name'] for net in self.config.get('networks', []) 962 if not net['routes_ipv4_externally']] 963 964 def get_internal_ipv6_networks(self): 965 """Get list of network names for internal IPv6 networks.""" 966 return [ 967 net['name'] for net in self.config.get('networks', []) 968 if not net['routes_ipv6_externally']] 969 970 def get_default_network(self): 971 """Get network used for default interactions.""" 972 for net in self.config.get('networks', []): 973 if net['default_interface']: 974 return net['name'] 975 return None 976 977 def get_nat_destination(self): 978 """Get network used for NAT destination.""" 979 for net in self.config.get('networks', []): 980 if net['nat_destination']: 981 return net['name'] 982 return None 983 984 def get_nat_source(self): 985 """Get network used for NAT source.""" 986 for net in self.config.get('networks', []): 987 if net.get('nat_source'): 988 return net['name'] 989 return None 990 991 def _get_extra_config(self, key, defaults=None): 992 """Fetch an arbitrary extra chunk of config, laying in defaults. 993 994 :param string key: name of the config section to fetch 995 :param dict defaults: (optional) default values to merge under the 996 found config 997 """ 998 defaults = _util.normalize_keys(defaults or {}) 999 if not key: 1000 return defaults 1001 return _util.merge_clouds( 1002 defaults, 1003 _util.normalize_keys(self._extra_config.get(key, {}))) 1004 1005 def get_client_config(self, name=None, defaults=None): 1006 """Get config settings for a named client. 1007 1008 Settings will also be looked for in a section called 'client'. 1009 If settings are found in both, they will be merged with the settings 1010 from the named section winning over the settings from client section, 1011 and both winning over provided defaults. 1012 1013 :param string name: 1014 Name of the config section to look for. 1015 :param dict defaults: 1016 Default settings to use. 1017 1018 :returns: 1019 A dict containing merged settings from the named section, the 1020 client section and the defaults. 1021 """ 1022 return self._get_extra_config( 1023 name, self._get_extra_config('client', defaults)) 1024 1025 def get_password_callback(self): 1026 return self._password_callback 1027 1028 def get_rate_limit(self, service_type=None): 1029 return self._get_service_config( 1030 'rate_limit', service_type=service_type) 1031 1032 def get_concurrency(self, service_type=None): 1033 return self._get_service_config( 1034 'concurrency', service_type=service_type) 1035 1036 def get_statsd_client(self): 1037 if not statsd: 1038 return None 1039 statsd_args = {} 1040 if self._statsd_host: 1041 statsd_args['host'] = self._statsd_host 1042 if self._statsd_port: 1043 statsd_args['port'] = self._statsd_port 1044 if statsd_args: 1045 try: 1046 return statsd.StatsClient(**statsd_args) 1047 except Exception: 1048 self.log.warning('Cannot establish connection to statsd') 1049 return None 1050 else: 1051 return None 1052 1053 def get_statsd_prefix(self): 1054 return self._statsd_prefix or 'openstack.api' 1055 1056 def get_prometheus_registry(self): 1057 if not self._collector_registry and prometheus_client: 1058 self._collector_registry = prometheus_client.REGISTRY 1059 return self._collector_registry 1060 1061 def get_prometheus_histogram(self): 1062 registry = self.get_prometheus_registry() 1063 if not registry or not prometheus_client: 1064 return 1065 # We have to hide a reference to the histogram on the registry 1066 # object, because it's collectors must be singletons for a given 1067 # registry but register at creation time. 1068 hist = getattr(registry, '_openstacksdk_histogram', None) 1069 if not hist: 1070 hist = prometheus_client.Histogram( 1071 'openstack_http_response_time', 1072 'Time taken for an http response to an OpenStack service', 1073 labelnames=[ 1074 'method', 'endpoint', 'service_type', 'status_code' 1075 ], 1076 registry=registry, 1077 ) 1078 registry._openstacksdk_histogram = hist 1079 return hist 1080 1081 def get_prometheus_counter(self): 1082 registry = self.get_prometheus_registry() 1083 if not registry or not prometheus_client: 1084 return 1085 counter = getattr(registry, '_openstacksdk_counter', None) 1086 if not counter: 1087 counter = prometheus_client.Counter( 1088 'openstack_http_requests', 1089 'Number of HTTP requests made to an OpenStack service', 1090 labelnames=[ 1091 'method', 'endpoint', 'service_type', 'status_code' 1092 ], 1093 registry=registry, 1094 ) 1095 registry._openstacksdk_counter = counter 1096 return counter 1097 1098 def has_service(self, service_type): 1099 service_type = service_type.lower().replace('-', '_') 1100 key = 'has_{service_type}'.format(service_type=service_type) 1101 return self.config.get( 1102 key, self._service_type_manager.is_official(service_type)) 1103 1104 def disable_service(self, service_type, reason=None): 1105 _disable_service(self.config, service_type, reason=reason) 1106 1107 def enable_service(self, service_type): 1108 service_type = service_type.lower().replace('-', '_') 1109 key = 'has_{service_type}'.format(service_type=service_type) 1110 self.config[key] = True 1111 1112 def get_disabled_reason(self, service_type): 1113 service_type = service_type.lower().replace('-', '_') 1114 d_key = _make_key('disabled_reason', service_type) 1115 return self.config.get(d_key) 1116 1117 def get_influxdb_client(self): 1118 influx_args = {} 1119 if not self._influxdb_config: 1120 return None 1121 use_udp = bool(self._influxdb_config.get('use_udp', False)) 1122 port = self._influxdb_config.get('port') 1123 if use_udp: 1124 influx_args['use_udp'] = True 1125 if 'port' in self._influxdb_config: 1126 if use_udp: 1127 influx_args['udp_port'] = port 1128 else: 1129 influx_args['port'] = port 1130 for key in ['host', 'username', 'password', 'database', 'timeout']: 1131 if key in self._influxdb_config: 1132 influx_args[key] = self._influxdb_config[key] 1133 if influxdb and influx_args: 1134 try: 1135 return influxdb.InfluxDBClient(**influx_args) 1136 except Exception: 1137 self.log.warning('Cannot establish connection to InfluxDB') 1138 else: 1139 self.log.warning('InfluxDB configuration is present, ' 1140 'but no client library is found.') 1141 return None 1142