1# Copyright 2012-2013 OpenStack Foundation 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); you may 4# not use this file except in compliance with the License. You may obtain 5# a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12# License for the specific language governing permissions and limitations 13# under the License. 14# 15 16"""Common client utilities""" 17 18import copy 19import getpass 20import logging 21import os 22import time 23import warnings 24 25from cliff import columns as cliff_columns 26from oslo_utils import importutils 27 28from osc_lib import exceptions 29from osc_lib.i18n import _ 30 31 32LOG = logging.getLogger(__name__) 33 34 35def backward_compat_col_lister(column_headers, columns, column_map): 36 """Convert the column headers to keep column backward compatibility. 37 38 Replace the new column name of column headers by old name, so that 39 the column headers can continue to support to show the old column name by 40 --column/-c option with old name, like: volume list -c 'Display Name' 41 42 :param column_headers: The column headers to be output in list command. 43 :param columns: The columns to be output. 44 :param column_map: The key of map is old column name, the value is new 45 column name, like: {'old_col': 'new_col'} 46 """ 47 if not columns: 48 return column_headers 49 # NOTE(RuiChen): column_headers may be a tuple in some code, like: 50 # volume v1, convert it to a list in order to change 51 # the column name. 52 column_headers = list(column_headers) 53 for old_col, new_col in column_map.items(): 54 if old_col in columns: 55 LOG.warning(_('The column "%(old_column)s" was deprecated, ' 56 'please use "%(new_column)s" replace.') % { 57 'old_column': old_col, 58 'new_column': new_col} 59 ) 60 if new_col in column_headers: 61 column_headers[column_headers.index(new_col)] = old_col 62 return column_headers 63 64 65def backward_compat_col_showone(show_object, columns, column_map): 66 """Convert the output object to keep column backward compatibility. 67 68 Replace the new column name of output object by old name, so that 69 the object can continue to support to show the old column name by 70 --column/-c option with old name, like: volume show -c 'display_name' 71 72 :param show_object: The object to be output in create/show commands. 73 :param columns: The columns to be output. 74 :param column_map: The key of map is old column name, the value is new 75 column name, like: {'old_col': 'new_col'} 76 """ 77 if not columns: 78 return show_object 79 80 show_object = copy.deepcopy(show_object) 81 for old_col, new_col in column_map.items(): 82 if old_col in columns: 83 LOG.warning(_('The column "%(old_column)s" was deprecated, ' 84 'please use "%(new_column)s" replace.') % { 85 'old_column': old_col, 86 'new_column': new_col} 87 ) 88 if new_col in show_object: 89 show_object.update({old_col: show_object.pop(new_col)}) 90 return show_object 91 92 93def build_kwargs_dict(arg_name, value): 94 """Return a dictionary containing `arg_name` if `value` is set.""" 95 kwargs = {} 96 if value: 97 kwargs[arg_name] = value 98 return kwargs 99 100 101def calculate_header_and_attrs(column_headers, attrs, parsed_args): 102 """Calculate headers and attribute names based on parsed_args.column. 103 104 When --column (-c) option is specified, this function calculates 105 column headers and expected API attribute names according to 106 the OSC header/column definitions. 107 108 This function also adjusts the content of parsed_args.columns 109 if API attribute names are used in parsed_args.columns. 110 This allows users to specify API attribute names in -c option. 111 112 :param column_headers: A tuple/list of column headers to display 113 :param attrs: a tuple/list of API attribute names. The order of 114 corresponding column header and API attribute name must match. 115 :param parsed_args: Parsed argument object returned by argparse parse_args 116 :returns: A tuple of calculated headers and API attribute names. 117 """ 118 if parsed_args.columns: 119 header_attr_map = dict(zip(column_headers, attrs)) 120 expected_attrs = [header_attr_map.get(c, c) 121 for c in parsed_args.columns] 122 attr_header_map = dict(zip(attrs, column_headers)) 123 expected_headers = [attr_header_map.get(c, c) 124 for c in parsed_args.columns] 125 # If attribute name is used in parsed_args.columns 126 # convert it into display names because cliff expects 127 # name in parsed_args.columns and name in column_headers matches. 128 parsed_args.columns = expected_headers 129 return expected_headers, expected_attrs 130 else: 131 return column_headers, attrs 132 133 134def env(*vars, **kwargs): 135 """Search for the first defined of possibly many env vars 136 137 Returns the first environment variable defined in vars, or 138 returns the default defined in kwargs. 139 """ 140 for v in vars: 141 value = os.environ.get(v, None) 142 if value: 143 return value 144 return kwargs.get('default', '') 145 146 147def find_min_match(items, sort_attr, **kwargs): 148 """Find all resources meeting the given minimum constraints 149 150 :param items: A List of objects to consider 151 :param sort_attr: Attribute to sort the resulting list 152 :param kwargs: A dict of attributes and their minimum values 153 :rtype: A list of resources osrted by sort_attr that meet the minimums 154 """ 155 156 def minimum_pieces_of_flair(item): 157 """Find lowest value greater than the minumum""" 158 159 result = True 160 for k in kwargs: 161 # AND together all of the given attribute results 162 result = result and kwargs[k] <= get_field(item, k) 163 return result 164 165 return sort_items(filter(minimum_pieces_of_flair, items), sort_attr) 166 167 168def find_resource(manager, name_or_id, **kwargs): 169 """Helper for the _find_* methods. 170 171 :param manager: A client manager class 172 :param name_or_id: The resource we are trying to find 173 :param kwargs: To be used in calling .find() 174 :rtype: The found resource 175 176 This method will attempt to find a resource in a variety of ways. 177 Primarily .get() methods will be called with `name_or_id` as an integer 178 value, and tried again as a string value. 179 180 If both fail, then a .find() is attempted, which is essentially calling 181 a .list() function with a 'name' query parameter that is set to 182 `name_or_id`. 183 184 Lastly, if any kwargs are passed in, they will be treated as additional 185 query parameters. This is particularly handy in the case of finding 186 resources in a domain. 187 188 """ 189 190 # Case 1: name_or_id is an ID, we need to call get() directly 191 # for example: /projects/454ad1c743e24edcad846d1118837cac 192 # For some projects, the name only will work. For keystone, this is not 193 # enough information, and domain information is necessary. 194 try: 195 return manager.get(name_or_id) 196 except Exception: 197 pass 198 199 if kwargs: 200 # Case 2: name_or_id is a name, but we have query args in kwargs 201 # for example: /projects/demo&domain_id=30524568d64447fbb3fa8b7891c10dd 202 try: 203 return manager.get(name_or_id, **kwargs) 204 except Exception: 205 pass 206 207 # Case 3: Try to get entity as integer id. Keystone does not have integer 208 # IDs, they are UUIDs, but some things in nova do, like flavors. 209 try: 210 if isinstance(name_or_id, int) or name_or_id.isdigit(): 211 return manager.get(int(name_or_id), **kwargs) 212 # FIXME(dtroyer): The exception to catch here is dependent on which 213 # client library the manager passed in belongs to. 214 # Eventually this should be pulled from a common set 215 # of client exceptions. 216 except Exception as ex: 217 if (type(ex).__name__ == 'NotFound' or 218 type(ex).__name__ == 'HTTPNotFound' or 219 type(ex).__name__ == 'TypeError'): 220 pass 221 else: 222 raise 223 224 # Case 4: Try to use find. 225 # Reset the kwargs here for find 226 if len(kwargs) == 0: 227 kwargs = {} 228 229 try: 230 # Prepare the kwargs for calling find 231 if 'NAME_ATTR' in manager.resource_class.__dict__: 232 # novaclient does this for oddball resources 233 kwargs[manager.resource_class.NAME_ATTR] = name_or_id 234 else: 235 kwargs['name'] = name_or_id 236 except Exception: 237 pass 238 239 # finally try to find entity by name 240 try: 241 return manager.find(**kwargs) 242 # FIXME(dtroyer): The exception to catch here is dependent on which 243 # client library the manager passed in belongs to. 244 # Eventually this should be pulled from a common set 245 # of client exceptions. 246 except Exception as ex: 247 if type(ex).__name__ == 'NotFound': 248 msg = _( 249 "No %(resource)s with a name or ID of '%(id)s' exists." 250 ) 251 raise exceptions.CommandError(msg % { 252 'resource': manager.resource_class.__name__.lower(), 253 'id': name_or_id, 254 }) 255 if type(ex).__name__ == 'NoUniqueMatch': 256 msg = _( 257 "More than one %(resource)s exists with the name '%(id)s'." 258 ) 259 raise exceptions.CommandError(msg % { 260 'resource': manager.resource_class.__name__.lower(), 261 'id': name_or_id, 262 }) 263 else: 264 pass 265 266 # Case 5: For client with no find function, list all resources and hope 267 # to find a matching name or ID. 268 count = 0 269 for resource in manager.list(): 270 if (resource.get('id') == name_or_id or 271 resource.get('name') == name_or_id): 272 count += 1 273 _resource = resource 274 if count == 0: 275 # we found no match, report back this error: 276 msg = _("Could not find resource %s") 277 raise exceptions.CommandError(msg % name_or_id) 278 elif count == 1: 279 return _resource 280 else: 281 # we found multiple matches, report back this error 282 msg = _("More than one resource exists with the name or ID '%s'.") 283 raise exceptions.CommandError(msg % name_or_id) 284 285 286def format_dict(data, prefix=None): 287 """Return a formatted string of key value pairs 288 289 :param data: a dict 290 :param prefix: the current parent keys in a recursive call 291 :rtype: a string formatted to key='value' 292 """ 293 294 if data is None: 295 return None 296 297 output = "" 298 for s in sorted(data): 299 if prefix: 300 key_str = ".".join([prefix, s]) 301 else: 302 key_str = s 303 if isinstance(data[s], dict): 304 # NOTE(dtroyer): Only append the separator chars here, quoting 305 # is completely handled in the terminal case. 306 output = output + format_dict(data[s], prefix=key_str) + ", " 307 elif data[s] is not None: 308 output = output + key_str + "='" + str(data[s]) + "', " 309 else: 310 output = output + key_str + "=, " 311 return output[:-2] 312 313 314def format_dict_of_list(data, separator='; '): 315 """Return a formatted string of key value pair 316 317 :param data: a dict, key is string, value is a list of string, for example: 318 {u'public': [u'2001:db8::8', u'172.24.4.6']} 319 :param separator: the separator to use between key/value pair 320 (default: '; ') 321 :return: a string formatted to {'key1'=['value1', 'value2']} with separated 322 by separator 323 """ 324 if data is None: 325 return None 326 327 output = [] 328 for key in sorted(data): 329 value = data[key] 330 if value is None: 331 continue 332 value_str = format_list(value) 333 group = "%s=%s" % (key, value_str) 334 output.append(group) 335 336 return separator.join(output) 337 338 339def format_list(data, separator=', '): 340 """Return a formatted strings 341 342 :param data: a list of strings 343 :param separator: the separator to use between strings (default: ', ') 344 :rtype: a string formatted based on separator 345 """ 346 if data is None: 347 return None 348 349 return separator.join(sorted(data)) 350 351 352def format_list_of_dicts(data): 353 """Return a formatted string of key value pairs for each dict 354 355 :param data: a list of dicts 356 :rtype: a string formatted to key='value' with dicts separated by new line 357 """ 358 if data is None: 359 return None 360 361 return '\n'.join(format_dict(i) for i in data) 362 363 364def format_size(size): 365 """Display size of a resource in a human readable format 366 367 :param string size: 368 The size of the resource in bytes. 369 370 :returns: 371 Returns the size in human-friendly format 372 :rtype string: 373 374 This function converts the size (provided in bytes) of a resource 375 into a human-friendly format such as K, M, G, T, P, E, Z 376 """ 377 378 suffix = ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z'] 379 base = 1000.0 380 index = 0 381 382 if size is None: 383 size = 0 384 while size >= base: 385 index = index + 1 386 size = size / base 387 388 padded = '%.1f' % size 389 stripped = padded.rstrip('0').rstrip('.') 390 391 return '%s%s' % (stripped, suffix[index]) 392 393 394def get_client_class(api_name, version, version_map): 395 """Returns the client class for the requested API version 396 397 :param api_name: the name of the API, e.g. 'compute', 'image', etc 398 :param version: the requested API version 399 :param version_map: a dict of client classes keyed by version 400 :rtype: a client class for the requested API version 401 """ 402 try: 403 client_path = version_map[str(version)] 404 except (KeyError, ValueError): 405 sorted_versions = sorted(version_map.keys(), 406 key=lambda s: list(map(int, s.split('.')))) 407 msg = _( 408 "Invalid %(api_name)s client version '%(version)s'. " 409 "must be one of: %(version_map)s" 410 ) 411 raise exceptions.UnsupportedVersion(msg % { 412 'api_name': api_name, 413 'version': version, 414 'version_map': ', '.join(sorted_versions), 415 }) 416 417 return importutils.import_class(client_path) 418 419 420def get_dict_properties(item, fields, mixed_case_fields=None, formatters=None): 421 """Return a tuple containing the item properties. 422 423 :param item: a single dict resource 424 :param fields: tuple of strings with the desired field names 425 :param mixed_case_fields: tuple of field names to preserve case 426 :param formatters: dictionary mapping field names to callables 427 to format the values 428 """ 429 if mixed_case_fields is None: 430 mixed_case_fields = [] 431 if formatters is None: 432 formatters = {} 433 434 row = [] 435 436 for field in fields: 437 if field in mixed_case_fields: 438 field_name = field.replace(' ', '_') 439 else: 440 field_name = field.lower().replace(' ', '_') 441 data = item[field_name] if field_name in item else '' 442 if field in formatters: 443 formatter = formatters[field] 444 if (isinstance(formatter, type) and issubclass( 445 formatter, cliff_columns.FormattableColumn)): 446 data = formatter(data) 447 elif callable(formatter): 448 warnings.warn( 449 'The usage of formatter functions is now discouraged. ' 450 'Consider using cliff.columns.FormattableColumn instead. ' 451 'See reviews linked with bug 1687955 for more detail.', 452 category=DeprecationWarning) 453 if data is not None: 454 data = formatter(data) 455 else: 456 msg = "Invalid formatter provided." 457 raise exceptions.CommandError(msg) 458 459 row.append(data) 460 return tuple(row) 461 462 463def get_effective_log_level(): 464 """Returns the lowest logging level considered by logging handlers 465 466 Retrieve and return the smallest log level set among the root 467 logger's handlers (in case of multiple handlers). 468 """ 469 root_log = logging.getLogger() 470 min_log_lvl = logging.CRITICAL 471 for handler in root_log.handlers: 472 min_log_lvl = min(min_log_lvl, handler.level) 473 return min_log_lvl 474 475 476def get_field(item, field): 477 try: 478 if isinstance(item, dict): 479 return item[field] 480 else: 481 return getattr(item, field) 482 except Exception: 483 msg = _("Resource doesn't have field %s") 484 raise exceptions.CommandError(msg % field) 485 486 487def get_item_properties(item, fields, mixed_case_fields=None, formatters=None): 488 """Return a tuple containing the item properties. 489 490 :param item: a single item resource (e.g. Server, Project, etc) 491 :param fields: tuple of strings with the desired field names 492 :param mixed_case_fields: tuple of field names to preserve case 493 :param formatters: dictionary mapping field names to callables 494 to format the values 495 """ 496 if mixed_case_fields is None: 497 mixed_case_fields = [] 498 if formatters is None: 499 formatters = {} 500 501 row = [] 502 503 for field in fields: 504 if field in mixed_case_fields: 505 field_name = field.replace(' ', '_') 506 else: 507 field_name = field.lower().replace(' ', '_') 508 data = getattr(item, field_name, '') 509 if field in formatters: 510 formatter = formatters[field] 511 if (isinstance(formatter, type) and issubclass( 512 formatter, cliff_columns.FormattableColumn)): 513 data = formatter(data) 514 elif callable(formatter): 515 warnings.warn( 516 'The usage of formatter functions is now discouraged. ' 517 'Consider using cliff.columns.FormattableColumn instead. ' 518 'See reviews linked with bug 1687955 for more detail.', 519 category=DeprecationWarning) 520 if data is not None: 521 data = formatter(data) 522 else: 523 msg = "Invalid formatter provided." 524 raise exceptions.CommandError(msg) 525 526 row.append(data) 527 return tuple(row) 528 529 530def get_password(stdin, prompt=None, confirm=True): 531 message = prompt or "User Password:" 532 if hasattr(stdin, 'isatty') and stdin.isatty(): 533 try: 534 while True: 535 first_pass = getpass.getpass(message) 536 if not confirm: 537 return first_pass 538 second_pass = getpass.getpass("Repeat " + message) 539 if first_pass == second_pass: 540 return first_pass 541 msg = _("The passwords entered were not the same") 542 print(msg) 543 except EOFError: # Ctl-D 544 msg = _("Error reading password") 545 raise exceptions.CommandError(msg) 546 msg = _("No terminal detected attempting to read password") 547 raise exceptions.CommandError(msg) 548 549 550def is_ascii(string): 551 try: 552 (string.decode('ascii') if isinstance(string, bytes) 553 else string.encode('ascii')) 554 return True 555 except (UnicodeEncodeError, UnicodeDecodeError): 556 return False 557 558 559def read_blob_file_contents(blob_file): 560 try: 561 with open(blob_file) as file: 562 blob = file.read().strip() 563 return blob 564 except IOError: 565 msg = _("Error occurred trying to read from file %s") 566 raise exceptions.CommandError(msg % blob_file) 567 568 569def sort_items(items, sort_str, sort_type=None): 570 """Sort items based on sort keys and sort directions given by sort_str. 571 572 :param items: a list or generator object of items 573 :param sort_str: a string defining the sort rules, the format is 574 '<key1>:[direction1],<key2>:[direction2]...', direction can be 'asc' 575 for ascending or 'desc' for descending, if direction is not given, 576 it's ascending by default 577 :return: sorted items 578 """ 579 if not sort_str: 580 return items 581 # items may be a generator object, transform it to a list 582 items = list(items) 583 sort_keys = sort_str.strip().split(',') 584 for sort_key in reversed(sort_keys): 585 reverse = False 586 if ':' in sort_key: 587 sort_key, direction = sort_key.split(':', 1) 588 if not sort_key: 589 msg = _("'<empty string>'' is not a valid sort key") 590 raise exceptions.CommandError(msg) 591 if direction not in ['asc', 'desc']: 592 if not direction: 593 direction = "<empty string>" 594 msg = _( 595 "'%(direction)s' is not a valid sort direction for " 596 "sort key %(sort_key)s, use 'asc' or 'desc' instead" 597 ) 598 raise exceptions.CommandError(msg % { 599 'direction': direction, 600 'sort_key': sort_key, 601 }) 602 if direction == 'desc': 603 reverse = True 604 605 def f(x): 606 # Attempts to convert items to same 'sort_type' if provided. 607 # This is due to Python 3 throwing TypeError if you attempt to 608 # compare different types 609 item = get_field(x, sort_key) 610 if sort_type and not isinstance(item, sort_type): 611 try: 612 item = sort_type(item) 613 except Exception: 614 # Can't convert, so no sensible way to compare 615 item = sort_type() 616 return item 617 618 items.sort(key=f, reverse=reverse) 619 620 return items 621 622 623def wait_for_delete(manager, 624 res_id, 625 status_field='status', 626 error_status=['error'], 627 exception_name=['NotFound'], 628 sleep_time=5, 629 timeout=300, 630 callback=None): 631 """Wait for resource deletion 632 633 :param manager: the manager from which we can get the resource 634 :param res_id: the resource id to watch 635 :param status_field: the status attribute in the returned resource object, 636 this is used to check for error states while the resource is being 637 deleted 638 :param error_status: a list of status strings for error 639 :param exception_name: a list of exception strings for deleted case 640 :param sleep_time: wait this long between checks (seconds) 641 :param timeout: check until this long (seconds) 642 :param callback: called per sleep cycle, useful to display progress; this 643 function is passed a progress value during each iteration of the wait 644 loop 645 :rtype: True on success, False if the resource has gone to error state or 646 the timeout has been reached 647 """ 648 total_time = 0 649 while total_time < timeout: 650 try: 651 # might not be a bad idea to re-use find_resource here if it was 652 # a bit more friendly in the exceptions it raised so we could just 653 # handle a NotFound exception here without parsing the message 654 res = manager.get(res_id) 655 except Exception as ex: 656 if type(ex).__name__ in exception_name: 657 return True 658 raise 659 660 status = getattr(res, status_field, '').lower() 661 if status in error_status: 662 return False 663 664 if callback: 665 progress = getattr(res, 'progress', None) or 0 666 callback(progress) 667 time.sleep(sleep_time) 668 total_time += sleep_time 669 670 # if we got this far we've timed out 671 return False 672 673 674def wait_for_status(status_f, 675 res_id, 676 status_field='status', 677 success_status=['active'], 678 error_status=['error'], 679 sleep_time=5, 680 callback=None): 681 """Wait for status change on a resource during a long-running operation 682 683 :param status_f: a status function that takes a single id argument 684 :param res_id: the resource id to watch 685 :param status_field: the status attribute in the returned resource object 686 :param success_status: a list of status strings for successful completion 687 :param error_status: a list of status strings for error 688 :param sleep_time: wait this long (seconds) 689 :param callback: called per sleep cycle, useful to display progress 690 :rtype: True on success 691 """ 692 while True: 693 res = status_f(res_id) 694 status = getattr(res, status_field, '').lower() 695 if status in success_status: 696 retval = True 697 break 698 elif status in error_status: 699 retval = False 700 break 701 if callback: 702 progress = getattr(res, 'progress', None) or 0 703 callback(progress) 704 time.sleep(sleep_time) 705 return retval 706 707 708def get_osc_show_columns_for_sdk_resource( 709 sdk_resource, 710 osc_column_map, 711 invisible_columns=None 712): 713 """Get and filter the display and attribute columns for an SDK resource. 714 715 Common utility function for preparing the output of an OSC show command. 716 Some of the columns may need to get renamed, others made invisible. 717 718 :param sdk_resource: An SDK resource 719 :param osc_column_map: A hash of mappings for display column names 720 :param invisible_columns: A list of invisible column names 721 722 :returns: Two tuples containing the names of the display and attribute 723 columns 724 """ 725 726 if getattr(sdk_resource, 'allow_get', None) is not None: 727 resource_dict = sdk_resource.to_dict( 728 body=True, headers=False, ignore_none=False) 729 else: 730 resource_dict = sdk_resource 731 732 # Build the OSC column names to display for the SDK resource. 733 attr_map = {} 734 display_columns = list(resource_dict.keys()) 735 invisible_columns = [] if invisible_columns is None else invisible_columns 736 for col_name in invisible_columns: 737 if col_name in display_columns: 738 display_columns.remove(col_name) 739 for sdk_attr, osc_attr in osc_column_map.items(): 740 if sdk_attr in display_columns: 741 attr_map[osc_attr] = sdk_attr 742 display_columns.remove(sdk_attr) 743 if osc_attr not in display_columns: 744 display_columns.append(osc_attr) 745 sorted_display_columns = sorted(display_columns) 746 747 # Build the SDK attribute names for the OSC column names. 748 attr_columns = [] 749 for column in sorted_display_columns: 750 new_column = attr_map[column] if column in attr_map else column 751 attr_columns.append(new_column) 752 return tuple(sorted_display_columns), tuple(attr_columns) 753