1# -*- coding: utf-8 -*- 2# Copyright 2018 Google Inc. All Rights Reserved. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15"""Shared utility methods that calculate, convert, and simplify units.""" 16 17from __future__ import absolute_import 18from __future__ import print_function 19from __future__ import division 20from __future__ import unicode_literals 21 22import math 23import re 24 25import six 26 27if six.PY3: 28 long = int 29 30# Binary exponentiation strings. 31_EXP_STRINGS = [ 32 (0, 'B', 'bit'), 33 (10, 'KiB', 'Kibit', 'K'), 34 (20, 'MiB', 'Mibit', 'M'), 35 (30, 'GiB', 'Gibit', 'G'), 36 (40, 'TiB', 'Tibit', 'T'), 37 (50, 'PiB', 'Pibit', 'P'), 38 (60, 'EiB', 'Eibit', 'E'), 39] 40 41_EXP_TEN_STRING = [ 42 (3, 'k'), 43 (6, 'm'), 44 (9, 'b'), 45 (12, 't'), 46 (15, 'q'), 47] 48 49 50# Define this method before constants below, as some call it to init themselves. 51def _GenerateSuffixRegex(): 52 """Creates a suffix regex for human-readable byte counts.""" 53 human_bytes_re = r'(?P<num>\d*\.\d+|\d+)\s*(?P<suffix>%s)?' 54 suffixes = [] 55 suffix_to_si = {} 56 for i, si in enumerate(_EXP_STRINGS): 57 si_suffixes = [s.lower() for s in list(si)[1:]] 58 for suffix in si_suffixes: 59 suffix_to_si[suffix] = i 60 suffixes.extend(si_suffixes) 61 human_bytes_re %= '|'.join(suffixes) 62 matcher = re.compile(human_bytes_re) 63 return suffix_to_si, matcher 64 65 66# TODO: These should include the unit in the name, e.g. BYTES_PER_KIB, or 67# BYTES_IN_ONE_KIB. 68ONE_KIB = 1024 69ONE_MIB = 1024 * ONE_KIB 70ONE_GIB = 1024 * ONE_MIB 71 72# TODO: Remove 2, 8, 10 MIB vars. 73TWO_MIB = 2 * ONE_MIB 74EIGHT_MIB = 8 * ONE_MIB 75TEN_MIB = 10 * ONE_MIB 76 77SECONDS_PER_DAY = long(60 * 60 * 24) 78SUFFIX_TO_SI, MATCH_HUMAN_BYTES = _GenerateSuffixRegex() 79 80 81def _RoundToNearestExponent(num): 82 i = 0 83 while i + 1 < len(_EXP_STRINGS) and num >= (2**_EXP_STRINGS[i + 1][0]): 84 i += 1 85 return i, round(float(num) / 2.0**_EXP_STRINGS[i][0], 2) 86 87 88def CalculateThroughput(total_bytes_transferred, total_elapsed_time): 89 """Calculates throughput and checks for a small total_elapsed_time. 90 91 Args: 92 total_bytes_transferred: Total bytes transferred in a period of time. 93 total_elapsed_time: The amount of time elapsed in seconds. 94 95 Returns: 96 The throughput as a float. 97 """ 98 if total_elapsed_time < 0.01: 99 total_elapsed_time = 0.01 100 return float(total_bytes_transferred) / float(total_elapsed_time) 101 102 103def DecimalShort(num): 104 """Creates a shorter string version for a given number of objects. 105 106 Args: 107 num: The number of objects to be shortened. 108 Returns: 109 shortened string version for this number. It takes the largest 110 scale (thousand, million or billion) smaller than the number and divides it 111 by that scale, indicated by a suffix with one decimal place. This will thus 112 create a string of at most 6 characters, assuming num < 10^18. 113 Example: 123456789 => 123.4m 114 """ 115 for divisor_exp, suffix in reversed(_EXP_TEN_STRING): 116 if num >= 10**divisor_exp: 117 quotient = '%.1lf' % (float(num) / 10**divisor_exp) 118 return quotient + suffix 119 return str(num) 120 121 122def DivideAndCeil(dividend, divisor): 123 """Returns ceil(dividend / divisor). 124 125 Takes care to avoid the pitfalls of floating point arithmetic that could 126 otherwise yield the wrong result for large numbers. 127 128 Args: 129 dividend: Dividend for the operation. 130 divisor: Divisor for the operation. 131 132 Returns: 133 Quotient. 134 """ 135 quotient = dividend // divisor 136 if (dividend % divisor) != 0: 137 quotient += 1 138 return quotient 139 140 141def HumanReadableToBytes(human_string): 142 """Tries to convert a human-readable string to a number of bytes. 143 144 Args: 145 human_string: A string supplied by user, e.g. '1M', '3 GiB'. 146 Returns: 147 An integer containing the number of bytes. 148 Raises: 149 ValueError: on an invalid string. 150 """ 151 human_string = human_string.lower() 152 m = MATCH_HUMAN_BYTES.match(human_string) 153 if m: 154 num = float(m.group('num')) 155 if m.group('suffix'): 156 power = _EXP_STRINGS[SUFFIX_TO_SI[m.group('suffix')]][0] 157 num *= (2.0**power) 158 num = int(round(num)) 159 return num 160 raise ValueError('Invalid byte string specified: %s' % human_string) 161 162 163def HumanReadableWithDecimalPlaces(number, decimal_places=1): 164 """Creates a human readable format for bytes with fixed decimal places. 165 166 Args: 167 number: The number of bytes. 168 decimal_places: The number of decimal places. 169 Returns: 170 String representing a readable format for number with decimal_places 171 decimal places. 172 """ 173 number_format = MakeHumanReadable(number).split() 174 num = str(int(round(10**decimal_places * float(number_format[0])))) 175 if num == '0': 176 number_format[0] = ('0' + 177 (('.' + 178 ('0' * decimal_places)) if decimal_places else '')) 179 else: 180 num_length = len(num) 181 if decimal_places: 182 num = (num[:num_length - decimal_places] + '.' + 183 num[num_length - decimal_places:]) 184 number_format[0] = num 185 return ' '.join(number_format) 186 187 188def MakeBitsHumanReadable(num): 189 """Generates human readable string for a number of bits. 190 191 Args: 192 num: The number, in bits. 193 194 Returns: 195 A string form of the number using bit size abbreviations (kbit, Mbit, etc.) 196 """ 197 i, rounded_val = _RoundToNearestExponent(num) 198 return '%g %s' % (rounded_val, _EXP_STRINGS[i][2]) 199 200 201def MakeHumanReadable(num): 202 """Generates human readable string for a number of bytes. 203 204 Args: 205 num: The number, in bytes. 206 207 Returns: 208 A string form of the number using size abbreviations (KiB, MiB, etc.). 209 """ 210 i, rounded_val = _RoundToNearestExponent(num) 211 return '%g %s' % (rounded_val, _EXP_STRINGS[i][1]) 212 213 214def Percentile(values, percent, key=lambda x: x): 215 """Find the percentile of a list of values. 216 217 Taken from: http://code.activestate.com/recipes/511478/ 218 219 Args: 220 values: a list of numeric values. Note that the values MUST BE already 221 sorted. 222 percent: a float value from 0.0 to 1.0. 223 key: optional key function to compute value from each element of the list 224 of values. 225 226 Returns: 227 The percentile of the values. 228 """ 229 if not values: 230 return None 231 k = (len(values) - 1) * percent 232 f = math.floor(k) 233 c = math.ceil(k) 234 if f == c: 235 return key(values[int(k)]) 236 d0 = key(values[int(f)]) * (c - k) 237 d1 = key(values[int(c)]) * (k - f) 238 return d0 + d1 239 240 241def PrettyTime(remaining_time): 242 """Creates a standard version for a given remaining time in seconds. 243 244 Created over using strftime because strftime seems to be 245 more suitable for a datetime object, rather than just a number of 246 seconds remaining. 247 Args: 248 remaining_time: The number of seconds remaining as a float, or a 249 string/None value indicating time was not correctly calculated. 250 Returns: 251 if remaining_time is a valid float, %H:%M:%D time remaining format with 252 the nearest integer from remaining_time (%H might be higher than 23). 253 Else, it returns the same message it received. 254 """ 255 remaining_time = int(round(remaining_time)) 256 hours = remaining_time // 3600 257 if hours >= 100: 258 # Too large to display with precision of minutes and seconds. 259 # If over 1000, saying 999+ hours should be enough. 260 return '%d+ hrs' % min(hours, 999) 261 remaining_time -= (3600 * hours) 262 minutes = remaining_time // 60 263 remaining_time -= (60 * minutes) 264 seconds = remaining_time 265 return (str('%02d' % hours) + ':' + str('%02d' % minutes) + ':' + 266 str('%02d' % seconds)) 267