1from __future__ import print_function 2 3import errno 4import itertools 5import math 6import numbers 7import os 8import platform 9import signal 10import subprocess 11import sys 12import threading 13 14 15def is_string(value): 16 try: 17 # Python 2 and Python 3 are different here. 18 return isinstance(value, basestring) 19 except NameError: 20 return isinstance(value, str) 21 22 23def pythonize_bool(value): 24 if value is None: 25 return False 26 if type(value) is bool: 27 return value 28 if isinstance(value, numbers.Number): 29 return value != 0 30 if is_string(value): 31 if value.lower() in ('1', 'true', 'on', 'yes'): 32 return True 33 if value.lower() in ('', '0', 'false', 'off', 'no'): 34 return False 35 raise ValueError('"{}" is not a valid boolean'.format(value)) 36 37 38def make_word_regex(word): 39 return r'\b' + word + r'\b' 40 41 42def to_bytes(s): 43 """Return the parameter as type 'bytes', possibly encoding it. 44 45 In Python2, the 'bytes' type is the same as 'str'. In Python3, they 46 are distinct. 47 48 """ 49 if isinstance(s, bytes): 50 # In Python2, this branch is taken for both 'str' and 'bytes'. 51 # In Python3, this branch is taken only for 'bytes'. 52 return s 53 # In Python2, 's' is a 'unicode' object. 54 # In Python3, 's' is a 'str' object. 55 # Encode to UTF-8 to get 'bytes' data. 56 return s.encode('utf-8') 57 58 59def to_string(b): 60 """Return the parameter as type 'str', possibly encoding it. 61 62 In Python2, the 'str' type is the same as 'bytes'. In Python3, the 63 'str' type is (essentially) Python2's 'unicode' type, and 'bytes' is 64 distinct. 65 66 """ 67 if isinstance(b, str): 68 # In Python2, this branch is taken for types 'str' and 'bytes'. 69 # In Python3, this branch is taken only for 'str'. 70 return b 71 if isinstance(b, bytes): 72 # In Python2, this branch is never taken ('bytes' is handled as 'str'). 73 # In Python3, this is true only for 'bytes'. 74 try: 75 return b.decode('utf-8') 76 except UnicodeDecodeError: 77 # If the value is not valid Unicode, return the default 78 # repr-line encoding. 79 return str(b) 80 81 # By this point, here's what we *don't* have: 82 # 83 # - In Python2: 84 # - 'str' or 'bytes' (1st branch above) 85 # - In Python3: 86 # - 'str' (1st branch above) 87 # - 'bytes' (2nd branch above) 88 # 89 # The last type we might expect is the Python2 'unicode' type. There is no 90 # 'unicode' type in Python3 (all the Python3 cases were already handled). In 91 # order to get a 'str' object, we need to encode the 'unicode' object. 92 try: 93 return b.encode('utf-8') 94 except AttributeError: 95 raise TypeError('not sure how to convert %s to %s' % (type(b), str)) 96 97 98def to_unicode(s): 99 """Return the parameter as type which supports unicode, possibly decoding 100 it. 101 102 In Python2, this is the unicode type. In Python3 it's the str type. 103 104 """ 105 if isinstance(s, bytes): 106 # In Python2, this branch is taken for both 'str' and 'bytes'. 107 # In Python3, this branch is taken only for 'bytes'. 108 return s.decode('utf-8') 109 return s 110 111 112def usable_core_count(): 113 """Return the number of cores the current process can use, if supported. 114 Otherwise, return the total number of cores (like `os.cpu_count()`). 115 Default to 1 if undetermined. 116 117 """ 118 try: 119 n = len(os.sched_getaffinity(0)) 120 except AttributeError: 121 n = os.cpu_count() or 1 122 123 # On Windows with more than 60 processes, multiprocessing's call to 124 # _winapi.WaitForMultipleObjects() prints an error and lit hangs. 125 if platform.system() == 'Windows': 126 return min(n, 60) 127 128 return n 129 130 131def mkdir(path): 132 try: 133 if platform.system() == 'Windows': 134 from ctypes import windll 135 from ctypes import GetLastError, WinError 136 137 path = os.path.abspath(path) 138 # Make sure that the path uses backslashes here, in case 139 # python would have happened to use forward slashes, as the 140 # NT path format only supports backslashes. 141 path = path.replace('/', '\\') 142 NTPath = to_unicode(r'\\?\%s' % path) 143 if not windll.kernel32.CreateDirectoryW(NTPath, None): 144 raise WinError(GetLastError()) 145 else: 146 os.mkdir(path) 147 except OSError: 148 e = sys.exc_info()[1] 149 # ignore EEXIST, which may occur during a race condition 150 if e.errno != errno.EEXIST: 151 raise 152 153 154def mkdir_p(path): 155 """mkdir_p(path) - Make the "path" directory, if it does not exist; this 156 will also make directories for any missing parent directories.""" 157 if not path or os.path.exists(path): 158 return 159 160 parent = os.path.dirname(path) 161 if parent != path: 162 mkdir_p(parent) 163 164 mkdir(path) 165 166 167def listdir_files(dirname, suffixes=None, exclude_filenames=None): 168 """Yields files in a directory. 169 170 Filenames that are not excluded by rules below are yielded one at a time, as 171 basenames (i.e., without dirname). 172 173 Files starting with '.' are always skipped. 174 175 If 'suffixes' is not None, then only filenames ending with one of its 176 members will be yielded. These can be extensions, like '.exe', or strings, 177 like 'Test'. (It is a lexicographic check; so an empty sequence will yield 178 nothing, but a single empty string will yield all filenames.) 179 180 If 'exclude_filenames' is not None, then none of the file basenames in it 181 will be yielded. 182 183 If specified, the containers for 'suffixes' and 'exclude_filenames' must 184 support membership checking for strs. 185 186 Args: 187 dirname: a directory path. 188 suffixes: (optional) a sequence of strings (set, list, etc.). 189 exclude_filenames: (optional) a sequence of strings. 190 191 Yields: 192 Filenames as returned by os.listdir (generally, str). 193 194 """ 195 if exclude_filenames is None: 196 exclude_filenames = set() 197 if suffixes is None: 198 suffixes = {''} 199 for filename in os.listdir(dirname): 200 if (os.path.isdir(os.path.join(dirname, filename)) or 201 filename.startswith('.') or 202 filename in exclude_filenames or 203 not any(filename.endswith(sfx) for sfx in suffixes)): 204 continue 205 yield filename 206 207 208def which(command, paths=None): 209 """which(command, [paths]) - Look up the given command in the paths string 210 (or the PATH environment variable, if unspecified).""" 211 212 if paths is None: 213 paths = os.environ.get('PATH', '') 214 215 # Check for absolute match first. 216 if os.path.isabs(command) and os.path.isfile(command): 217 return os.path.normcase(os.path.normpath(command)) 218 219 # Would be nice if Python had a lib function for this. 220 if not paths: 221 paths = os.defpath 222 223 # Get suffixes to search. 224 # On Cygwin, 'PATHEXT' may exist but it should not be used. 225 if os.pathsep == ';': 226 pathext = os.environ.get('PATHEXT', '').split(';') 227 else: 228 pathext = [''] 229 230 # Search the paths... 231 for path in paths.split(os.pathsep): 232 for ext in pathext: 233 p = os.path.join(path, command + ext) 234 if os.path.exists(p) and not os.path.isdir(p): 235 return os.path.normcase(os.path.normpath(p)) 236 237 return None 238 239 240def checkToolsPath(dir, tools): 241 for tool in tools: 242 if not os.path.exists(os.path.join(dir, tool)): 243 return False 244 return True 245 246 247def whichTools(tools, paths): 248 for path in paths.split(os.pathsep): 249 if checkToolsPath(path, tools): 250 return path 251 return None 252 253 254def printHistogram(items, title='Items'): 255 items.sort(key=lambda item: item[1]) 256 257 maxValue = max([v for _, v in items]) 258 259 # Select first "nice" bar height that produces more than 10 bars. 260 power = int(math.ceil(math.log(maxValue, 10))) 261 for inc in itertools.cycle((5, 2, 2.5, 1)): 262 barH = inc * 10**power 263 N = int(math.ceil(maxValue / barH)) 264 if N > 10: 265 break 266 elif inc == 1: 267 power -= 1 268 269 histo = [set() for i in range(N)] 270 for name, v in items: 271 bin = min(int(N * v / maxValue), N - 1) 272 histo[bin].add(name) 273 274 barW = 40 275 hr = '-' * (barW + 34) 276 print('Slowest %s:' % title) 277 print(hr) 278 for name, value in reversed(items[-20:]): 279 print('%.2fs: %s' % (value, name)) 280 print('\n%s Times:' % title) 281 print(hr) 282 pDigits = int(math.ceil(math.log(maxValue, 10))) 283 pfDigits = max(0, 3 - pDigits) 284 if pfDigits: 285 pDigits += pfDigits + 1 286 cDigits = int(math.ceil(math.log(len(items), 10))) 287 print('[%s] :: [%s] :: [%s]' % ('Range'.center((pDigits + 1) * 2 + 3), 288 'Percentage'.center(barW), 289 'Count'.center(cDigits * 2 + 1))) 290 print(hr) 291 for i, row in reversed(list(enumerate(histo))): 292 pct = float(len(row)) / len(items) 293 w = int(barW * pct) 294 print('[%*.*fs,%*.*fs) :: [%s%s] :: [%*d/%*d]' % ( 295 pDigits, pfDigits, i * barH, pDigits, pfDigits, (i + 1) * barH, 296 '*' * w, ' ' * (barW - w), cDigits, len(row), cDigits, len(items))) 297 print(hr) 298 299 300class ExecuteCommandTimeoutException(Exception): 301 def __init__(self, msg, out, err, exitCode): 302 assert isinstance(msg, str) 303 assert isinstance(out, str) 304 assert isinstance(err, str) 305 assert isinstance(exitCode, int) 306 self.msg = msg 307 self.out = out 308 self.err = err 309 self.exitCode = exitCode 310 311 312# Close extra file handles on UNIX (on Windows this cannot be done while 313# also redirecting input). 314kUseCloseFDs = not (platform.system() == 'Windows') 315 316 317def executeCommand(command, cwd=None, env=None, input=None, timeout=0): 318 """Execute command ``command`` (list of arguments or string) with. 319 320 * working directory ``cwd`` (str), use None to use the current 321 working directory 322 * environment ``env`` (dict), use None for none 323 * Input to the command ``input`` (str), use string to pass 324 no input. 325 * Max execution time ``timeout`` (int) seconds. Use 0 for no timeout. 326 327 Returns a tuple (out, err, exitCode) where 328 * ``out`` (str) is the standard output of running the command 329 * ``err`` (str) is the standard error of running the command 330 * ``exitCode`` (int) is the exitCode of running the command 331 332 If the timeout is hit an ``ExecuteCommandTimeoutException`` 333 is raised. 334 335 """ 336 if input is not None: 337 input = to_bytes(input) 338 p = subprocess.Popen(command, cwd=cwd, 339 stdin=subprocess.PIPE, 340 stdout=subprocess.PIPE, 341 stderr=subprocess.PIPE, 342 env=env, close_fds=kUseCloseFDs) 343 timerObject = None 344 # FIXME: Because of the way nested function scopes work in Python 2.x we 345 # need to use a reference to a mutable object rather than a plain 346 # bool. In Python 3 we could use the "nonlocal" keyword but we need 347 # to support Python 2 as well. 348 hitTimeOut = [False] 349 try: 350 if timeout > 0: 351 def killProcess(): 352 # We may be invoking a shell so we need to kill the 353 # process and all its children. 354 hitTimeOut[0] = True 355 killProcessAndChildren(p.pid) 356 357 timerObject = threading.Timer(timeout, killProcess) 358 timerObject.start() 359 360 out, err = p.communicate(input=input) 361 exitCode = p.wait() 362 finally: 363 if timerObject != None: 364 timerObject.cancel() 365 366 # Ensure the resulting output is always of string type. 367 out = to_string(out) 368 err = to_string(err) 369 370 if hitTimeOut[0]: 371 raise ExecuteCommandTimeoutException( 372 msg='Reached timeout of {} seconds'.format(timeout), 373 out=out, 374 err=err, 375 exitCode=exitCode 376 ) 377 378 # Detect Ctrl-C in subprocess. 379 if exitCode == -signal.SIGINT: 380 raise KeyboardInterrupt 381 382 return out, err, exitCode 383 384 385def usePlatformSdkOnDarwin(config, lit_config): 386 # On Darwin, support relocatable SDKs by providing Clang with a 387 # default system root path. 388 if 'darwin' in config.target_triple: 389 try: 390 cmd = subprocess.Popen(['xcrun', '--show-sdk-path', '--sdk', 'macosx'], 391 stdout=subprocess.PIPE, stderr=subprocess.PIPE) 392 out, err = cmd.communicate() 393 out = out.strip() 394 res = cmd.wait() 395 except OSError: 396 res = -1 397 if res == 0 and out: 398 sdk_path = out.decode() 399 lit_config.note('using SDKROOT: %r' % sdk_path) 400 config.environment['SDKROOT'] = sdk_path 401 402 403def findPlatformSdkVersionOnMacOS(config, lit_config): 404 if 'darwin' in config.target_triple: 405 try: 406 cmd = subprocess.Popen(['xcrun', '--show-sdk-version', '--sdk', 'macosx'], 407 stdout=subprocess.PIPE, stderr=subprocess.PIPE) 408 out, err = cmd.communicate() 409 out = out.strip() 410 res = cmd.wait() 411 except OSError: 412 res = -1 413 if res == 0 and out: 414 return out.decode() 415 return None 416 417def killProcessAndChildrenIsSupported(): 418 """ 419 Returns a tuple (<supported> , <error message>) 420 where 421 `<supported>` is True if `killProcessAndChildren()` is supported on 422 the current host, returns False otherwise. 423 `<error message>` is an empty string if `<supported>` is True, 424 otherwise is contains a string describing why the function is 425 not supported. 426 """ 427 if platform.system() == 'AIX': 428 return (True, "") 429 try: 430 import psutil # noqa: F401 431 return (True, "") 432 except ImportError: 433 return (False, "Requires the Python psutil module but it could" 434 " not be found. Try installing it via pip or via" 435 " your operating system's package manager.") 436 437def killProcessAndChildren(pid): 438 """This function kills a process with ``pid`` and all its running children 439 (recursively). It is currently implemented using the psutil module on some 440 platforms which provides a simple platform neutral implementation. 441 442 TODO: Reimplement this without using psutil on all platforms so we can 443 remove our dependency on it. 444 445 """ 446 if platform.system() == 'AIX': 447 subprocess.call('kill -kill $(ps -o pid= -L{})'.format(pid), shell=True) 448 else: 449 import psutil 450 try: 451 psutilProc = psutil.Process(pid) 452 # Handle the different psutil API versions 453 try: 454 # psutil >= 2.x 455 children_iterator = psutilProc.children(recursive=True) 456 except AttributeError: 457 # psutil 1.x 458 children_iterator = psutilProc.get_children(recursive=True) 459 for child in children_iterator: 460 try: 461 child.kill() 462 except psutil.NoSuchProcess: 463 pass 464 psutilProc.kill() 465 except psutil.NoSuchProcess: 466 pass 467