1from __future__ import print_function 2 3import errno 4import itertools 5import math 6import numbers 7import os 8import platform 9import signal 10import subprocess 11import sys 12import threading 13 14 15def is_string(value): 16 try: 17 # Python 2 and Python 3 are different here. 18 return isinstance(value, basestring) 19 except NameError: 20 return isinstance(value, str) 21 22 23def pythonize_bool(value): 24 if value is None: 25 return False 26 if type(value) is bool: 27 return value 28 if isinstance(value, numbers.Number): 29 return value != 0 30 if is_string(value): 31 if value.lower() in ('1', 'true', 'on', 'yes'): 32 return True 33 if value.lower() in ('', '0', 'false', 'off', 'no'): 34 return False 35 raise ValueError('"{}" is not a valid boolean'.format(value)) 36 37 38def make_word_regex(word): 39 return r'\b' + word + r'\b' 40 41 42def to_bytes(s): 43 """Return the parameter as type 'bytes', possibly encoding it. 44 45 In Python2, the 'bytes' type is the same as 'str'. In Python3, they 46 are distinct. 47 48 """ 49 if isinstance(s, bytes): 50 # In Python2, this branch is taken for both 'str' and 'bytes'. 51 # In Python3, this branch is taken only for 'bytes'. 52 return s 53 # In Python2, 's' is a 'unicode' object. 54 # In Python3, 's' is a 'str' object. 55 # Encode to UTF-8 to get 'bytes' data. 56 return s.encode('utf-8') 57 58 59def to_string(b): 60 """Return the parameter as type 'str', possibly encoding it. 61 62 In Python2, the 'str' type is the same as 'bytes'. In Python3, the 63 'str' type is (essentially) Python2's 'unicode' type, and 'bytes' is 64 distinct. 65 66 """ 67 if isinstance(b, str): 68 # In Python2, this branch is taken for types 'str' and 'bytes'. 69 # In Python3, this branch is taken only for 'str'. 70 return b 71 if isinstance(b, bytes): 72 # In Python2, this branch is never taken ('bytes' is handled as 'str'). 73 # In Python3, this is true only for 'bytes'. 74 try: 75 return b.decode('utf-8') 76 except UnicodeDecodeError: 77 # If the value is not valid Unicode, return the default 78 # repr-line encoding. 79 return str(b) 80 81 # By this point, here's what we *don't* have: 82 # 83 # - In Python2: 84 # - 'str' or 'bytes' (1st branch above) 85 # - In Python3: 86 # - 'str' (1st branch above) 87 # - 'bytes' (2nd branch above) 88 # 89 # The last type we might expect is the Python2 'unicode' type. There is no 90 # 'unicode' type in Python3 (all the Python3 cases were already handled). In 91 # order to get a 'str' object, we need to encode the 'unicode' object. 92 try: 93 return b.encode('utf-8') 94 except AttributeError: 95 raise TypeError('not sure how to convert %s to %s' % (type(b), str)) 96 97 98def to_unicode(s): 99 """Return the parameter as type which supports unicode, possibly decoding 100 it. 101 102 In Python2, this is the unicode type. In Python3 it's the str type. 103 104 """ 105 if isinstance(s, bytes): 106 # In Python2, this branch is taken for both 'str' and 'bytes'. 107 # In Python3, this branch is taken only for 'bytes'. 108 return s.decode('utf-8') 109 return s 110 111 112def usable_core_count(): 113 """Return the number of cores the current process can use, if supported. 114 Otherwise, return the total number of cores (like `os.cpu_count()`). 115 Default to 1 if undetermined. 116 117 """ 118 try: 119 n = len(os.sched_getaffinity(0)) 120 except AttributeError: 121 n = os.cpu_count() or 1 122 123 # On Windows with more than 60 processes, multiprocessing's call to 124 # _winapi.WaitForMultipleObjects() prints an error and lit hangs. 125 if platform.system() == 'Windows': 126 return min(n, 60) 127 128 return n 129 130 131def mkdir(path): 132 try: 133 if platform.system() == 'Windows': 134 from ctypes import windll 135 from ctypes import GetLastError, WinError 136 137 path = os.path.abspath(path) 138 # Make sure that the path uses backslashes here, in case 139 # python would have happened to use forward slashes, as the 140 # NT path format only supports backslashes. 141 path = path.replace('/', '\\') 142 NTPath = to_unicode(r'\\?\%s' % path) 143 if not windll.kernel32.CreateDirectoryW(NTPath, None): 144 raise WinError(GetLastError()) 145 else: 146 os.mkdir(path) 147 except OSError: 148 e = sys.exc_info()[1] 149 # ignore EEXIST, which may occur during a race condition 150 if e.errno != errno.EEXIST: 151 raise 152 153 154def mkdir_p(path): 155 """mkdir_p(path) - Make the "path" directory, if it does not exist; this 156 will also make directories for any missing parent directories.""" 157 if not path or os.path.exists(path): 158 return 159 160 parent = os.path.dirname(path) 161 if parent != path: 162 mkdir_p(parent) 163 164 mkdir(path) 165 166 167def listdir_files(dirname, suffixes=None, exclude_filenames=None): 168 """Yields files in a directory. 169 170 Filenames that are not excluded by rules below are yielded one at a time, as 171 basenames (i.e., without dirname). 172 173 Files starting with '.' are always skipped. 174 175 If 'suffixes' is not None, then only filenames ending with one of its 176 members will be yielded. These can be extensions, like '.exe', or strings, 177 like 'Test'. (It is a lexicographic check; so an empty sequence will yield 178 nothing, but a single empty string will yield all filenames.) 179 180 If 'exclude_filenames' is not None, then none of the file basenames in it 181 will be yielded. 182 183 If specified, the containers for 'suffixes' and 'exclude_filenames' must 184 support membership checking for strs. 185 186 Args: 187 dirname: a directory path. 188 suffixes: (optional) a sequence of strings (set, list, etc.). 189 exclude_filenames: (optional) a sequence of strings. 190 191 Yields: 192 Filenames as returned by os.listdir (generally, str). 193 194 """ 195 if exclude_filenames is None: 196 exclude_filenames = set() 197 if suffixes is None: 198 suffixes = {''} 199 for filename in os.listdir(dirname): 200 if (os.path.isdir(os.path.join(dirname, filename)) or 201 filename.startswith('.') or 202 filename in exclude_filenames or 203 not any(filename.endswith(sfx) for sfx in suffixes)): 204 continue 205 yield filename 206 207 208def which(command, paths=None): 209 """which(command, [paths]) - Look up the given command in the paths string 210 (or the PATH environment variable, if unspecified).""" 211 212 if paths is None: 213 paths = os.environ.get('PATH', '') 214 215 # Check for absolute match first. 216 if os.path.isabs(command) and os.path.isfile(command): 217 return os.path.normcase(os.path.normpath(command)) 218 219 # Would be nice if Python had a lib function for this. 220 if not paths: 221 paths = os.defpath 222 223 # Get suffixes to search. 224 # On Cygwin, 'PATHEXT' may exist but it should not be used. 225 if os.pathsep == ';': 226 pathext = os.environ.get('PATHEXT', '').split(';') 227 else: 228 pathext = [''] 229 230 # Search the paths... 231 for path in paths.split(os.pathsep): 232 for ext in pathext: 233 p = os.path.join(path, command + ext) 234 if os.path.exists(p) and not os.path.isdir(p): 235 return os.path.normcase(os.path.abspath(p)) 236 237 return None 238 239 240def checkToolsPath(dir, tools): 241 for tool in tools: 242 if not os.path.exists(os.path.join(dir, tool)): 243 return False 244 return True 245 246 247def whichTools(tools, paths): 248 for path in paths.split(os.pathsep): 249 if checkToolsPath(path, tools): 250 return path 251 return None 252 253 254def printHistogram(items, title='Items'): 255 items.sort(key=lambda item: item[1]) 256 257 maxValue = max([v for _, v in items]) 258 259 # Select first "nice" bar height that produces more than 10 bars. 260 power = int(math.ceil(math.log(maxValue, 10))) 261 for inc in itertools.cycle((5, 2, 2.5, 1)): 262 barH = inc * 10**power 263 N = int(math.ceil(maxValue / barH)) 264 if N > 10: 265 break 266 elif inc == 1: 267 power -= 1 268 269 histo = [set() for i in range(N)] 270 for name, v in items: 271 bin = min(int(N * v / maxValue), N - 1) 272 histo[bin].add(name) 273 274 barW = 40 275 hr = '-' * (barW + 34) 276 print('Slowest %s:' % title) 277 print(hr) 278 for name, value in reversed(items[-20:]): 279 print('%.2fs: %s' % (value, name)) 280 print('\n%s Times:' % title) 281 print(hr) 282 pDigits = int(math.ceil(math.log(maxValue, 10))) 283 pfDigits = max(0, 3 - pDigits) 284 if pfDigits: 285 pDigits += pfDigits + 1 286 cDigits = int(math.ceil(math.log(len(items), 10))) 287 print('[%s] :: [%s] :: [%s]' % ('Range'.center((pDigits + 1) * 2 + 3), 288 'Percentage'.center(barW), 289 'Count'.center(cDigits * 2 + 1))) 290 print(hr) 291 for i, row in reversed(list(enumerate(histo))): 292 pct = float(len(row)) / len(items) 293 w = int(barW * pct) 294 print('[%*.*fs,%*.*fs) :: [%s%s] :: [%*d/%*d]' % ( 295 pDigits, pfDigits, i * barH, pDigits, pfDigits, (i + 1) * barH, 296 '*' * w, ' ' * (barW - w), cDigits, len(row), cDigits, len(items))) 297 print(hr) 298 299 300class ExecuteCommandTimeoutException(Exception): 301 def __init__(self, msg, out, err, exitCode): 302 assert isinstance(msg, str) 303 assert isinstance(out, str) 304 assert isinstance(err, str) 305 assert isinstance(exitCode, int) 306 self.msg = msg 307 self.out = out 308 self.err = err 309 self.exitCode = exitCode 310 311 312# Close extra file handles on UNIX (on Windows this cannot be done while 313# also redirecting input). 314kUseCloseFDs = not (platform.system() == 'Windows') 315 316 317def executeCommand(command, cwd=None, env=None, input=None, timeout=0, 318 redirect_stderr=False): 319 """Execute command ``command`` (list of arguments or string) with. 320 321 * working directory ``cwd`` (str), use None to use the current 322 working directory 323 * environment ``env`` (dict), use None for none 324 * Input to the command ``input`` (str), use string to pass 325 no input. 326 * Max execution time ``timeout`` (int) seconds. Use 0 for no timeout. 327 * ``redirect_stderr`` (bool), use True if redirect stderr to stdout 328 329 Returns a tuple (out, err, exitCode) where 330 * ``out`` (str) is the standard output of running the command 331 * ``err`` (str) is the standard error of running the command 332 * ``exitCode`` (int) is the exitCode of running the command 333 334 If the timeout is hit an ``ExecuteCommandTimeoutException`` 335 is raised. 336 337 """ 338 if input is not None: 339 input = to_bytes(input) 340 err_out = subprocess.STDOUT if redirect_stderr else subprocess.PIPE 341 p = subprocess.Popen(command, cwd=cwd, 342 stdin=subprocess.PIPE, 343 stdout=subprocess.PIPE, 344 stderr=err_out, 345 env=env, close_fds=kUseCloseFDs) 346 timerObject = None 347 # FIXME: Because of the way nested function scopes work in Python 2.x we 348 # need to use a reference to a mutable object rather than a plain 349 # bool. In Python 3 we could use the "nonlocal" keyword but we need 350 # to support Python 2 as well. 351 hitTimeOut = [False] 352 try: 353 if timeout > 0: 354 def killProcess(): 355 # We may be invoking a shell so we need to kill the 356 # process and all its children. 357 hitTimeOut[0] = True 358 killProcessAndChildren(p.pid) 359 360 timerObject = threading.Timer(timeout, killProcess) 361 timerObject.start() 362 363 out, err = p.communicate(input=input) 364 exitCode = p.wait() 365 finally: 366 if timerObject != None: 367 timerObject.cancel() 368 369 # Ensure the resulting output is always of string type. 370 out = to_string(out) 371 err = '' if redirect_stderr else to_string(err) 372 373 if hitTimeOut[0]: 374 raise ExecuteCommandTimeoutException( 375 msg='Reached timeout of {} seconds'.format(timeout), 376 out=out, 377 err=err, 378 exitCode=exitCode 379 ) 380 381 # Detect Ctrl-C in subprocess. 382 if exitCode == -signal.SIGINT: 383 raise KeyboardInterrupt 384 385 return out, err, exitCode 386 387 388def isMacOSTriple(target_triple): 389 """Whether the given target triple is for macOS, 390 e.g. x86_64-apple-darwin, arm64-apple-macos 391 """ 392 return 'darwin' in target_triple or 'macos' in target_triple 393 394 395def usePlatformSdkOnDarwin(config, lit_config): 396 # On Darwin, support relocatable SDKs by providing Clang with a 397 # default system root path. 398 if isMacOSTriple(config.target_triple): 399 try: 400 cmd = subprocess.Popen(['xcrun', '--show-sdk-path', '--sdk', 'macosx'], 401 stdout=subprocess.PIPE, stderr=subprocess.PIPE) 402 out, err = cmd.communicate() 403 out = out.strip() 404 res = cmd.wait() 405 except OSError: 406 res = -1 407 if res == 0 and out: 408 sdk_path = out.decode() 409 lit_config.note('using SDKROOT: %r' % sdk_path) 410 config.environment['SDKROOT'] = sdk_path 411 412 413def findPlatformSdkVersionOnMacOS(config, lit_config): 414 if isMacOSTriple(config.target_triple): 415 try: 416 cmd = subprocess.Popen(['xcrun', '--show-sdk-version', '--sdk', 'macosx'], 417 stdout=subprocess.PIPE, stderr=subprocess.PIPE) 418 out, err = cmd.communicate() 419 out = out.strip() 420 res = cmd.wait() 421 except OSError: 422 res = -1 423 if res == 0 and out: 424 return out.decode() 425 return None 426 427def killProcessAndChildrenIsSupported(): 428 """ 429 Returns a tuple (<supported> , <error message>) 430 where 431 `<supported>` is True if `killProcessAndChildren()` is supported on 432 the current host, returns False otherwise. 433 `<error message>` is an empty string if `<supported>` is True, 434 otherwise is contains a string describing why the function is 435 not supported. 436 """ 437 if platform.system() == 'AIX': 438 return (True, "") 439 try: 440 import psutil # noqa: F401 441 return (True, "") 442 except ImportError: 443 return (False, "Requires the Python psutil module but it could" 444 " not be found. Try installing it via pip or via" 445 " your operating system's package manager.") 446 447def killProcessAndChildren(pid): 448 """This function kills a process with ``pid`` and all its running children 449 (recursively). It is currently implemented using the psutil module on some 450 platforms which provides a simple platform neutral implementation. 451 452 TODO: Reimplement this without using psutil on all platforms so we can 453 remove our dependency on it. 454 455 """ 456 if platform.system() == 'AIX': 457 subprocess.call('kill -kill $(ps -o pid= -L{})'.format(pid), shell=True) 458 else: 459 import psutil 460 try: 461 psutilProc = psutil.Process(pid) 462 # Handle the different psutil API versions 463 try: 464 # psutil >= 2.x 465 children_iterator = psutilProc.children(recursive=True) 466 except AttributeError: 467 # psutil 1.x 468 children_iterator = psutilProc.get_children(recursive=True) 469 for child in children_iterator: 470 try: 471 child.kill() 472 except psutil.NoSuchProcess: 473 pass 474 psutilProc.kill() 475 except psutil.NoSuchProcess: 476 pass 477