1from __future__ import print_function 2 3import errno 4import itertools 5import math 6import numbers 7import os 8import platform 9import signal 10import subprocess 11import sys 12import threading 13 14 15def norm_path(path): 16 path = os.path.realpath(path) 17 path = os.path.normpath(path) 18 path = os.path.normcase(path) 19 return path 20 21 22def is_string(value): 23 try: 24 # Python 2 and Python 3 are different here. 25 return isinstance(value, basestring) 26 except NameError: 27 return isinstance(value, str) 28 29 30def pythonize_bool(value): 31 if value is None: 32 return False 33 if type(value) is bool: 34 return value 35 if isinstance(value, numbers.Number): 36 return value != 0 37 if is_string(value): 38 if value.lower() in ('1', 'true', 'on', 'yes'): 39 return True 40 if value.lower() in ('', '0', 'false', 'off', 'no'): 41 return False 42 raise ValueError('"{}" is not a valid boolean'.format(value)) 43 44 45def make_word_regex(word): 46 return r'\b' + word + r'\b' 47 48 49def to_bytes(s): 50 """Return the parameter as type 'bytes', possibly encoding it. 51 52 In Python2, the 'bytes' type is the same as 'str'. In Python3, they 53 are distinct. 54 55 """ 56 if isinstance(s, bytes): 57 # In Python2, this branch is taken for both 'str' and 'bytes'. 58 # In Python3, this branch is taken only for 'bytes'. 59 return s 60 # In Python2, 's' is a 'unicode' object. 61 # In Python3, 's' is a 'str' object. 62 # Encode to UTF-8 to get 'bytes' data. 63 return s.encode('utf-8') 64 65 66def to_string(b): 67 """Return the parameter as type 'str', possibly encoding it. 68 69 In Python2, the 'str' type is the same as 'bytes'. In Python3, the 70 'str' type is (essentially) Python2's 'unicode' type, and 'bytes' is 71 distinct. 72 73 """ 74 if isinstance(b, str): 75 # In Python2, this branch is taken for types 'str' and 'bytes'. 76 # In Python3, this branch is taken only for 'str'. 77 return b 78 if isinstance(b, bytes): 79 # In Python2, this branch is never taken ('bytes' is handled as 'str'). 80 # In Python3, this is true only for 'bytes'. 81 try: 82 return b.decode('utf-8') 83 except UnicodeDecodeError: 84 # If the value is not valid Unicode, return the default 85 # repr-line encoding. 86 return str(b) 87 88 # By this point, here's what we *don't* have: 89 # 90 # - In Python2: 91 # - 'str' or 'bytes' (1st branch above) 92 # - In Python3: 93 # - 'str' (1st branch above) 94 # - 'bytes' (2nd branch above) 95 # 96 # The last type we might expect is the Python2 'unicode' type. There is no 97 # 'unicode' type in Python3 (all the Python3 cases were already handled). In 98 # order to get a 'str' object, we need to encode the 'unicode' object. 99 try: 100 return b.encode('utf-8') 101 except AttributeError: 102 raise TypeError('not sure how to convert %s to %s' % (type(b), str)) 103 104 105def to_unicode(s): 106 """Return the parameter as type which supports unicode, possibly decoding 107 it. 108 109 In Python2, this is the unicode type. In Python3 it's the str type. 110 111 """ 112 if isinstance(s, bytes): 113 # In Python2, this branch is taken for both 'str' and 'bytes'. 114 # In Python3, this branch is taken only for 'bytes'. 115 return s.decode('utf-8') 116 return s 117 118 119def detectCPUs(): 120 """Detects the number of CPUs on a system. 121 122 Cribbed from pp. 123 124 """ 125 # Linux, Unix and MacOS: 126 if hasattr(os, 'sysconf'): 127 if 'SC_NPROCESSORS_ONLN' in os.sysconf_names: 128 # Linux & Unix: 129 ncpus = os.sysconf('SC_NPROCESSORS_ONLN') 130 if isinstance(ncpus, int) and ncpus > 0: 131 return ncpus 132 else: # OSX: 133 return int(subprocess.check_output(['sysctl', '-n', 'hw.ncpu'], 134 stderr=subprocess.STDOUT)) 135 # Windows: 136 if 'NUMBER_OF_PROCESSORS' in os.environ: 137 ncpus = int(os.environ['NUMBER_OF_PROCESSORS']) 138 if ncpus > 0: 139 # With more than 32 processes, process creation often fails with 140 # "Too many open files". FIXME: Check if there's a better fix. 141 return min(ncpus, 32) 142 return 1 # Default 143 144 145def mkdir(path): 146 try: 147 if platform.system() == 'Windows': 148 from ctypes import windll 149 from ctypes import GetLastError, WinError 150 151 path = os.path.abspath(path) 152 NTPath = to_unicode(r'\\?\%s' % path) 153 if not windll.kernel32.CreateDirectoryW(NTPath, None): 154 raise WinError(GetLastError()) 155 else: 156 os.mkdir(path) 157 except OSError: 158 e = sys.exc_info()[1] 159 # ignore EEXIST, which may occur during a race condition 160 if e.errno != errno.EEXIST: 161 raise 162 163 164def mkdir_p(path): 165 """mkdir_p(path) - Make the "path" directory, if it does not exist; this 166 will also make directories for any missing parent directories.""" 167 if not path or os.path.exists(path): 168 return 169 170 parent = os.path.dirname(path) 171 if parent != path: 172 mkdir_p(parent) 173 174 mkdir(path) 175 176 177def listdir_files(dirname, suffixes=None, exclude_filenames=None): 178 """Yields files in a directory. 179 180 Filenames that are not excluded by rules below are yielded one at a time, as 181 basenames (i.e., without dirname). 182 183 Files starting with '.' are always skipped. 184 185 If 'suffixes' is not None, then only filenames ending with one of its 186 members will be yielded. These can be extensions, like '.exe', or strings, 187 like 'Test'. (It is a lexicographic check; so an empty sequence will yield 188 nothing, but a single empty string will yield all filenames.) 189 190 If 'exclude_filenames' is not None, then none of the file basenames in it 191 will be yielded. 192 193 If specified, the containers for 'suffixes' and 'exclude_filenames' must 194 support membership checking for strs. 195 196 Args: 197 dirname: a directory path. 198 suffixes: (optional) a sequence of strings (set, list, etc.). 199 exclude_filenames: (optional) a sequence of strings. 200 201 Yields: 202 Filenames as returned by os.listdir (generally, str). 203 204 """ 205 if exclude_filenames is None: 206 exclude_filenames = set() 207 if suffixes is None: 208 suffixes = {''} 209 for filename in os.listdir(dirname): 210 if (os.path.isdir(os.path.join(dirname, filename)) or 211 filename.startswith('.') or 212 filename in exclude_filenames or 213 not any(filename.endswith(sfx) for sfx in suffixes)): 214 continue 215 yield filename 216 217 218def which(command, paths=None): 219 """which(command, [paths]) - Look up the given command in the paths string 220 (or the PATH environment variable, if unspecified).""" 221 222 if paths is None: 223 paths = os.environ.get('PATH', '') 224 225 # Check for absolute match first. 226 if os.path.isabs(command) and os.path.isfile(command): 227 return os.path.normcase(os.path.normpath(command)) 228 229 # Would be nice if Python had a lib function for this. 230 if not paths: 231 paths = os.defpath 232 233 # Get suffixes to search. 234 # On Cygwin, 'PATHEXT' may exist but it should not be used. 235 if os.pathsep == ';': 236 pathext = os.environ.get('PATHEXT', '').split(';') 237 else: 238 pathext = [''] 239 240 # Search the paths... 241 for path in paths.split(os.pathsep): 242 for ext in pathext: 243 p = os.path.join(path, command + ext) 244 if os.path.exists(p) and not os.path.isdir(p): 245 return os.path.normcase(os.path.normpath(p)) 246 247 return None 248 249 250def checkToolsPath(dir, tools): 251 for tool in tools: 252 if not os.path.exists(os.path.join(dir, tool)): 253 return False 254 return True 255 256 257def whichTools(tools, paths): 258 for path in paths.split(os.pathsep): 259 if checkToolsPath(path, tools): 260 return path 261 return None 262 263 264def printHistogram(items, title='Items'): 265 items.sort(key=lambda item: item[1]) 266 267 maxValue = max([v for _, v in items]) 268 269 # Select first "nice" bar height that produces more than 10 bars. 270 power = int(math.ceil(math.log(maxValue, 10))) 271 for inc in itertools.cycle((5, 2, 2.5, 1)): 272 barH = inc * 10**power 273 N = int(math.ceil(maxValue / barH)) 274 if N > 10: 275 break 276 elif inc == 1: 277 power -= 1 278 279 histo = [set() for i in range(N)] 280 for name, v in items: 281 bin = min(int(N * v / maxValue), N - 1) 282 histo[bin].add(name) 283 284 barW = 40 285 hr = '-' * (barW + 34) 286 print('\nSlowest %s:' % title) 287 print(hr) 288 for name, value in items[-20:]: 289 print('%.2fs: %s' % (value, name)) 290 print('\n%s Times:' % title) 291 print(hr) 292 pDigits = int(math.ceil(math.log(maxValue, 10))) 293 pfDigits = max(0, 3 - pDigits) 294 if pfDigits: 295 pDigits += pfDigits + 1 296 cDigits = int(math.ceil(math.log(len(items), 10))) 297 print('[%s] :: [%s] :: [%s]' % ('Range'.center((pDigits + 1) * 2 + 3), 298 'Percentage'.center(barW), 299 'Count'.center(cDigits * 2 + 1))) 300 print(hr) 301 for i, row in enumerate(histo): 302 pct = float(len(row)) / len(items) 303 w = int(barW * pct) 304 print('[%*.*fs,%*.*fs) :: [%s%s] :: [%*d/%*d]' % ( 305 pDigits, pfDigits, i * barH, pDigits, pfDigits, (i + 1) * barH, 306 '*' * w, ' ' * (barW - w), cDigits, len(row), cDigits, len(items))) 307 308 309class ExecuteCommandTimeoutException(Exception): 310 def __init__(self, msg, out, err, exitCode): 311 assert isinstance(msg, str) 312 assert isinstance(out, str) 313 assert isinstance(err, str) 314 assert isinstance(exitCode, int) 315 self.msg = msg 316 self.out = out 317 self.err = err 318 self.exitCode = exitCode 319 320 321# Close extra file handles on UNIX (on Windows this cannot be done while 322# also redirecting input). 323kUseCloseFDs = not (platform.system() == 'Windows') 324 325 326def executeCommand(command, cwd=None, env=None, input=None, timeout=0): 327 """Execute command ``command`` (list of arguments or string) with. 328 329 * working directory ``cwd`` (str), use None to use the current 330 working directory 331 * environment ``env`` (dict), use None for none 332 * Input to the command ``input`` (str), use string to pass 333 no input. 334 * Max execution time ``timeout`` (int) seconds. Use 0 for no timeout. 335 336 Returns a tuple (out, err, exitCode) where 337 * ``out`` (str) is the standard output of running the command 338 * ``err`` (str) is the standard error of running the command 339 * ``exitCode`` (int) is the exitCode of running the command 340 341 If the timeout is hit an ``ExecuteCommandTimeoutException`` 342 is raised. 343 344 """ 345 if input is not None: 346 input = to_bytes(input) 347 p = subprocess.Popen(command, cwd=cwd, 348 stdin=subprocess.PIPE, 349 stdout=subprocess.PIPE, 350 stderr=subprocess.PIPE, 351 env=env, close_fds=kUseCloseFDs) 352 timerObject = None 353 # FIXME: Because of the way nested function scopes work in Python 2.x we 354 # need to use a reference to a mutable object rather than a plain 355 # bool. In Python 3 we could use the "nonlocal" keyword but we need 356 # to support Python 2 as well. 357 hitTimeOut = [False] 358 try: 359 if timeout > 0: 360 def killProcess(): 361 # We may be invoking a shell so we need to kill the 362 # process and all its children. 363 hitTimeOut[0] = True 364 killProcessAndChildren(p.pid) 365 366 timerObject = threading.Timer(timeout, killProcess) 367 timerObject.start() 368 369 out, err = p.communicate(input=input) 370 exitCode = p.wait() 371 finally: 372 if timerObject != None: 373 timerObject.cancel() 374 375 # Ensure the resulting output is always of string type. 376 out = to_string(out) 377 err = to_string(err) 378 379 if hitTimeOut[0]: 380 raise ExecuteCommandTimeoutException( 381 msg='Reached timeout of {} seconds'.format(timeout), 382 out=out, 383 err=err, 384 exitCode=exitCode 385 ) 386 387 # Detect Ctrl-C in subprocess. 388 if exitCode == -signal.SIGINT: 389 raise KeyboardInterrupt 390 391 return out, err, exitCode 392 393 394def usePlatformSdkOnDarwin(config, lit_config): 395 # On Darwin, support relocatable SDKs by providing Clang with a 396 # default system root path. 397 if 'darwin' in config.target_triple: 398 try: 399 cmd = subprocess.Popen(['xcrun', '--show-sdk-path', '--sdk', 'macosx'], 400 stdout=subprocess.PIPE, stderr=subprocess.PIPE) 401 out, err = cmd.communicate() 402 out = out.strip() 403 res = cmd.wait() 404 except OSError: 405 res = -1 406 if res == 0 and out: 407 sdk_path = out.decode() 408 lit_config.note('using SDKROOT: %r' % sdk_path) 409 config.environment['SDKROOT'] = sdk_path 410 411 412def findPlatformSdkVersionOnMacOS(config, lit_config): 413 if 'darwin' in config.target_triple: 414 try: 415 cmd = subprocess.Popen(['xcrun', '--show-sdk-version', '--sdk', 'macosx'], 416 stdout=subprocess.PIPE, stderr=subprocess.PIPE) 417 out, err = cmd.communicate() 418 out = out.strip() 419 res = cmd.wait() 420 except OSError: 421 res = -1 422 if res == 0 and out: 423 return out.decode() 424 return None 425 426 427def killProcessAndChildren(pid): 428 """This function kills a process with ``pid`` and all its running children 429 (recursively). It is currently implemented using the psutil module which 430 provides a simple platform neutral implementation. 431 432 TODO: Reimplement this without using psutil so we can remove 433 our dependency on it. 434 435 """ 436 import psutil 437 try: 438 psutilProc = psutil.Process(pid) 439 # Handle the different psutil API versions 440 try: 441 # psutil >= 2.x 442 children_iterator = psutilProc.children(recursive=True) 443 except AttributeError: 444 # psutil 1.x 445 children_iterator = psutilProc.get_children(recursive=True) 446 for child in children_iterator: 447 try: 448 child.kill() 449 except psutil.NoSuchProcess: 450 pass 451 psutilProc.kill() 452 except psutil.NoSuchProcess: 453 pass 454 455 456try: 457 import win32api 458except ImportError: 459 win32api = None 460 461def abort_now(): 462 """Abort the current process without doing any exception teardown""" 463 sys.stdout.flush() 464 if win32api: 465 win32api.TerminateProcess(win32api.GetCurrentProcess(), 3) 466 else: 467 os.kill(0, 9) 468