1import errno 2import itertools 3import math 4import numbers 5import os 6import platform 7import signal 8import subprocess 9import sys 10import threading 11 12 13def norm_path(path): 14 path = os.path.realpath(path) 15 path = os.path.normpath(path) 16 path = os.path.normcase(path) 17 return path 18 19 20def is_string(value): 21 try: 22 # Python 2 and Python 3 are different here. 23 return isinstance(value, basestring) 24 except NameError: 25 return isinstance(value, str) 26 27 28def pythonize_bool(value): 29 if value is None: 30 return False 31 if type(value) is bool: 32 return value 33 if isinstance(value, numbers.Number): 34 return value != 0 35 if is_string(value): 36 if value.lower() in ('1', 'true', 'on', 'yes'): 37 return True 38 if value.lower() in ('', '0', 'false', 'off', 'no'): 39 return False 40 raise ValueError('"{}" is not a valid boolean'.format(value)) 41 42 43def make_word_regex(word): 44 return r'\b' + word + r'\b' 45 46 47def to_bytes(s): 48 """Return the parameter as type 'bytes', possibly encoding it. 49 50 In Python2, the 'bytes' type is the same as 'str'. In Python3, they 51 are distinct. 52 53 """ 54 if isinstance(s, bytes): 55 # In Python2, this branch is taken for both 'str' and 'bytes'. 56 # In Python3, this branch is taken only for 'bytes'. 57 return s 58 # In Python2, 's' is a 'unicode' object. 59 # In Python3, 's' is a 'str' object. 60 # Encode to UTF-8 to get 'bytes' data. 61 return s.encode('utf-8') 62 63 64def to_string(b): 65 """Return the parameter as type 'str', possibly encoding it. 66 67 In Python2, the 'str' type is the same as 'bytes'. In Python3, the 68 'str' type is (essentially) Python2's 'unicode' type, and 'bytes' is 69 distinct. 70 71 """ 72 if isinstance(b, str): 73 # In Python2, this branch is taken for types 'str' and 'bytes'. 74 # In Python3, this branch is taken only for 'str'. 75 return b 76 if isinstance(b, bytes): 77 # In Python2, this branch is never taken ('bytes' is handled as 'str'). 78 # In Python3, this is true only for 'bytes'. 79 try: 80 return b.decode('utf-8') 81 except UnicodeDecodeError: 82 # If the value is not valid Unicode, return the default 83 # repr-line encoding. 84 return str(b) 85 86 # By this point, here's what we *don't* have: 87 # 88 # - In Python2: 89 # - 'str' or 'bytes' (1st branch above) 90 # - In Python3: 91 # - 'str' (1st branch above) 92 # - 'bytes' (2nd branch above) 93 # 94 # The last type we might expect is the Python2 'unicode' type. There is no 95 # 'unicode' type in Python3 (all the Python3 cases were already handled). In 96 # order to get a 'str' object, we need to encode the 'unicode' object. 97 try: 98 return b.encode('utf-8') 99 except AttributeError: 100 raise TypeError('not sure how to convert %s to %s' % (type(b), str)) 101 102 103def detectCPUs(): 104 """Detects the number of CPUs on a system. 105 106 Cribbed from pp. 107 108 """ 109 # Linux, Unix and MacOS: 110 if hasattr(os, 'sysconf'): 111 if 'SC_NPROCESSORS_ONLN' in os.sysconf_names: 112 # Linux & Unix: 113 ncpus = os.sysconf('SC_NPROCESSORS_ONLN') 114 if isinstance(ncpus, int) and ncpus > 0: 115 return ncpus 116 else: # OSX: 117 return int(subprocess.check_output(['sysctl', '-n', 'hw.ncpu'], 118 stderr=subprocess.STDOUT)) 119 # Windows: 120 if 'NUMBER_OF_PROCESSORS' in os.environ: 121 ncpus = int(os.environ['NUMBER_OF_PROCESSORS']) 122 if ncpus > 0: 123 # With more than 32 processes, process creation often fails with 124 # "Too many open files". FIXME: Check if there's a better fix. 125 return min(ncpus, 32) 126 return 1 # Default 127 128 129def mkdir_p(path): 130 """mkdir_p(path) - Make the "path" directory, if it does not exist; this 131 will also make directories for any missing parent directories.""" 132 if not path or os.path.exists(path): 133 return 134 135 parent = os.path.dirname(path) 136 if parent != path: 137 mkdir_p(parent) 138 139 try: 140 os.mkdir(path) 141 except OSError: 142 e = sys.exc_info()[1] 143 # Ignore EEXIST, which may occur during a race condition. 144 if e.errno != errno.EEXIST: 145 raise 146 147 148def listdir_files(dirname, suffixes=None, exclude_filenames=None): 149 """Yields files in a directory. 150 151 Filenames that are not excluded by rules below are yielded one at a time, as 152 basenames (i.e., without dirname). 153 154 Files starting with '.' are always skipped. 155 156 If 'suffixes' is not None, then only filenames ending with one of its 157 members will be yielded. These can be extensions, like '.exe', or strings, 158 like 'Test'. (It is a lexicographic check; so an empty sequence will yield 159 nothing, but a single empty string will yield all filenames.) 160 161 If 'exclude_filenames' is not None, then none of the file basenames in it 162 will be yielded. 163 164 If specified, the containers for 'suffixes' and 'exclude_filenames' must 165 support membership checking for strs. 166 167 Args: 168 dirname: a directory path. 169 suffixes: (optional) a sequence of strings (set, list, etc.). 170 exclude_filenames: (optional) a sequence of strings. 171 172 Yields: 173 Filenames as returned by os.listdir (generally, str). 174 175 """ 176 if exclude_filenames is None: 177 exclude_filenames = set() 178 if suffixes is None: 179 suffixes = {''} 180 for filename in os.listdir(dirname): 181 if (os.path.isdir(os.path.join(dirname, filename)) or 182 filename.startswith('.') or 183 filename in exclude_filenames or 184 not any(filename.endswith(sfx) for sfx in suffixes)): 185 continue 186 yield filename 187 188 189def which(command, paths=None): 190 """which(command, [paths]) - Look up the given command in the paths string 191 (or the PATH environment variable, if unspecified).""" 192 193 if paths is None: 194 paths = os.environ.get('PATH', '') 195 196 # Check for absolute match first. 197 if os.path.isabs(command) and os.path.isfile(command): 198 return os.path.normpath(command) 199 200 # Would be nice if Python had a lib function for this. 201 if not paths: 202 paths = os.defpath 203 204 # Get suffixes to search. 205 # On Cygwin, 'PATHEXT' may exist but it should not be used. 206 if os.pathsep == ';': 207 pathext = os.environ.get('PATHEXT', '').split(';') 208 else: 209 pathext = [''] 210 211 # Search the paths... 212 for path in paths.split(os.pathsep): 213 for ext in pathext: 214 p = os.path.join(path, command + ext) 215 if os.path.exists(p) and not os.path.isdir(p): 216 return os.path.normpath(p) 217 218 return None 219 220 221def checkToolsPath(dir, tools): 222 for tool in tools: 223 if not os.path.exists(os.path.join(dir, tool)): 224 return False 225 return True 226 227 228def whichTools(tools, paths): 229 for path in paths.split(os.pathsep): 230 if checkToolsPath(path, tools): 231 return path 232 return None 233 234 235def printHistogram(items, title='Items'): 236 items.sort(key=lambda item: item[1]) 237 238 maxValue = max([v for _, v in items]) 239 240 # Select first "nice" bar height that produces more than 10 bars. 241 power = int(math.ceil(math.log(maxValue, 10))) 242 for inc in itertools.cycle((5, 2, 2.5, 1)): 243 barH = inc * 10**power 244 N = int(math.ceil(maxValue / barH)) 245 if N > 10: 246 break 247 elif inc == 1: 248 power -= 1 249 250 histo = [set() for i in range(N)] 251 for name, v in items: 252 bin = min(int(N * v / maxValue), N - 1) 253 histo[bin].add(name) 254 255 barW = 40 256 hr = '-' * (barW + 34) 257 print('\nSlowest %s:' % title) 258 print(hr) 259 for name, value in items[-20:]: 260 print('%.2fs: %s' % (value, name)) 261 print('\n%s Times:' % title) 262 print(hr) 263 pDigits = int(math.ceil(math.log(maxValue, 10))) 264 pfDigits = max(0, 3 - pDigits) 265 if pfDigits: 266 pDigits += pfDigits + 1 267 cDigits = int(math.ceil(math.log(len(items), 10))) 268 print('[%s] :: [%s] :: [%s]' % ('Range'.center((pDigits + 1) * 2 + 3), 269 'Percentage'.center(barW), 270 'Count'.center(cDigits * 2 + 1))) 271 print(hr) 272 for i, row in enumerate(histo): 273 pct = float(len(row)) / len(items) 274 w = int(barW * pct) 275 print('[%*.*fs,%*.*fs) :: [%s%s] :: [%*d/%*d]' % ( 276 pDigits, pfDigits, i * barH, pDigits, pfDigits, (i + 1) * barH, 277 '*' * w, ' ' * (barW - w), cDigits, len(row), cDigits, len(items))) 278 279 280class ExecuteCommandTimeoutException(Exception): 281 def __init__(self, msg, out, err, exitCode): 282 assert isinstance(msg, str) 283 assert isinstance(out, str) 284 assert isinstance(err, str) 285 assert isinstance(exitCode, int) 286 self.msg = msg 287 self.out = out 288 self.err = err 289 self.exitCode = exitCode 290 291 292# Close extra file handles on UNIX (on Windows this cannot be done while 293# also redirecting input). 294kUseCloseFDs = not (platform.system() == 'Windows') 295 296 297def executeCommand(command, cwd=None, env=None, input=None, timeout=0): 298 """Execute command ``command`` (list of arguments or string) with. 299 300 * working directory ``cwd`` (str), use None to use the current 301 working directory 302 * environment ``env`` (dict), use None for none 303 * Input to the command ``input`` (str), use string to pass 304 no input. 305 * Max execution time ``timeout`` (int) seconds. Use 0 for no timeout. 306 307 Returns a tuple (out, err, exitCode) where 308 * ``out`` (str) is the standard output of running the command 309 * ``err`` (str) is the standard error of running the command 310 * ``exitCode`` (int) is the exitCode of running the command 311 312 If the timeout is hit an ``ExecuteCommandTimeoutException`` 313 is raised. 314 315 """ 316 if input is not None: 317 input = to_bytes(input) 318 p = subprocess.Popen(command, cwd=cwd, 319 stdin=subprocess.PIPE, 320 stdout=subprocess.PIPE, 321 stderr=subprocess.PIPE, 322 env=env, close_fds=kUseCloseFDs) 323 timerObject = None 324 # FIXME: Because of the way nested function scopes work in Python 2.x we 325 # need to use a reference to a mutable object rather than a plain 326 # bool. In Python 3 we could use the "nonlocal" keyword but we need 327 # to support Python 2 as well. 328 hitTimeOut = [False] 329 try: 330 if timeout > 0: 331 def killProcess(): 332 # We may be invoking a shell so we need to kill the 333 # process and all its children. 334 hitTimeOut[0] = True 335 killProcessAndChildren(p.pid) 336 337 timerObject = threading.Timer(timeout, killProcess) 338 timerObject.start() 339 340 out, err = p.communicate(input=input) 341 exitCode = p.wait() 342 finally: 343 if timerObject != None: 344 timerObject.cancel() 345 346 # Ensure the resulting output is always of string type. 347 out = to_string(out) 348 err = to_string(err) 349 350 if hitTimeOut[0]: 351 raise ExecuteCommandTimeoutException( 352 msg='Reached timeout of {} seconds'.format(timeout), 353 out=out, 354 err=err, 355 exitCode=exitCode 356 ) 357 358 # Detect Ctrl-C in subprocess. 359 if exitCode == -signal.SIGINT: 360 raise KeyboardInterrupt 361 362 return out, err, exitCode 363 364 365def usePlatformSdkOnDarwin(config, lit_config): 366 # On Darwin, support relocatable SDKs by providing Clang with a 367 # default system root path. 368 if 'darwin' in config.target_triple: 369 try: 370 cmd = subprocess.Popen(['xcrun', '--show-sdk-path', '--sdk', 'macosx'], 371 stdout=subprocess.PIPE, stderr=subprocess.PIPE) 372 out, err = cmd.communicate() 373 out = out.strip() 374 res = cmd.wait() 375 except OSError: 376 res = -1 377 if res == 0 and out: 378 sdk_path = out 379 lit_config.note('using SDKROOT: %r' % sdk_path) 380 config.environment['SDKROOT'] = sdk_path 381 382 383def findPlatformSdkVersionOnMacOS(config, lit_config): 384 if 'darwin' in config.target_triple: 385 try: 386 cmd = subprocess.Popen(['xcrun', '--show-sdk-version', '--sdk', 'macosx'], 387 stdout=subprocess.PIPE, stderr=subprocess.PIPE) 388 out, err = cmd.communicate() 389 out = out.strip() 390 res = cmd.wait() 391 except OSError: 392 res = -1 393 if res == 0 and out: 394 return out 395 return None 396 397 398def killProcessAndChildren(pid): 399 """This function kills a process with ``pid`` and all its running children 400 (recursively). It is currently implemented using the psutil module which 401 provides a simple platform neutral implementation. 402 403 TODO: Reimplement this without using psutil so we can remove 404 our dependency on it. 405 406 """ 407 import psutil 408 try: 409 psutilProc = psutil.Process(pid) 410 # Handle the different psutil API versions 411 try: 412 # psutil >= 2.x 413 children_iterator = psutilProc.children(recursive=True) 414 except AttributeError: 415 # psutil 1.x 416 children_iterator = psutilProc.get_children(recursive=True) 417 for child in children_iterator: 418 try: 419 child.kill() 420 except psutil.NoSuchProcess: 421 pass 422 psutilProc.kill() 423 except psutil.NoSuchProcess: 424 pass 425