1from __future__ import print_function
2
3import errno
4import itertools
5import math
6import numbers
7import os
8import platform
9import signal
10import subprocess
11import sys
12import threading
13
14
15def norm_path(path):
16    path = os.path.realpath(path)
17    path = os.path.normpath(path)
18    path = os.path.normcase(path)
19    return path
20
21
22def is_string(value):
23    try:
24        # Python 2 and Python 3 are different here.
25        return isinstance(value, basestring)
26    except NameError:
27        return isinstance(value, str)
28
29
30def pythonize_bool(value):
31    if value is None:
32        return False
33    if type(value) is bool:
34        return value
35    if isinstance(value, numbers.Number):
36        return value != 0
37    if is_string(value):
38        if value.lower() in ('1', 'true', 'on', 'yes'):
39            return True
40        if value.lower() in ('', '0', 'false', 'off', 'no'):
41            return False
42    raise ValueError('"{}" is not a valid boolean'.format(value))
43
44
45def make_word_regex(word):
46    return r'\b' + word + r'\b'
47
48
49def to_bytes(s):
50    """Return the parameter as type 'bytes', possibly encoding it.
51
52    In Python2, the 'bytes' type is the same as 'str'. In Python3, they
53    are distinct.
54
55    """
56    if isinstance(s, bytes):
57        # In Python2, this branch is taken for both 'str' and 'bytes'.
58        # In Python3, this branch is taken only for 'bytes'.
59        return s
60    # In Python2, 's' is a 'unicode' object.
61    # In Python3, 's' is a 'str' object.
62    # Encode to UTF-8 to get 'bytes' data.
63    return s.encode('utf-8')
64
65
66def to_string(b):
67    """Return the parameter as type 'str', possibly encoding it.
68
69    In Python2, the 'str' type is the same as 'bytes'. In Python3, the
70    'str' type is (essentially) Python2's 'unicode' type, and 'bytes' is
71    distinct.
72
73    """
74    if isinstance(b, str):
75        # In Python2, this branch is taken for types 'str' and 'bytes'.
76        # In Python3, this branch is taken only for 'str'.
77        return b
78    if isinstance(b, bytes):
79        # In Python2, this branch is never taken ('bytes' is handled as 'str').
80        # In Python3, this is true only for 'bytes'.
81        try:
82            return b.decode('utf-8')
83        except UnicodeDecodeError:
84            # If the value is not valid Unicode, return the default
85            # repr-line encoding.
86            return str(b)
87
88    # By this point, here's what we *don't* have:
89    #
90    #  - In Python2:
91    #    - 'str' or 'bytes' (1st branch above)
92    #  - In Python3:
93    #    - 'str' (1st branch above)
94    #    - 'bytes' (2nd branch above)
95    #
96    # The last type we might expect is the Python2 'unicode' type. There is no
97    # 'unicode' type in Python3 (all the Python3 cases were already handled). In
98    # order to get a 'str' object, we need to encode the 'unicode' object.
99    try:
100        return b.encode('utf-8')
101    except AttributeError:
102        raise TypeError('not sure how to convert %s to %s' % (type(b), str))
103
104
105def to_unicode(s):
106    """Return the parameter as type which supports unicode, possibly decoding
107    it.
108
109    In Python2, this is the unicode type. In Python3 it's the str type.
110
111    """
112    if isinstance(s, bytes):
113        # In Python2, this branch is taken for both 'str' and 'bytes'.
114        # In Python3, this branch is taken only for 'bytes'.
115        return s.decode('utf-8')
116    return s
117
118
119def detectCPUs():
120    """Detects the number of CPUs on a system.
121
122    Cribbed from pp.
123
124    """
125    # Linux, Unix and MacOS:
126    if hasattr(os, 'sysconf'):
127        if 'SC_NPROCESSORS_ONLN' in os.sysconf_names:
128            # Linux & Unix:
129            ncpus = os.sysconf('SC_NPROCESSORS_ONLN')
130            if isinstance(ncpus, int) and ncpus > 0:
131                return ncpus
132        else:  # OSX:
133            return int(subprocess.check_output(['sysctl', '-n', 'hw.ncpu'],
134                                               stderr=subprocess.STDOUT))
135    # Windows:
136    if 'NUMBER_OF_PROCESSORS' in os.environ:
137        ncpus = int(os.environ['NUMBER_OF_PROCESSORS'])
138        if ncpus > 0:
139            # With more than 32 processes, process creation often fails with
140            # "Too many open files".  FIXME: Check if there's a better fix.
141            return min(ncpus, 32)
142    return 1  # Default
143
144
145def mkdir(path):
146    try:
147        if platform.system() == 'Windows':
148            from ctypes import windll
149            from ctypes import GetLastError, WinError
150
151            path = os.path.abspath(path)
152            NTPath = to_unicode(r'\\?\%s' % path)
153            if not windll.kernel32.CreateDirectoryW(NTPath, None):
154                raise WinError(GetLastError())
155        else:
156            os.mkdir(path)
157    except OSError:
158        e = sys.exc_info()[1]
159        # ignore EEXIST, which may occur during a race condition
160        if e.errno != errno.EEXIST:
161            raise
162
163
164def mkdir_p(path):
165    """mkdir_p(path) - Make the "path" directory, if it does not exist; this
166    will also make directories for any missing parent directories."""
167    if not path or os.path.exists(path):
168        return
169
170    parent = os.path.dirname(path)
171    if parent != path:
172        mkdir_p(parent)
173
174    mkdir(path)
175
176
177def listdir_files(dirname, suffixes=None, exclude_filenames=None):
178    """Yields files in a directory.
179
180    Filenames that are not excluded by rules below are yielded one at a time, as
181    basenames (i.e., without dirname).
182
183    Files starting with '.' are always skipped.
184
185    If 'suffixes' is not None, then only filenames ending with one of its
186    members will be yielded. These can be extensions, like '.exe', or strings,
187    like 'Test'. (It is a lexicographic check; so an empty sequence will yield
188    nothing, but a single empty string will yield all filenames.)
189
190    If 'exclude_filenames' is not None, then none of the file basenames in it
191    will be yielded.
192
193    If specified, the containers for 'suffixes' and 'exclude_filenames' must
194    support membership checking for strs.
195
196    Args:
197        dirname: a directory path.
198        suffixes: (optional) a sequence of strings (set, list, etc.).
199        exclude_filenames: (optional) a sequence of strings.
200
201    Yields:
202        Filenames as returned by os.listdir (generally, str).
203
204    """
205    if exclude_filenames is None:
206        exclude_filenames = set()
207    if suffixes is None:
208        suffixes = {''}
209    for filename in os.listdir(dirname):
210        if (os.path.isdir(os.path.join(dirname, filename)) or
211            filename.startswith('.') or
212            filename in exclude_filenames or
213                not any(filename.endswith(sfx) for sfx in suffixes)):
214            continue
215        yield filename
216
217
218def which(command, paths=None):
219    """which(command, [paths]) - Look up the given command in the paths string
220    (or the PATH environment variable, if unspecified)."""
221
222    if paths is None:
223        paths = os.environ.get('PATH', '')
224
225    # Check for absolute match first.
226    if os.path.isabs(command) and os.path.isfile(command):
227        return os.path.normcase(os.path.normpath(command))
228
229    # Would be nice if Python had a lib function for this.
230    if not paths:
231        paths = os.defpath
232
233    # Get suffixes to search.
234    # On Cygwin, 'PATHEXT' may exist but it should not be used.
235    if os.pathsep == ';':
236        pathext = os.environ.get('PATHEXT', '').split(';')
237    else:
238        pathext = ['']
239
240    # Search the paths...
241    for path in paths.split(os.pathsep):
242        for ext in pathext:
243            p = os.path.join(path, command + ext)
244            if os.path.exists(p) and not os.path.isdir(p):
245                return os.path.normcase(os.path.normpath(p))
246
247    return None
248
249
250def checkToolsPath(dir, tools):
251    for tool in tools:
252        if not os.path.exists(os.path.join(dir, tool)):
253            return False
254    return True
255
256
257def whichTools(tools, paths):
258    for path in paths.split(os.pathsep):
259        if checkToolsPath(path, tools):
260            return path
261    return None
262
263
264def printHistogram(items, title='Items'):
265    items.sort(key=lambda item: item[1])
266
267    maxValue = max([v for _, v in items])
268
269    # Select first "nice" bar height that produces more than 10 bars.
270    power = int(math.ceil(math.log(maxValue, 10)))
271    for inc in itertools.cycle((5, 2, 2.5, 1)):
272        barH = inc * 10**power
273        N = int(math.ceil(maxValue / barH))
274        if N > 10:
275            break
276        elif inc == 1:
277            power -= 1
278
279    histo = [set() for i in range(N)]
280    for name, v in items:
281        bin = min(int(N * v / maxValue), N - 1)
282        histo[bin].add(name)
283
284    barW = 40
285    hr = '-' * (barW + 34)
286    print('\nSlowest %s:' % title)
287    print(hr)
288    for name, value in items[-20:]:
289        print('%.2fs: %s' % (value, name))
290    print('\n%s Times:' % title)
291    print(hr)
292    pDigits = int(math.ceil(math.log(maxValue, 10)))
293    pfDigits = max(0, 3 - pDigits)
294    if pfDigits:
295        pDigits += pfDigits + 1
296    cDigits = int(math.ceil(math.log(len(items), 10)))
297    print('[%s] :: [%s] :: [%s]' % ('Range'.center((pDigits + 1) * 2 + 3),
298                                    'Percentage'.center(barW),
299                                    'Count'.center(cDigits * 2 + 1)))
300    print(hr)
301    for i, row in enumerate(histo):
302        pct = float(len(row)) / len(items)
303        w = int(barW * pct)
304        print('[%*.*fs,%*.*fs) :: [%s%s] :: [%*d/%*d]' % (
305            pDigits, pfDigits, i * barH, pDigits, pfDigits, (i + 1) * barH,
306            '*' * w, ' ' * (barW - w), cDigits, len(row), cDigits, len(items)))
307
308
309class ExecuteCommandTimeoutException(Exception):
310    def __init__(self, msg, out, err, exitCode):
311        assert isinstance(msg, str)
312        assert isinstance(out, str)
313        assert isinstance(err, str)
314        assert isinstance(exitCode, int)
315        self.msg = msg
316        self.out = out
317        self.err = err
318        self.exitCode = exitCode
319
320
321# Close extra file handles on UNIX (on Windows this cannot be done while
322# also redirecting input).
323kUseCloseFDs = not (platform.system() == 'Windows')
324
325
326def executeCommand(command, cwd=None, env=None, input=None, timeout=0):
327    """Execute command ``command`` (list of arguments or string) with.
328
329    * working directory ``cwd`` (str), use None to use the current
330      working directory
331    * environment ``env`` (dict), use None for none
332    * Input to the command ``input`` (str), use string to pass
333      no input.
334    * Max execution time ``timeout`` (int) seconds. Use 0 for no timeout.
335
336    Returns a tuple (out, err, exitCode) where
337    * ``out`` (str) is the standard output of running the command
338    * ``err`` (str) is the standard error of running the command
339    * ``exitCode`` (int) is the exitCode of running the command
340
341    If the timeout is hit an ``ExecuteCommandTimeoutException``
342    is raised.
343
344    """
345    if input is not None:
346        input = to_bytes(input)
347    p = subprocess.Popen(command, cwd=cwd,
348                         stdin=subprocess.PIPE,
349                         stdout=subprocess.PIPE,
350                         stderr=subprocess.PIPE,
351                         env=env, close_fds=kUseCloseFDs)
352    timerObject = None
353    # FIXME: Because of the way nested function scopes work in Python 2.x we
354    # need to use a reference to a mutable object rather than a plain
355    # bool. In Python 3 we could use the "nonlocal" keyword but we need
356    # to support Python 2 as well.
357    hitTimeOut = [False]
358    try:
359        if timeout > 0:
360            def killProcess():
361                # We may be invoking a shell so we need to kill the
362                # process and all its children.
363                hitTimeOut[0] = True
364                killProcessAndChildren(p.pid)
365
366            timerObject = threading.Timer(timeout, killProcess)
367            timerObject.start()
368
369        out, err = p.communicate(input=input)
370        exitCode = p.wait()
371    finally:
372        if timerObject != None:
373            timerObject.cancel()
374
375    # Ensure the resulting output is always of string type.
376    out = to_string(out)
377    err = to_string(err)
378
379    if hitTimeOut[0]:
380        raise ExecuteCommandTimeoutException(
381            msg='Reached timeout of {} seconds'.format(timeout),
382            out=out,
383            err=err,
384            exitCode=exitCode
385        )
386
387    # Detect Ctrl-C in subprocess.
388    if exitCode == -signal.SIGINT:
389        raise KeyboardInterrupt
390
391    return out, err, exitCode
392
393
394def usePlatformSdkOnDarwin(config, lit_config):
395    # On Darwin, support relocatable SDKs by providing Clang with a
396    # default system root path.
397    if 'darwin' in config.target_triple:
398        try:
399            cmd = subprocess.Popen(['xcrun', '--show-sdk-path', '--sdk', 'macosx'],
400                                   stdout=subprocess.PIPE, stderr=subprocess.PIPE)
401            out, err = cmd.communicate()
402            out = out.strip()
403            res = cmd.wait()
404        except OSError:
405            res = -1
406        if res == 0 and out:
407            sdk_path = out.decode()
408            lit_config.note('using SDKROOT: %r' % sdk_path)
409            config.environment['SDKROOT'] = sdk_path
410
411
412def findPlatformSdkVersionOnMacOS(config, lit_config):
413    if 'darwin' in config.target_triple:
414        try:
415            cmd = subprocess.Popen(['xcrun', '--show-sdk-version', '--sdk', 'macosx'],
416                                   stdout=subprocess.PIPE, stderr=subprocess.PIPE)
417            out, err = cmd.communicate()
418            out = out.strip()
419            res = cmd.wait()
420        except OSError:
421            res = -1
422        if res == 0 and out:
423            return out.decode()
424    return None
425
426
427def killProcessAndChildren(pid):
428    """This function kills a process with ``pid`` and all its running children
429    (recursively). It is currently implemented using the psutil module which
430    provides a simple platform neutral implementation.
431
432    TODO: Reimplement this without using psutil so we can       remove
433    our dependency on it.
434
435    """
436    import psutil
437    try:
438        psutilProc = psutil.Process(pid)
439        # Handle the different psutil API versions
440        try:
441            # psutil >= 2.x
442            children_iterator = psutilProc.children(recursive=True)
443        except AttributeError:
444            # psutil 1.x
445            children_iterator = psutilProc.get_children(recursive=True)
446        for child in children_iterator:
447            try:
448                child.kill()
449            except psutil.NoSuchProcess:
450                pass
451        psutilProc.kill()
452    except psutil.NoSuchProcess:
453        pass
454
455
456try:
457    import win32api
458except ImportError:
459    win32api = None
460
461def abort_now():
462    """Abort the current process without doing any exception teardown"""
463    sys.stdout.flush()
464    if win32api:
465        win32api.TerminateProcess(win32api.GetCurrentProcess(), 3)
466    else:
467        os.kill(0, 9)
468