1from __future__ import print_function
2
3import errno
4import itertools
5import math
6import numbers
7import os
8import platform
9import signal
10import subprocess
11import sys
12import threading
13
14
15def is_string(value):
16    try:
17        # Python 2 and Python 3 are different here.
18        return isinstance(value, basestring)
19    except NameError:
20        return isinstance(value, str)
21
22
23def pythonize_bool(value):
24    if value is None:
25        return False
26    if type(value) is bool:
27        return value
28    if isinstance(value, numbers.Number):
29        return value != 0
30    if is_string(value):
31        if value.lower() in ('1', 'true', 'on', 'yes'):
32            return True
33        if value.lower() in ('', '0', 'false', 'off', 'no'):
34            return False
35    raise ValueError('"{}" is not a valid boolean'.format(value))
36
37
38def make_word_regex(word):
39    return r'\b' + word + r'\b'
40
41
42def to_bytes(s):
43    """Return the parameter as type 'bytes', possibly encoding it.
44
45    In Python2, the 'bytes' type is the same as 'str'. In Python3, they
46    are distinct.
47
48    """
49    if isinstance(s, bytes):
50        # In Python2, this branch is taken for both 'str' and 'bytes'.
51        # In Python3, this branch is taken only for 'bytes'.
52        return s
53    # In Python2, 's' is a 'unicode' object.
54    # In Python3, 's' is a 'str' object.
55    # Encode to UTF-8 to get 'bytes' data.
56    return s.encode('utf-8')
57
58
59def to_string(b):
60    """Return the parameter as type 'str', possibly encoding it.
61
62    In Python2, the 'str' type is the same as 'bytes'. In Python3, the
63    'str' type is (essentially) Python2's 'unicode' type, and 'bytes' is
64    distinct.
65
66    """
67    if isinstance(b, str):
68        # In Python2, this branch is taken for types 'str' and 'bytes'.
69        # In Python3, this branch is taken only for 'str'.
70        return b
71    if isinstance(b, bytes):
72        # In Python2, this branch is never taken ('bytes' is handled as 'str').
73        # In Python3, this is true only for 'bytes'.
74        try:
75            return b.decode('utf-8')
76        except UnicodeDecodeError:
77            # If the value is not valid Unicode, return the default
78            # repr-line encoding.
79            return str(b)
80
81    # By this point, here's what we *don't* have:
82    #
83    #  - In Python2:
84    #    - 'str' or 'bytes' (1st branch above)
85    #  - In Python3:
86    #    - 'str' (1st branch above)
87    #    - 'bytes' (2nd branch above)
88    #
89    # The last type we might expect is the Python2 'unicode' type. There is no
90    # 'unicode' type in Python3 (all the Python3 cases were already handled). In
91    # order to get a 'str' object, we need to encode the 'unicode' object.
92    try:
93        return b.encode('utf-8')
94    except AttributeError:
95        raise TypeError('not sure how to convert %s to %s' % (type(b), str))
96
97
98def to_unicode(s):
99    """Return the parameter as type which supports unicode, possibly decoding
100    it.
101
102    In Python2, this is the unicode type. In Python3 it's the str type.
103
104    """
105    if isinstance(s, bytes):
106        # In Python2, this branch is taken for both 'str' and 'bytes'.
107        # In Python3, this branch is taken only for 'bytes'.
108        return s.decode('utf-8')
109    return s
110
111
112def usable_core_count():
113    """Return the number of cores the current process can use, if supported.
114    Otherwise, return the total number of cores (like `os.cpu_count()`).
115    Default to 1 if undetermined.
116
117    """
118    try:
119        n = len(os.sched_getaffinity(0))
120    except AttributeError:
121        n = os.cpu_count() or 1
122
123    # On Windows with more than 60 processes, multiprocessing's call to
124    # _winapi.WaitForMultipleObjects() prints an error and lit hangs.
125    if platform.system() == 'Windows':
126        return min(n, 60)
127
128    return n
129
130
131def mkdir(path):
132    try:
133        if platform.system() == 'Windows':
134            from ctypes import windll
135            from ctypes import GetLastError, WinError
136
137            path = os.path.abspath(path)
138            # Make sure that the path uses backslashes here, in case
139            # python would have happened to use forward slashes, as the
140            # NT path format only supports backslashes.
141            path = path.replace('/', '\\')
142            NTPath = to_unicode(r'\\?\%s' % path)
143            if not windll.kernel32.CreateDirectoryW(NTPath, None):
144                raise WinError(GetLastError())
145        else:
146            os.mkdir(path)
147    except OSError:
148        e = sys.exc_info()[1]
149        # ignore EEXIST, which may occur during a race condition
150        if e.errno != errno.EEXIST:
151            raise
152
153
154def mkdir_p(path):
155    """mkdir_p(path) - Make the "path" directory, if it does not exist; this
156    will also make directories for any missing parent directories."""
157    if not path or os.path.exists(path):
158        return
159
160    parent = os.path.dirname(path)
161    if parent != path:
162        mkdir_p(parent)
163
164    mkdir(path)
165
166
167def listdir_files(dirname, suffixes=None, exclude_filenames=None):
168    """Yields files in a directory.
169
170    Filenames that are not excluded by rules below are yielded one at a time, as
171    basenames (i.e., without dirname).
172
173    Files starting with '.' are always skipped.
174
175    If 'suffixes' is not None, then only filenames ending with one of its
176    members will be yielded. These can be extensions, like '.exe', or strings,
177    like 'Test'. (It is a lexicographic check; so an empty sequence will yield
178    nothing, but a single empty string will yield all filenames.)
179
180    If 'exclude_filenames' is not None, then none of the file basenames in it
181    will be yielded.
182
183    If specified, the containers for 'suffixes' and 'exclude_filenames' must
184    support membership checking for strs.
185
186    Args:
187        dirname: a directory path.
188        suffixes: (optional) a sequence of strings (set, list, etc.).
189        exclude_filenames: (optional) a sequence of strings.
190
191    Yields:
192        Filenames as returned by os.listdir (generally, str).
193
194    """
195    if exclude_filenames is None:
196        exclude_filenames = set()
197    if suffixes is None:
198        suffixes = {''}
199    for filename in os.listdir(dirname):
200        if (os.path.isdir(os.path.join(dirname, filename)) or
201            filename.startswith('.') or
202            filename in exclude_filenames or
203                not any(filename.endswith(sfx) for sfx in suffixes)):
204            continue
205        yield filename
206
207
208def which(command, paths=None):
209    """which(command, [paths]) - Look up the given command in the paths string
210    (or the PATH environment variable, if unspecified)."""
211
212    if paths is None:
213        paths = os.environ.get('PATH', '')
214
215    # Check for absolute match first.
216    if os.path.isabs(command) and os.path.isfile(command):
217        return os.path.normcase(os.path.normpath(command))
218
219    # Would be nice if Python had a lib function for this.
220    if not paths:
221        paths = os.defpath
222
223    # Get suffixes to search.
224    # On Cygwin, 'PATHEXT' may exist but it should not be used.
225    if os.pathsep == ';':
226        pathext = os.environ.get('PATHEXT', '').split(';')
227    else:
228        pathext = ['']
229
230    # Search the paths...
231    for path in paths.split(os.pathsep):
232        for ext in pathext:
233            p = os.path.join(path, command + ext)
234            if os.path.exists(p) and not os.path.isdir(p):
235                return os.path.normcase(os.path.normpath(p))
236
237    return None
238
239
240def checkToolsPath(dir, tools):
241    for tool in tools:
242        if not os.path.exists(os.path.join(dir, tool)):
243            return False
244    return True
245
246
247def whichTools(tools, paths):
248    for path in paths.split(os.pathsep):
249        if checkToolsPath(path, tools):
250            return path
251    return None
252
253
254def printHistogram(items, title='Items'):
255    items.sort(key=lambda item: item[1])
256
257    maxValue = max([v for _, v in items])
258
259    # Select first "nice" bar height that produces more than 10 bars.
260    power = int(math.ceil(math.log(maxValue, 10)))
261    for inc in itertools.cycle((5, 2, 2.5, 1)):
262        barH = inc * 10**power
263        N = int(math.ceil(maxValue / barH))
264        if N > 10:
265            break
266        elif inc == 1:
267            power -= 1
268
269    histo = [set() for i in range(N)]
270    for name, v in items:
271        bin = min(int(N * v / maxValue), N - 1)
272        histo[bin].add(name)
273
274    barW = 40
275    hr = '-' * (barW + 34)
276    print('Slowest %s:' % title)
277    print(hr)
278    for name, value in reversed(items[-20:]):
279        print('%.2fs: %s' % (value, name))
280    print('\n%s Times:' % title)
281    print(hr)
282    pDigits = int(math.ceil(math.log(maxValue, 10)))
283    pfDigits = max(0, 3 - pDigits)
284    if pfDigits:
285        pDigits += pfDigits + 1
286    cDigits = int(math.ceil(math.log(len(items), 10)))
287    print('[%s] :: [%s] :: [%s]' % ('Range'.center((pDigits + 1) * 2 + 3),
288                                    'Percentage'.center(barW),
289                                    'Count'.center(cDigits * 2 + 1)))
290    print(hr)
291    for i, row in reversed(list(enumerate(histo))):
292        pct = float(len(row)) / len(items)
293        w = int(barW * pct)
294        print('[%*.*fs,%*.*fs) :: [%s%s] :: [%*d/%*d]' % (
295            pDigits, pfDigits, i * barH, pDigits, pfDigits, (i + 1) * barH,
296            '*' * w, ' ' * (barW - w), cDigits, len(row), cDigits, len(items)))
297    print(hr)
298
299
300class ExecuteCommandTimeoutException(Exception):
301    def __init__(self, msg, out, err, exitCode):
302        assert isinstance(msg, str)
303        assert isinstance(out, str)
304        assert isinstance(err, str)
305        assert isinstance(exitCode, int)
306        self.msg = msg
307        self.out = out
308        self.err = err
309        self.exitCode = exitCode
310
311
312# Close extra file handles on UNIX (on Windows this cannot be done while
313# also redirecting input).
314kUseCloseFDs = not (platform.system() == 'Windows')
315
316
317def executeCommand(command, cwd=None, env=None, input=None, timeout=0):
318    """Execute command ``command`` (list of arguments or string) with.
319
320    * working directory ``cwd`` (str), use None to use the current
321      working directory
322    * environment ``env`` (dict), use None for none
323    * Input to the command ``input`` (str), use string to pass
324      no input.
325    * Max execution time ``timeout`` (int) seconds. Use 0 for no timeout.
326
327    Returns a tuple (out, err, exitCode) where
328    * ``out`` (str) is the standard output of running the command
329    * ``err`` (str) is the standard error of running the command
330    * ``exitCode`` (int) is the exitCode of running the command
331
332    If the timeout is hit an ``ExecuteCommandTimeoutException``
333    is raised.
334
335    """
336    if input is not None:
337        input = to_bytes(input)
338    p = subprocess.Popen(command, cwd=cwd,
339                         stdin=subprocess.PIPE,
340                         stdout=subprocess.PIPE,
341                         stderr=subprocess.PIPE,
342                         env=env, close_fds=kUseCloseFDs)
343    timerObject = None
344    # FIXME: Because of the way nested function scopes work in Python 2.x we
345    # need to use a reference to a mutable object rather than a plain
346    # bool. In Python 3 we could use the "nonlocal" keyword but we need
347    # to support Python 2 as well.
348    hitTimeOut = [False]
349    try:
350        if timeout > 0:
351            def killProcess():
352                # We may be invoking a shell so we need to kill the
353                # process and all its children.
354                hitTimeOut[0] = True
355                killProcessAndChildren(p.pid)
356
357            timerObject = threading.Timer(timeout, killProcess)
358            timerObject.start()
359
360        out, err = p.communicate(input=input)
361        exitCode = p.wait()
362    finally:
363        if timerObject != None:
364            timerObject.cancel()
365
366    # Ensure the resulting output is always of string type.
367    out = to_string(out)
368    err = to_string(err)
369
370    if hitTimeOut[0]:
371        raise ExecuteCommandTimeoutException(
372            msg='Reached timeout of {} seconds'.format(timeout),
373            out=out,
374            err=err,
375            exitCode=exitCode
376        )
377
378    # Detect Ctrl-C in subprocess.
379    if exitCode == -signal.SIGINT:
380        raise KeyboardInterrupt
381
382    return out, err, exitCode
383
384
385def usePlatformSdkOnDarwin(config, lit_config):
386    # On Darwin, support relocatable SDKs by providing Clang with a
387    # default system root path.
388    if 'darwin' in config.target_triple:
389        try:
390            cmd = subprocess.Popen(['xcrun', '--show-sdk-path', '--sdk', 'macosx'],
391                                   stdout=subprocess.PIPE, stderr=subprocess.PIPE)
392            out, err = cmd.communicate()
393            out = out.strip()
394            res = cmd.wait()
395        except OSError:
396            res = -1
397        if res == 0 and out:
398            sdk_path = out.decode()
399            lit_config.note('using SDKROOT: %r' % sdk_path)
400            config.environment['SDKROOT'] = sdk_path
401
402
403def findPlatformSdkVersionOnMacOS(config, lit_config):
404    if 'darwin' in config.target_triple:
405        try:
406            cmd = subprocess.Popen(['xcrun', '--show-sdk-version', '--sdk', 'macosx'],
407                                   stdout=subprocess.PIPE, stderr=subprocess.PIPE)
408            out, err = cmd.communicate()
409            out = out.strip()
410            res = cmd.wait()
411        except OSError:
412            res = -1
413        if res == 0 and out:
414            return out.decode()
415    return None
416
417def killProcessAndChildrenIsSupported():
418    """
419        Returns a tuple (<supported> , <error message>)
420        where
421        `<supported>` is True if `killProcessAndChildren()` is supported on
422            the current host, returns False otherwise.
423        `<error message>` is an empty string if `<supported>` is True,
424            otherwise is contains a string describing why the function is
425            not supported.
426    """
427    if platform.system() == 'AIX':
428        return (True, "")
429    try:
430        import psutil  # noqa: F401
431        return (True, "")
432    except ImportError:
433        return (False,  "Requires the Python psutil module but it could"
434                        " not be found. Try installing it via pip or via"
435                        " your operating system's package manager.")
436
437def killProcessAndChildren(pid):
438    """This function kills a process with ``pid`` and all its running children
439    (recursively). It is currently implemented using the psutil module on some
440    platforms which provides a simple platform neutral implementation.
441
442    TODO: Reimplement this without using psutil on all platforms so we can
443    remove our dependency on it.
444
445    """
446    if platform.system() == 'AIX':
447        subprocess.call('kill -kill $(ps -o pid= -L{})'.format(pid), shell=True)
448    else:
449        import psutil
450        try:
451            psutilProc = psutil.Process(pid)
452            # Handle the different psutil API versions
453            try:
454                # psutil >= 2.x
455                children_iterator = psutilProc.children(recursive=True)
456            except AttributeError:
457                # psutil 1.x
458                children_iterator = psutilProc.get_children(recursive=True)
459            for child in children_iterator:
460                try:
461                    child.kill()
462                except psutil.NoSuchProcess:
463                    pass
464            psutilProc.kill()
465        except psutil.NoSuchProcess:
466            pass
467