1import errno
2import itertools
3import math
4import numbers
5import os
6import platform
7import signal
8import subprocess
9import sys
10import threading
11
12
13def norm_path(path):
14    path = os.path.realpath(path)
15    path = os.path.normpath(path)
16    path = os.path.normcase(path)
17    return path
18
19
20def is_string(value):
21    try:
22        # Python 2 and Python 3 are different here.
23        return isinstance(value, basestring)
24    except NameError:
25        return isinstance(value, str)
26
27
28def pythonize_bool(value):
29    if value is None:
30        return False
31    if type(value) is bool:
32        return value
33    if isinstance(value, numbers.Number):
34        return value != 0
35    if is_string(value):
36        if value.lower() in ('1', 'true', 'on', 'yes'):
37            return True
38        if value.lower() in ('', '0', 'false', 'off', 'no'):
39            return False
40    raise ValueError('"{}" is not a valid boolean'.format(value))
41
42
43def make_word_regex(word):
44    return r'\b' + word + r'\b'
45
46
47def to_bytes(s):
48    """Return the parameter as type 'bytes', possibly encoding it.
49
50    In Python2, the 'bytes' type is the same as 'str'. In Python3, they
51    are distinct.
52
53    """
54    if isinstance(s, bytes):
55        # In Python2, this branch is taken for both 'str' and 'bytes'.
56        # In Python3, this branch is taken only for 'bytes'.
57        return s
58    # In Python2, 's' is a 'unicode' object.
59    # In Python3, 's' is a 'str' object.
60    # Encode to UTF-8 to get 'bytes' data.
61    return s.encode('utf-8')
62
63
64def to_string(b):
65    """Return the parameter as type 'str', possibly encoding it.
66
67    In Python2, the 'str' type is the same as 'bytes'. In Python3, the
68    'str' type is (essentially) Python2's 'unicode' type, and 'bytes' is
69    distinct.
70
71    """
72    if isinstance(b, str):
73        # In Python2, this branch is taken for types 'str' and 'bytes'.
74        # In Python3, this branch is taken only for 'str'.
75        return b
76    if isinstance(b, bytes):
77        # In Python2, this branch is never taken ('bytes' is handled as 'str').
78        # In Python3, this is true only for 'bytes'.
79        try:
80            return b.decode('utf-8')
81        except UnicodeDecodeError:
82            # If the value is not valid Unicode, return the default
83            # repr-line encoding.
84            return str(b)
85
86    # By this point, here's what we *don't* have:
87    #
88    #  - In Python2:
89    #    - 'str' or 'bytes' (1st branch above)
90    #  - In Python3:
91    #    - 'str' (1st branch above)
92    #    - 'bytes' (2nd branch above)
93    #
94    # The last type we might expect is the Python2 'unicode' type. There is no
95    # 'unicode' type in Python3 (all the Python3 cases were already handled). In
96    # order to get a 'str' object, we need to encode the 'unicode' object.
97    try:
98        return b.encode('utf-8')
99    except AttributeError:
100        raise TypeError('not sure how to convert %s to %s' % (type(b), str))
101
102
103def detectCPUs():
104    """Detects the number of CPUs on a system.
105
106    Cribbed from pp.
107
108    """
109    # Linux, Unix and MacOS:
110    if hasattr(os, 'sysconf'):
111        if 'SC_NPROCESSORS_ONLN' in os.sysconf_names:
112            # Linux & Unix:
113            ncpus = os.sysconf('SC_NPROCESSORS_ONLN')
114            if isinstance(ncpus, int) and ncpus > 0:
115                return ncpus
116        else:  # OSX:
117            return int(subprocess.check_output(['sysctl', '-n', 'hw.ncpu'],
118                                               stderr=subprocess.STDOUT))
119    # Windows:
120    if 'NUMBER_OF_PROCESSORS' in os.environ:
121        ncpus = int(os.environ['NUMBER_OF_PROCESSORS'])
122        if ncpus > 0:
123            # With more than 32 processes, process creation often fails with
124            # "Too many open files".  FIXME: Check if there's a better fix.
125            return min(ncpus, 32)
126    return 1  # Default
127
128
129def mkdir_p(path):
130    """mkdir_p(path) - Make the "path" directory, if it does not exist; this
131    will also make directories for any missing parent directories."""
132    if not path or os.path.exists(path):
133        return
134
135    parent = os.path.dirname(path)
136    if parent != path:
137        mkdir_p(parent)
138
139    try:
140        os.mkdir(path)
141    except OSError:
142        e = sys.exc_info()[1]
143        # Ignore EEXIST, which may occur during a race condition.
144        if e.errno != errno.EEXIST:
145            raise
146
147
148def listdir_files(dirname, suffixes=None, exclude_filenames=None):
149    """Yields files in a directory.
150
151    Filenames that are not excluded by rules below are yielded one at a time, as
152    basenames (i.e., without dirname).
153
154    Files starting with '.' are always skipped.
155
156    If 'suffixes' is not None, then only filenames ending with one of its
157    members will be yielded. These can be extensions, like '.exe', or strings,
158    like 'Test'. (It is a lexicographic check; so an empty sequence will yield
159    nothing, but a single empty string will yield all filenames.)
160
161    If 'exclude_filenames' is not None, then none of the file basenames in it
162    will be yielded.
163
164    If specified, the containers for 'suffixes' and 'exclude_filenames' must
165    support membership checking for strs.
166
167    Args:
168        dirname: a directory path.
169        suffixes: (optional) a sequence of strings (set, list, etc.).
170        exclude_filenames: (optional) a sequence of strings.
171
172    Yields:
173        Filenames as returned by os.listdir (generally, str).
174
175    """
176    if exclude_filenames is None:
177        exclude_filenames = set()
178    if suffixes is None:
179        suffixes = {''}
180    for filename in os.listdir(dirname):
181        if (os.path.isdir(os.path.join(dirname, filename)) or
182            filename.startswith('.') or
183            filename in exclude_filenames or
184                not any(filename.endswith(sfx) for sfx in suffixes)):
185            continue
186        yield filename
187
188
189def which(command, paths=None):
190    """which(command, [paths]) - Look up the given command in the paths string
191    (or the PATH environment variable, if unspecified)."""
192
193    if paths is None:
194        paths = os.environ.get('PATH', '')
195
196    # Check for absolute match first.
197    if os.path.isabs(command) and os.path.isfile(command):
198        return os.path.normpath(command)
199
200    # Would be nice if Python had a lib function for this.
201    if not paths:
202        paths = os.defpath
203
204    # Get suffixes to search.
205    # On Cygwin, 'PATHEXT' may exist but it should not be used.
206    if os.pathsep == ';':
207        pathext = os.environ.get('PATHEXT', '').split(';')
208    else:
209        pathext = ['']
210
211    # Search the paths...
212    for path in paths.split(os.pathsep):
213        for ext in pathext:
214            p = os.path.join(path, command + ext)
215            if os.path.exists(p) and not os.path.isdir(p):
216                return os.path.normpath(p)
217
218    return None
219
220
221def checkToolsPath(dir, tools):
222    for tool in tools:
223        if not os.path.exists(os.path.join(dir, tool)):
224            return False
225    return True
226
227
228def whichTools(tools, paths):
229    for path in paths.split(os.pathsep):
230        if checkToolsPath(path, tools):
231            return path
232    return None
233
234
235def printHistogram(items, title='Items'):
236    items.sort(key=lambda item: item[1])
237
238    maxValue = max([v for _, v in items])
239
240    # Select first "nice" bar height that produces more than 10 bars.
241    power = int(math.ceil(math.log(maxValue, 10)))
242    for inc in itertools.cycle((5, 2, 2.5, 1)):
243        barH = inc * 10**power
244        N = int(math.ceil(maxValue / barH))
245        if N > 10:
246            break
247        elif inc == 1:
248            power -= 1
249
250    histo = [set() for i in range(N)]
251    for name, v in items:
252        bin = min(int(N * v / maxValue), N - 1)
253        histo[bin].add(name)
254
255    barW = 40
256    hr = '-' * (barW + 34)
257    print('\nSlowest %s:' % title)
258    print(hr)
259    for name, value in items[-20:]:
260        print('%.2fs: %s' % (value, name))
261    print('\n%s Times:' % title)
262    print(hr)
263    pDigits = int(math.ceil(math.log(maxValue, 10)))
264    pfDigits = max(0, 3 - pDigits)
265    if pfDigits:
266        pDigits += pfDigits + 1
267    cDigits = int(math.ceil(math.log(len(items), 10)))
268    print('[%s] :: [%s] :: [%s]' % ('Range'.center((pDigits + 1) * 2 + 3),
269                                    'Percentage'.center(barW),
270                                    'Count'.center(cDigits * 2 + 1)))
271    print(hr)
272    for i, row in enumerate(histo):
273        pct = float(len(row)) / len(items)
274        w = int(barW * pct)
275        print('[%*.*fs,%*.*fs) :: [%s%s] :: [%*d/%*d]' % (
276            pDigits, pfDigits, i * barH, pDigits, pfDigits, (i + 1) * barH,
277            '*' * w, ' ' * (barW - w), cDigits, len(row), cDigits, len(items)))
278
279
280class ExecuteCommandTimeoutException(Exception):
281    def __init__(self, msg, out, err, exitCode):
282        assert isinstance(msg, str)
283        assert isinstance(out, str)
284        assert isinstance(err, str)
285        assert isinstance(exitCode, int)
286        self.msg = msg
287        self.out = out
288        self.err = err
289        self.exitCode = exitCode
290
291
292# Close extra file handles on UNIX (on Windows this cannot be done while
293# also redirecting input).
294kUseCloseFDs = not (platform.system() == 'Windows')
295
296
297def executeCommand(command, cwd=None, env=None, input=None, timeout=0):
298    """Execute command ``command`` (list of arguments or string) with.
299
300    * working directory ``cwd`` (str), use None to use the current
301      working directory
302    * environment ``env`` (dict), use None for none
303    * Input to the command ``input`` (str), use string to pass
304      no input.
305    * Max execution time ``timeout`` (int) seconds. Use 0 for no timeout.
306
307    Returns a tuple (out, err, exitCode) where
308    * ``out`` (str) is the standard output of running the command
309    * ``err`` (str) is the standard error of running the command
310    * ``exitCode`` (int) is the exitCode of running the command
311
312    If the timeout is hit an ``ExecuteCommandTimeoutException``
313    is raised.
314
315    """
316    if input is not None:
317        input = to_bytes(input)
318    p = subprocess.Popen(command, cwd=cwd,
319                         stdin=subprocess.PIPE,
320                         stdout=subprocess.PIPE,
321                         stderr=subprocess.PIPE,
322                         env=env, close_fds=kUseCloseFDs)
323    timerObject = None
324    # FIXME: Because of the way nested function scopes work in Python 2.x we
325    # need to use a reference to a mutable object rather than a plain
326    # bool. In Python 3 we could use the "nonlocal" keyword but we need
327    # to support Python 2 as well.
328    hitTimeOut = [False]
329    try:
330        if timeout > 0:
331            def killProcess():
332                # We may be invoking a shell so we need to kill the
333                # process and all its children.
334                hitTimeOut[0] = True
335                killProcessAndChildren(p.pid)
336
337            timerObject = threading.Timer(timeout, killProcess)
338            timerObject.start()
339
340        out, err = p.communicate(input=input)
341        exitCode = p.wait()
342    finally:
343        if timerObject != None:
344            timerObject.cancel()
345
346    # Ensure the resulting output is always of string type.
347    out = to_string(out)
348    err = to_string(err)
349
350    if hitTimeOut[0]:
351        raise ExecuteCommandTimeoutException(
352            msg='Reached timeout of {} seconds'.format(timeout),
353            out=out,
354            err=err,
355            exitCode=exitCode
356        )
357
358    # Detect Ctrl-C in subprocess.
359    if exitCode == -signal.SIGINT:
360        raise KeyboardInterrupt
361
362    return out, err, exitCode
363
364
365def usePlatformSdkOnDarwin(config, lit_config):
366    # On Darwin, support relocatable SDKs by providing Clang with a
367    # default system root path.
368    if 'darwin' in config.target_triple:
369        try:
370            cmd = subprocess.Popen(['xcrun', '--show-sdk-path', '--sdk', 'macosx'],
371                                   stdout=subprocess.PIPE, stderr=subprocess.PIPE)
372            out, err = cmd.communicate()
373            out = out.strip()
374            res = cmd.wait()
375        except OSError:
376            res = -1
377        if res == 0 and out:
378            sdk_path = out
379            lit_config.note('using SDKROOT: %r' % sdk_path)
380            config.environment['SDKROOT'] = sdk_path
381
382
383def findPlatformSdkVersionOnMacOS(config, lit_config):
384    if 'darwin' in config.target_triple:
385        try:
386            cmd = subprocess.Popen(['xcrun', '--show-sdk-version', '--sdk', 'macosx'],
387                                   stdout=subprocess.PIPE, stderr=subprocess.PIPE)
388            out, err = cmd.communicate()
389            out = out.strip()
390            res = cmd.wait()
391        except OSError:
392            res = -1
393        if res == 0 and out:
394            return out
395    return None
396
397
398def killProcessAndChildren(pid):
399    """This function kills a process with ``pid`` and all its running children
400    (recursively). It is currently implemented using the psutil module which
401    provides a simple platform neutral implementation.
402
403    TODO: Reimplement this without using psutil so we can       remove
404    our dependency on it.
405
406    """
407    import psutil
408    try:
409        psutilProc = psutil.Process(pid)
410        # Handle the different psutil API versions
411        try:
412            # psutil >= 2.x
413            children_iterator = psutilProc.children(recursive=True)
414        except AttributeError:
415            # psutil 1.x
416            children_iterator = psutilProc.get_children(recursive=True)
417        for child in children_iterator:
418            try:
419                child.kill()
420            except psutil.NoSuchProcess:
421                pass
422        psutilProc.kill()
423    except psutil.NoSuchProcess:
424        pass
425