1#!/usr/bin/env python
2
3# This Source Code Form is subject to the terms of the Mozilla Public
4# License, v. 2.0. If a copy of the MPL was not distributed with this file,
5# You can obtain one at http://mozilla.org/MPL/2.0/.
6
7"""Utility functions for mozrunner"""
8
9import mozinfo
10import os
11import sys
12
13__all__ = ['findInPath', 'get_metadata_from_egg']
14
15
16# python package method metadata by introspection
17try:
18    import pkg_resources
19
20    def get_metadata_from_egg(module):
21        ret = {}
22        try:
23            dist = pkg_resources.get_distribution(module)
24        except pkg_resources.DistributionNotFound:
25            return {}
26        if dist.has_metadata("PKG-INFO"):
27            key = None
28            value = ""
29            for line in dist.get_metadata("PKG-INFO").splitlines():
30                # see http://www.python.org/dev/peps/pep-0314/
31                if key == 'Description':
32                    # descriptions can be long
33                    if not line or line[0].isspace():
34                        value += '\n' + line
35                        continue
36                    else:
37                        key = key.strip()
38                        value = value.strip()
39                        ret[key] = value
40
41                key, value = line.split(':', 1)
42                key = key.strip()
43                value = value.strip()
44                ret[key] = value
45        if dist.has_metadata("requires.txt"):
46            ret["Dependencies"] = "\n" + dist.get_metadata("requires.txt")
47        return ret
48except ImportError:
49    # package resources not avaialable
50    def get_metadata_from_egg(module):
51        return {}
52
53
54def findInPath(fileName, path=os.environ['PATH']):
55    """python equivalent of which; should really be in the stdlib"""
56    dirs = path.split(os.pathsep)
57    for dir in dirs:
58        if os.path.isfile(os.path.join(dir, fileName)):
59            return os.path.join(dir, fileName)
60        if mozinfo.isWin:
61            if os.path.isfile(os.path.join(dir, fileName + ".exe")):
62                return os.path.join(dir, fileName + ".exe")
63
64if __name__ == '__main__':
65    for i in sys.argv[1:]:
66        print findInPath(i)
67
68
69def _find_marionette_in_args(*args, **kwargs):
70    try:
71        m = [a for a in args + tuple(kwargs.values()) if hasattr(a, 'session')][0]
72    except IndexError:
73        print("Can only apply decorator to function using a marionette object")
74        raise
75    return m
76
77
78def _raw_log():
79    import logging
80    return logging.getLogger(__name__)
81
82
83def test_environment(xrePath, env=None, crashreporter=True, debugger=False,
84                     dmdPath=None, lsanPath=None, log=None):
85    """
86    populate OS environment variables for mochitest and reftests.
87
88    Originally comes from automationutils.py. Don't use that for new code.
89    """
90
91    env = os.environ.copy() if env is None else env
92    log = log or _raw_log()
93
94    assert os.path.isabs(xrePath)
95
96    if mozinfo.isMac:
97        ldLibraryPath = os.path.join(os.path.dirname(xrePath), "MacOS")
98    else:
99        ldLibraryPath = xrePath
100
101    envVar = None
102    dmdLibrary = None
103    preloadEnvVar = None
104    if 'toolkit' in mozinfo.info and mozinfo.info['toolkit'] == "gonk":
105        # Skip all of this, it's only valid for the host.
106        pass
107    elif mozinfo.isUnix:
108        envVar = "LD_LIBRARY_PATH"
109        env['MOZILLA_FIVE_HOME'] = xrePath
110        dmdLibrary = "libdmd.so"
111        preloadEnvVar = "LD_PRELOAD"
112    elif mozinfo.isMac:
113        envVar = "DYLD_LIBRARY_PATH"
114        dmdLibrary = "libdmd.dylib"
115        preloadEnvVar = "DYLD_INSERT_LIBRARIES"
116    elif mozinfo.isWin:
117        envVar = "PATH"
118        dmdLibrary = "dmd.dll"
119        preloadEnvVar = "MOZ_REPLACE_MALLOC_LIB"
120    if envVar:
121        envValue = ((env.get(envVar), str(ldLibraryPath))
122                    if mozinfo.isWin
123                    else (ldLibraryPath, dmdPath, env.get(envVar)))
124        env[envVar] = os.path.pathsep.join([path for path in envValue if path])
125
126    if dmdPath and dmdLibrary and preloadEnvVar:
127        env[preloadEnvVar] = os.path.join(dmdPath, dmdLibrary)
128
129    # crashreporter
130    env['GNOME_DISABLE_CRASH_DIALOG'] = '1'
131    env['XRE_NO_WINDOWS_CRASH_DIALOG'] = '1'
132
133    if crashreporter and not debugger:
134        env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
135        env['MOZ_CRASHREPORTER'] = '1'
136    else:
137        env['MOZ_CRASHREPORTER_DISABLE'] = '1'
138
139    # Crash on non-local network connections by default.
140    # MOZ_DISABLE_NONLOCAL_CONNECTIONS can be set to "0" to temporarily
141    # enable non-local connections for the purposes of local testing.  Don't
142    # override the user's choice here.  See bug 1049688.
143    env.setdefault('MOZ_DISABLE_NONLOCAL_CONNECTIONS', '1')
144
145    # Set WebRTC logging in case it is not set yet
146    env.setdefault(
147        'MOZ_LOG',
148        'signaling:3,mtransport:4,DataChannel:4,jsep:4,MediaPipelineFactory:4'
149    )
150    env.setdefault('R_LOG_LEVEL', '6')
151    env.setdefault('R_LOG_DESTINATION', 'stderr')
152    env.setdefault('R_LOG_VERBOSE', '1')
153
154    # ASan specific environment stuff
155    asan = bool(mozinfo.info.get("asan"))
156    if asan and (mozinfo.isLinux or mozinfo.isMac):
157        try:
158            # Symbolizer support
159            llvmsym = os.path.join(xrePath, "llvm-symbolizer")
160            if os.path.isfile(llvmsym):
161                env["ASAN_SYMBOLIZER_PATH"] = llvmsym
162                log.info("INFO | runtests.py | ASan using symbolizer at %s"
163                         % llvmsym)
164            else:
165                log.info("TEST-UNEXPECTED-FAIL | runtests.py | Failed to find"
166                         " ASan symbolizer at %s" % llvmsym)
167
168            # Returns total system memory in kilobytes.
169            # Works only on unix-like platforms where `free` is in the path.
170            totalMemory = int(os.popen("free").readlines()[1].split()[1])
171
172            # Only 4 GB RAM or less available? Use custom ASan options to reduce
173            # the amount of resources required to do the tests. Standard options
174            # will otherwise lead to OOM conditions on the current test slaves.
175            message = "INFO | runtests.py | ASan running in %s configuration"
176            asanOptions = []
177            if totalMemory <= 1024 * 1024 * 4:
178                message = message % 'low-memory'
179                asanOptions = [
180                    'quarantine_size=50331648', 'malloc_context_size=5']
181            else:
182                message = message % 'default memory'
183
184            if lsanPath:
185                log.info("LSan enabled.")
186                asanOptions.append('detect_leaks=1')
187                lsanOptions = ["exitcode=0"]
188                # Uncomment out the next line to report the addresses of leaked objects.
189                # lsanOptions.append("report_objects=1")
190                suppressionsFile = os.path.join(
191                    lsanPath, 'lsan_suppressions.txt')
192                if os.path.exists(suppressionsFile):
193                    log.info("LSan using suppression file " + suppressionsFile)
194                    lsanOptions.append("suppressions=" + suppressionsFile)
195                else:
196                    log.info("WARNING | runtests.py | LSan suppressions file"
197                             " does not exist! " + suppressionsFile)
198                env["LSAN_OPTIONS"] = ':'.join(lsanOptions)
199
200            if len(asanOptions):
201                env['ASAN_OPTIONS'] = ':'.join(asanOptions)
202
203        except OSError as err:
204            log.info("Failed determine available memory, disabling ASan"
205                     " low-memory configuration: %s" % err.strerror)
206        except:
207            log.info("Failed determine available memory, disabling ASan"
208                     " low-memory configuration")
209        else:
210            log.info(message)
211
212    tsan = bool(mozinfo.info.get("tsan"))
213    if tsan and mozinfo.isLinux:
214        # Symbolizer support.
215        llvmsym = os.path.join(xrePath, "llvm-symbolizer")
216        if os.path.isfile(llvmsym):
217            env["TSAN_OPTIONS"] = "external_symbolizer_path=%s" % llvmsym
218            log.info("INFO | runtests.py | TSan using symbolizer at %s"
219                     % llvmsym)
220        else:
221            log.info("TEST-UNEXPECTED-FAIL | runtests.py | Failed to find TSan"
222                     " symbolizer at %s" % llvmsym)
223
224    return env
225
226
227def get_stack_fixer_function(utilityPath, symbolsPath):
228    """
229    Return a stack fixing function, if possible, to use on output lines.
230
231    A stack fixing function checks if a line conforms to the output from
232    MozFormatCodeAddressDetails.  If the line does not, the line is returned
233    unchanged.  If the line does, an attempt is made to convert the
234    file+offset into something human-readable (e.g. a function name).
235    """
236    if not mozinfo.info.get('debug'):
237        return None
238
239    def import_stack_fixer_module(module_name):
240        sys.path.insert(0, utilityPath)
241        module = __import__(module_name, globals(), locals(), [])
242        sys.path.pop(0)
243        return module
244
245    if symbolsPath and os.path.exists(symbolsPath):
246        # Run each line through a function in fix_stack_using_bpsyms.py (uses breakpad
247        # symbol files).
248        # This method is preferred for Tinderbox builds, since native
249        # symbols may have been stripped.
250        stack_fixer_module = import_stack_fixer_module(
251            'fix_stack_using_bpsyms')
252
253        def stack_fixer_function(line):
254            return stack_fixer_module.fixSymbols(line, symbolsPath)
255
256    elif mozinfo.isMac:
257        # Run each line through fix_macosx_stack.py (uses atos).
258        # This method is preferred for developer machines, so we don't
259        # have to run "make buildsymbols".
260        stack_fixer_module = import_stack_fixer_module(
261            'fix_macosx_stack')
262
263        def stack_fixer_function(line):
264            return stack_fixer_module.fixSymbols(line)
265
266    elif mozinfo.isLinux:
267        # Run each line through fix_linux_stack.py (uses addr2line).
268        # This method is preferred for developer machines, so we don't
269        # have to run "make buildsymbols".
270        stack_fixer_module = import_stack_fixer_module(
271            'fix_linux_stack')
272
273        def stack_fixer_function(line):
274            return stack_fixer_module.fixSymbols(line)
275
276    else:
277        return None
278
279    return stack_fixer_function
280