1#!/usr/bin/env python 2 3# This Source Code Form is subject to the terms of the Mozilla Public 4# License, v. 2.0. If a copy of the MPL was not distributed with this file, 5# You can obtain one at http://mozilla.org/MPL/2.0/. 6 7"""Utility functions for mozrunner""" 8 9import mozinfo 10import os 11import sys 12 13__all__ = ['findInPath', 'get_metadata_from_egg'] 14 15 16# python package method metadata by introspection 17try: 18 import pkg_resources 19 20 def get_metadata_from_egg(module): 21 ret = {} 22 try: 23 dist = pkg_resources.get_distribution(module) 24 except pkg_resources.DistributionNotFound: 25 return {} 26 if dist.has_metadata("PKG-INFO"): 27 key = None 28 value = "" 29 for line in dist.get_metadata("PKG-INFO").splitlines(): 30 # see http://www.python.org/dev/peps/pep-0314/ 31 if key == 'Description': 32 # descriptions can be long 33 if not line or line[0].isspace(): 34 value += '\n' + line 35 continue 36 else: 37 key = key.strip() 38 value = value.strip() 39 ret[key] = value 40 41 key, value = line.split(':', 1) 42 key = key.strip() 43 value = value.strip() 44 ret[key] = value 45 if dist.has_metadata("requires.txt"): 46 ret["Dependencies"] = "\n" + dist.get_metadata("requires.txt") 47 return ret 48except ImportError: 49 # package resources not avaialable 50 def get_metadata_from_egg(module): 51 return {} 52 53 54def findInPath(fileName, path=os.environ['PATH']): 55 """python equivalent of which; should really be in the stdlib""" 56 dirs = path.split(os.pathsep) 57 for dir in dirs: 58 if os.path.isfile(os.path.join(dir, fileName)): 59 return os.path.join(dir, fileName) 60 if mozinfo.isWin: 61 if os.path.isfile(os.path.join(dir, fileName + ".exe")): 62 return os.path.join(dir, fileName + ".exe") 63 64if __name__ == '__main__': 65 for i in sys.argv[1:]: 66 print findInPath(i) 67 68 69def _find_marionette_in_args(*args, **kwargs): 70 try: 71 m = [a for a in args + tuple(kwargs.values()) if hasattr(a, 'session')][0] 72 except IndexError: 73 print("Can only apply decorator to function using a marionette object") 74 raise 75 return m 76 77 78def _raw_log(): 79 import logging 80 return logging.getLogger(__name__) 81 82 83def test_environment(xrePath, env=None, crashreporter=True, debugger=False, 84 dmdPath=None, lsanPath=None, log=None): 85 """ 86 populate OS environment variables for mochitest and reftests. 87 88 Originally comes from automationutils.py. Don't use that for new code. 89 """ 90 91 env = os.environ.copy() if env is None else env 92 log = log or _raw_log() 93 94 assert os.path.isabs(xrePath) 95 96 if mozinfo.isMac: 97 ldLibraryPath = os.path.join(os.path.dirname(xrePath), "MacOS") 98 else: 99 ldLibraryPath = xrePath 100 101 envVar = None 102 dmdLibrary = None 103 preloadEnvVar = None 104 if 'toolkit' in mozinfo.info and mozinfo.info['toolkit'] == "gonk": 105 # Skip all of this, it's only valid for the host. 106 pass 107 elif mozinfo.isUnix: 108 envVar = "LD_LIBRARY_PATH" 109 env['MOZILLA_FIVE_HOME'] = xrePath 110 dmdLibrary = "libdmd.so" 111 preloadEnvVar = "LD_PRELOAD" 112 elif mozinfo.isMac: 113 envVar = "DYLD_LIBRARY_PATH" 114 dmdLibrary = "libdmd.dylib" 115 preloadEnvVar = "DYLD_INSERT_LIBRARIES" 116 elif mozinfo.isWin: 117 envVar = "PATH" 118 dmdLibrary = "dmd.dll" 119 preloadEnvVar = "MOZ_REPLACE_MALLOC_LIB" 120 if envVar: 121 envValue = ((env.get(envVar), str(ldLibraryPath)) 122 if mozinfo.isWin 123 else (ldLibraryPath, dmdPath, env.get(envVar))) 124 env[envVar] = os.path.pathsep.join([path for path in envValue if path]) 125 126 if dmdPath and dmdLibrary and preloadEnvVar: 127 env[preloadEnvVar] = os.path.join(dmdPath, dmdLibrary) 128 129 # crashreporter 130 env['GNOME_DISABLE_CRASH_DIALOG'] = '1' 131 env['XRE_NO_WINDOWS_CRASH_DIALOG'] = '1' 132 133 if crashreporter and not debugger: 134 env['MOZ_CRASHREPORTER_NO_REPORT'] = '1' 135 env['MOZ_CRASHREPORTER'] = '1' 136 else: 137 env['MOZ_CRASHREPORTER_DISABLE'] = '1' 138 139 # Crash on non-local network connections by default. 140 # MOZ_DISABLE_NONLOCAL_CONNECTIONS can be set to "0" to temporarily 141 # enable non-local connections for the purposes of local testing. Don't 142 # override the user's choice here. See bug 1049688. 143 env.setdefault('MOZ_DISABLE_NONLOCAL_CONNECTIONS', '1') 144 145 # Set WebRTC logging in case it is not set yet 146 env.setdefault( 147 'MOZ_LOG', 148 'signaling:3,mtransport:4,DataChannel:4,jsep:4,MediaPipelineFactory:4' 149 ) 150 env.setdefault('R_LOG_LEVEL', '6') 151 env.setdefault('R_LOG_DESTINATION', 'stderr') 152 env.setdefault('R_LOG_VERBOSE', '1') 153 154 # ASan specific environment stuff 155 asan = bool(mozinfo.info.get("asan")) 156 if asan and (mozinfo.isLinux or mozinfo.isMac): 157 try: 158 # Symbolizer support 159 llvmsym = os.path.join(xrePath, "llvm-symbolizer") 160 if os.path.isfile(llvmsym): 161 env["ASAN_SYMBOLIZER_PATH"] = llvmsym 162 log.info("INFO | runtests.py | ASan using symbolizer at %s" 163 % llvmsym) 164 else: 165 log.info("TEST-UNEXPECTED-FAIL | runtests.py | Failed to find" 166 " ASan symbolizer at %s" % llvmsym) 167 168 # Returns total system memory in kilobytes. 169 # Works only on unix-like platforms where `free` is in the path. 170 totalMemory = int(os.popen("free").readlines()[1].split()[1]) 171 172 # Only 4 GB RAM or less available? Use custom ASan options to reduce 173 # the amount of resources required to do the tests. Standard options 174 # will otherwise lead to OOM conditions on the current test slaves. 175 message = "INFO | runtests.py | ASan running in %s configuration" 176 asanOptions = [] 177 if totalMemory <= 1024 * 1024 * 4: 178 message = message % 'low-memory' 179 asanOptions = [ 180 'quarantine_size=50331648', 'malloc_context_size=5'] 181 else: 182 message = message % 'default memory' 183 184 if lsanPath: 185 log.info("LSan enabled.") 186 asanOptions.append('detect_leaks=1') 187 lsanOptions = ["exitcode=0"] 188 # Uncomment out the next line to report the addresses of leaked objects. 189 # lsanOptions.append("report_objects=1") 190 suppressionsFile = os.path.join( 191 lsanPath, 'lsan_suppressions.txt') 192 if os.path.exists(suppressionsFile): 193 log.info("LSan using suppression file " + suppressionsFile) 194 lsanOptions.append("suppressions=" + suppressionsFile) 195 else: 196 log.info("WARNING | runtests.py | LSan suppressions file" 197 " does not exist! " + suppressionsFile) 198 env["LSAN_OPTIONS"] = ':'.join(lsanOptions) 199 200 if len(asanOptions): 201 env['ASAN_OPTIONS'] = ':'.join(asanOptions) 202 203 except OSError as err: 204 log.info("Failed determine available memory, disabling ASan" 205 " low-memory configuration: %s" % err.strerror) 206 except: 207 log.info("Failed determine available memory, disabling ASan" 208 " low-memory configuration") 209 else: 210 log.info(message) 211 212 tsan = bool(mozinfo.info.get("tsan")) 213 if tsan and mozinfo.isLinux: 214 # Symbolizer support. 215 llvmsym = os.path.join(xrePath, "llvm-symbolizer") 216 if os.path.isfile(llvmsym): 217 env["TSAN_OPTIONS"] = "external_symbolizer_path=%s" % llvmsym 218 log.info("INFO | runtests.py | TSan using symbolizer at %s" 219 % llvmsym) 220 else: 221 log.info("TEST-UNEXPECTED-FAIL | runtests.py | Failed to find TSan" 222 " symbolizer at %s" % llvmsym) 223 224 return env 225 226 227def get_stack_fixer_function(utilityPath, symbolsPath): 228 """ 229 Return a stack fixing function, if possible, to use on output lines. 230 231 A stack fixing function checks if a line conforms to the output from 232 MozFormatCodeAddressDetails. If the line does not, the line is returned 233 unchanged. If the line does, an attempt is made to convert the 234 file+offset into something human-readable (e.g. a function name). 235 """ 236 if not mozinfo.info.get('debug'): 237 return None 238 239 def import_stack_fixer_module(module_name): 240 sys.path.insert(0, utilityPath) 241 module = __import__(module_name, globals(), locals(), []) 242 sys.path.pop(0) 243 return module 244 245 if symbolsPath and os.path.exists(symbolsPath): 246 # Run each line through a function in fix_stack_using_bpsyms.py (uses breakpad 247 # symbol files). 248 # This method is preferred for Tinderbox builds, since native 249 # symbols may have been stripped. 250 stack_fixer_module = import_stack_fixer_module( 251 'fix_stack_using_bpsyms') 252 253 def stack_fixer_function(line): 254 return stack_fixer_module.fixSymbols(line, symbolsPath) 255 256 elif mozinfo.isMac: 257 # Run each line through fix_macosx_stack.py (uses atos). 258 # This method is preferred for developer machines, so we don't 259 # have to run "make buildsymbols". 260 stack_fixer_module = import_stack_fixer_module( 261 'fix_macosx_stack') 262 263 def stack_fixer_function(line): 264 return stack_fixer_module.fixSymbols(line) 265 266 elif mozinfo.isLinux: 267 # Run each line through fix_linux_stack.py (uses addr2line). 268 # This method is preferred for developer machines, so we don't 269 # have to run "make buildsymbols". 270 stack_fixer_module = import_stack_fixer_module( 271 'fix_linux_stack') 272 273 def stack_fixer_function(line): 274 return stack_fixer_module.fixSymbols(line) 275 276 else: 277 return None 278 279 return stack_fixer_function 280