1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this file, 3# You can obtain one at http://mozilla.org/MPL/2.0/. 4 5from __future__ import absolute_import, print_function 6 7import glob 8import os 9import re 10import shutil 11import signal 12import subprocess 13import sys 14import tempfile 15import urllib2 16import zipfile 17from collections import namedtuple 18 19import mozfile 20import mozinfo 21import mozlog 22 23__all__ = [ 24 'check_for_crashes', 25 'check_for_java_exception', 26 'kill_and_get_minidump', 27 'log_crashes', 28 'cleanup_pending_crash_reports', 29] 30 31 32StackInfo = namedtuple("StackInfo", 33 ["minidump_path", 34 "signature", 35 "stackwalk_stdout", 36 "stackwalk_stderr", 37 "stackwalk_retcode", 38 "stackwalk_errors", 39 "extra"]) 40 41 42def get_logger(): 43 structured_logger = mozlog.get_default_logger("mozcrash") 44 if structured_logger is None: 45 return mozlog.unstructured.getLogger('mozcrash') 46 return structured_logger 47 48 49def check_for_crashes(dump_directory, 50 symbols_path=None, 51 stackwalk_binary=None, 52 dump_save_path=None, 53 test_name=None, 54 quiet=False): 55 """ 56 Print a stack trace for minidump files left behind by a crashing program. 57 58 `dump_directory` will be searched for minidump files. Any minidump files found will 59 have `stackwalk_binary` executed on them, with `symbols_path` passed as an extra 60 argument. 61 62 `stackwalk_binary` should be a path to the minidump_stackwalk binary. 63 If `stackwalk_binary` is not set, the MINIDUMP_STACKWALK environment variable 64 will be checked and its value used if it is not empty. 65 66 `symbols_path` should be a path to a directory containing symbols to use for 67 dump processing. This can either be a path to a directory containing Breakpad-format 68 symbols, or a URL to a zip file containing a set of symbols. 69 70 If `dump_save_path` is set, it should be a path to a directory in which to copy minidump 71 files for safekeeping after a stack trace has been printed. If not set, the environment 72 variable MINIDUMP_SAVE_PATH will be checked and its value used if it is not empty. 73 74 If `test_name` is set it will be used as the test name in log output. If not set the 75 filename of the calling function will be used. 76 77 If `quiet` is set, no PROCESS-CRASH message will be printed to stdout if a 78 crash is detected. 79 80 Returns number of minidump files found. 81 """ 82 83 # try to get the caller's filename if no test name is given 84 if test_name is None: 85 try: 86 test_name = os.path.basename(sys._getframe(1).f_code.co_filename) 87 except Exception: 88 test_name = "unknown" 89 90 crash_info = CrashInfo(dump_directory, symbols_path, dump_save_path=dump_save_path, 91 stackwalk_binary=stackwalk_binary) 92 93 if not crash_info.has_dumps: 94 return False 95 96 crash_count = 0 97 for info in crash_info: 98 crash_count += 1 99 if not quiet: 100 stackwalk_output = ["Crash dump filename: %s" % info.minidump_path] 101 if info.stackwalk_stderr: 102 stackwalk_output.append("stderr from minidump_stackwalk:") 103 stackwalk_output.append(info.stackwalk_stderr) 104 elif info.stackwalk_stdout is not None: 105 stackwalk_output.append(info.stackwalk_stdout) 106 if info.stackwalk_retcode is not None and info.stackwalk_retcode != 0: 107 stackwalk_output.append("minidump_stackwalk exited with return code %d" % 108 info.stackwalk_retcode) 109 signature = info.signature if info.signature else "unknown top frame" 110 print("PROCESS-CRASH | %s | application crashed [%s]" % (test_name, 111 signature)) 112 print('\n'.join(stackwalk_output)) 113 print('\n'.join(info.stackwalk_errors)) 114 115 return crash_count 116 117 118def log_crashes(logger, 119 dump_directory, 120 symbols_path, 121 process=None, 122 test=None, 123 stackwalk_binary=None, 124 dump_save_path=None): 125 """Log crashes using a structured logger""" 126 crash_count = 0 127 for info in CrashInfo(dump_directory, symbols_path, dump_save_path=dump_save_path, 128 stackwalk_binary=stackwalk_binary): 129 crash_count += 1 130 kwargs = info._asdict() 131 kwargs.pop("extra") 132 logger.crash(process=process, test=test, **kwargs) 133 return crash_count 134 135 136class CrashInfo(object): 137 """Get information about a crash based on dump files. 138 139 Typical usage is to iterate over the CrashInfo object. This returns StackInfo 140 objects, one for each crash dump file that is found in the dump_directory. 141 142 :param dump_directory: Path to search for minidump files 143 :param symbols_path: Path to a path to a directory containing symbols to use for 144 dump processing. This can either be a path to a directory 145 containing Breakpad-format symbols, or a URL to a zip file 146 containing a set of symbols. 147 :param dump_save_path: Path to which to save the dump files. If this is None, 148 the MINIDUMP_SAVE_PATH environment variable will be used. 149 :param stackwalk_binary: Path to the minidump_stackwalk binary. If this is None, 150 the MINIDUMP_STACKWALK environment variable will be used 151 as the path to the minidump binary.""" 152 153 def __init__(self, dump_directory, symbols_path, dump_save_path=None, 154 stackwalk_binary=None): 155 self.dump_directory = dump_directory 156 self.symbols_path = symbols_path 157 self.remove_symbols = False 158 159 if dump_save_path is None: 160 dump_save_path = os.environ.get('MINIDUMP_SAVE_PATH', None) 161 self.dump_save_path = dump_save_path 162 163 if stackwalk_binary is None: 164 stackwalk_binary = os.environ.get('MINIDUMP_STACKWALK', None) 165 self.stackwalk_binary = stackwalk_binary 166 167 self.logger = get_logger() 168 self._dump_files = None 169 170 def _get_symbols(self): 171 # If no symbols path has been set create a temporary folder to let the 172 # minidump stackwalk download the symbols. 173 if not self.symbols_path: 174 self.symbols_path = tempfile.mkdtemp() 175 self.remove_symbols = True 176 177 # This updates self.symbols_path so we only download once. 178 if mozfile.is_url(self.symbols_path): 179 self.remove_symbols = True 180 self.logger.info("Downloading symbols from: %s" % self.symbols_path) 181 # Get the symbols and write them to a temporary zipfile 182 data = urllib2.urlopen(self.symbols_path) 183 with tempfile.TemporaryFile() as symbols_file: 184 symbols_file.write(data.read()) 185 # extract symbols to a temporary directory (which we'll delete after 186 # processing all crashes) 187 self.symbols_path = tempfile.mkdtemp() 188 with zipfile.ZipFile(symbols_file, 'r') as zfile: 189 mozfile.extract_zip(zfile, self.symbols_path) 190 191 @property 192 def dump_files(self): 193 """List of tuple (path_to_dump_file, path_to_extra_file) for each dump 194 file in self.dump_directory. The extra files may not exist.""" 195 if self._dump_files is None: 196 self._dump_files = [(path, os.path.splitext(path)[0] + '.extra') for path in 197 glob.glob(os.path.join(self.dump_directory, '*.dmp'))] 198 max_dumps = 10 199 if len(self._dump_files) > max_dumps: 200 self.logger.warning("Found %d dump files -- limited to %d!" % 201 (len(self._dump_files), max_dumps)) 202 del self._dump_files[max_dumps:] 203 204 return self._dump_files 205 206 @property 207 def has_dumps(self): 208 """Boolean indicating whether any crash dump files were found in the 209 current directory""" 210 return len(self.dump_files) > 0 211 212 def __iter__(self): 213 for path, extra in self.dump_files: 214 rv = self._process_dump_file(path, extra) 215 yield rv 216 217 if self.remove_symbols: 218 mozfile.remove(self.symbols_path) 219 220 def _process_dump_file(self, path, extra): 221 """Process a single dump file using self.stackwalk_binary, and return a 222 tuple containing properties of the crash dump. 223 224 :param path: Path to the minidump file to analyse 225 :return: A StackInfo tuple with the fields:: 226 minidump_path: Path of the dump file 227 signature: The top frame of the stack trace, or None if it 228 could not be determined. 229 stackwalk_stdout: String of stdout data from stackwalk 230 stackwalk_stderr: String of stderr data from stackwalk or 231 None if it succeeded 232 stackwalk_retcode: Return code from stackwalk 233 stackwalk_errors: List of errors in human-readable form that prevented 234 stackwalk being launched. 235 """ 236 self._get_symbols() 237 238 errors = [] 239 signature = None 240 include_stderr = False 241 out = None 242 err = None 243 retcode = None 244 if (self.symbols_path and self.stackwalk_binary and 245 os.path.exists(self.stackwalk_binary) and 246 os.access(self.stackwalk_binary, os.X_OK)): 247 248 command = [ 249 self.stackwalk_binary, 250 path, 251 self.symbols_path 252 ] 253 self.logger.info('Copy/paste: ' + ' '.join(command)) 254 # run minidump_stackwalk 255 p = subprocess.Popen( 256 command, 257 stdout=subprocess.PIPE, 258 stderr=subprocess.PIPE 259 ) 260 (out, err) = p.communicate() 261 retcode = p.returncode 262 263 if len(out) > 3: 264 # minidump_stackwalk is chatty, 265 # so ignore stderr when it succeeds. 266 # The top frame of the crash is always the line after "Thread N (crashed)" 267 # Examples: 268 # 0 libc.so + 0xa888 269 # 0 libnss3.so!nssCertificate_Destroy [certificate.c : 102 + 0x0] 270 # 0 mozjs.dll!js::GlobalObject::getDebuggers() [GlobalObject.cpp:89df18f9b6da : 580 + 0x0] # noqa 271 # 0 libxul.so!void js::gc::MarkInternal<JSObject>(JSTracer*, JSObject**) 272 # [Marking.cpp : 92 + 0x28] 273 lines = out.splitlines() 274 for i, line in enumerate(lines): 275 if "(crashed)" in line: 276 match = re.search(r"^ 0 (?:.*!)?(?:void )?([^\[]+)", lines[i + 1]) 277 if match: 278 signature = "@ %s" % match.group(1).strip() 279 break 280 else: 281 include_stderr = True 282 283 else: 284 if not self.symbols_path: 285 errors.append("No symbols path given, can't process dump.") 286 if not self.stackwalk_binary: 287 errors.append("MINIDUMP_STACKWALK not set, can't process dump.") 288 elif self.stackwalk_binary and not os.path.exists(self.stackwalk_binary): 289 errors.append("MINIDUMP_STACKWALK binary not found: %s" % self.stackwalk_binary) 290 elif not os.access(self.stackwalk_binary, os.X_OK): 291 errors.append('This user cannot execute the MINIDUMP_STACKWALK binary.') 292 293 if self.dump_save_path: 294 self._save_dump_file(path, extra) 295 296 if os.path.exists(path): 297 mozfile.remove(path) 298 if os.path.exists(extra): 299 mozfile.remove(extra) 300 301 return StackInfo(path, 302 signature, 303 out, 304 err if include_stderr else None, 305 retcode, 306 errors, 307 extra) 308 309 def _save_dump_file(self, path, extra): 310 if os.path.isfile(self.dump_save_path): 311 os.unlink(self.dump_save_path) 312 if not os.path.isdir(self.dump_save_path): 313 try: 314 os.makedirs(self.dump_save_path) 315 except OSError: 316 pass 317 318 shutil.move(path, self.dump_save_path) 319 self.logger.info("Saved minidump as %s" % 320 os.path.join(self.dump_save_path, os.path.basename(path))) 321 322 if os.path.isfile(extra): 323 shutil.move(extra, self.dump_save_path) 324 self.logger.info("Saved app info as %s" % 325 os.path.join(self.dump_save_path, os.path.basename(extra))) 326 327 328def check_for_java_exception(logcat, test_name=None, quiet=False): 329 """ 330 Print a summary of a fatal Java exception, if present in the provided 331 logcat output. 332 333 Example: 334 PROCESS-CRASH | <test-name> | java-exception java.lang.NullPointerException at org.mozilla.gecko.GeckoApp$21.run(GeckoApp.java:1833) # noqa 335 336 `logcat` should be a list of strings. 337 338 If `test_name` is set it will be used as the test name in log output. If not set the 339 filename of the calling function will be used. 340 341 If `quiet` is set, no PROCESS-CRASH message will be printed to stdout if a 342 crash is detected. 343 344 Returns True if a fatal Java exception was found, False otherwise. 345 """ 346 347 # try to get the caller's filename if no test name is given 348 if test_name is None: 349 try: 350 test_name = os.path.basename(sys._getframe(1).f_code.co_filename) 351 except Exception: 352 test_name = "unknown" 353 354 found_exception = False 355 356 for i, line in enumerate(logcat): 357 # Logs will be of form: 358 # 359 # 01-30 20:15:41.937 E/GeckoAppShell( 1703): >>> REPORTING UNCAUGHT EXCEPTION FROM THREAD 9 ("GeckoBackgroundThread") # noqa 360 # 01-30 20:15:41.937 E/GeckoAppShell( 1703): java.lang.NullPointerException 361 # 01-30 20:15:41.937 E/GeckoAppShell( 1703): at org.mozilla.gecko.GeckoApp$21.run(GeckoApp.java:1833) # noqa 362 # 01-30 20:15:41.937 E/GeckoAppShell( 1703): at android.os.Handler.handleCallback(Handler.java:587) # noqa 363 if "REPORTING UNCAUGHT EXCEPTION" in line: 364 # Strip away the date, time, logcat tag and pid from the next two lines and 365 # concatenate the remainder to form a concise summary of the exception. 366 found_exception = True 367 if len(logcat) >= i + 3: 368 logre = re.compile(r".*\): \t?(.*)") 369 m = logre.search(logcat[i + 1]) 370 if m and m.group(1): 371 exception_type = m.group(1) 372 m = logre.search(logcat[i + 2]) 373 if m and m.group(1): 374 exception_location = m.group(1) 375 if not quiet: 376 print("PROCESS-CRASH | %s | java-exception %s %s" % (test_name, 377 exception_type, 378 exception_location)) 379 else: 380 print("Automation Error: java exception in logcat at line " 381 "%d of %d: %s" % (i, len(logcat), line)) 382 break 383 384 return found_exception 385 386 387if mozinfo.isWin: 388 import ctypes 389 import uuid 390 391 kernel32 = ctypes.windll.kernel32 392 OpenProcess = kernel32.OpenProcess 393 CloseHandle = kernel32.CloseHandle 394 395 def write_minidump(pid, dump_directory, utility_path): 396 """ 397 Write a minidump for a process. 398 399 :param pid: PID of the process to write a minidump for. 400 :param dump_directory: Directory in which to write the minidump. 401 """ 402 PROCESS_QUERY_INFORMATION = 0x0400 403 PROCESS_VM_READ = 0x0010 404 GENERIC_READ = 0x80000000 405 GENERIC_WRITE = 0x40000000 406 CREATE_ALWAYS = 2 407 FILE_ATTRIBUTE_NORMAL = 0x80 408 INVALID_HANDLE_VALUE = -1 409 410 file_name = os.path.join(dump_directory, 411 str(uuid.uuid4()) + ".dmp") 412 413 if (mozinfo.info['bits'] != ctypes.sizeof(ctypes.c_voidp) * 8 and 414 utility_path): 415 # We're not going to be able to write a minidump with ctypes if our 416 # python process was compiled for a different architecture than 417 # firefox, so we invoke the minidumpwriter utility program. 418 419 log = get_logger() 420 minidumpwriter = os.path.normpath(os.path.join(utility_path, 421 "minidumpwriter.exe")) 422 log.info("Using %s to write a dump to %s for [%d]" % 423 (minidumpwriter, file_name, pid)) 424 if not os.path.exists(minidumpwriter): 425 log.error("minidumpwriter not found in %s" % utility_path) 426 return 427 428 if isinstance(file_name, unicode): 429 # Convert to a byte string before sending to the shell. 430 file_name = file_name.encode(sys.getfilesystemencoding()) 431 432 status = subprocess.Popen([minidumpwriter, str(pid), file_name]).wait() 433 if status: 434 log.error("minidumpwriter exited with status: %d" % status) 435 return 436 437 proc_handle = OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, 438 0, pid) 439 if not proc_handle: 440 return 441 442 if not isinstance(file_name, unicode): 443 # Convert to unicode explicitly so our path will be valid as input 444 # to CreateFileW 445 file_name = unicode(file_name, sys.getfilesystemencoding()) 446 447 file_handle = kernel32.CreateFileW(file_name, 448 GENERIC_READ | GENERIC_WRITE, 449 0, 450 None, 451 CREATE_ALWAYS, 452 FILE_ATTRIBUTE_NORMAL, 453 None) 454 if file_handle != INVALID_HANDLE_VALUE: 455 ctypes.windll.dbghelp.MiniDumpWriteDump(proc_handle, 456 pid, 457 file_handle, 458 # Dump type - MiniDumpNormal 459 0, 460 # Exception parameter 461 None, 462 # User stream parameter 463 None, 464 # Callback parameter 465 None) 466 CloseHandle(file_handle) 467 CloseHandle(proc_handle) 468 469 def kill_pid(pid): 470 """ 471 Terminate a process with extreme prejudice. 472 473 :param pid: PID of the process to terminate. 474 """ 475 PROCESS_TERMINATE = 0x0001 476 WAIT_OBJECT_0 = 0x0 477 WAIT_FAILED = -1 478 logger = get_logger() 479 handle = OpenProcess(PROCESS_TERMINATE, 0, pid) 480 if handle: 481 if kernel32.TerminateProcess(handle, 1): 482 # TerminateProcess is async; wait up to 30 seconds for process to 483 # actually terminate, then give up so that clients are not kept 484 # waiting indefinitely for hung processes. 485 status = kernel32.WaitForSingleObject(handle, 30000) 486 if status == WAIT_FAILED: 487 err = kernel32.GetLastError() 488 logger.warning("kill_pid(): wait failed (%d) terminating pid %d: error %d" % 489 (status, pid, err)) 490 elif status != WAIT_OBJECT_0: 491 logger.warning("kill_pid(): wait failed (%d) terminating pid %d" % 492 (status, pid)) 493 else: 494 err = kernel32.GetLastError() 495 logger.warning("kill_pid(): unable to terminate pid %d: %d" % 496 (pid, err)) 497 CloseHandle(handle) 498 else: 499 err = kernel32.GetLastError() 500 logger.warning("kill_pid(): unable to get handle for pid %d: %d" % 501 (pid, err)) 502else: 503 def kill_pid(pid): 504 """ 505 Terminate a process with extreme prejudice. 506 507 :param pid: PID of the process to terminate. 508 """ 509 os.kill(pid, signal.SIGKILL) 510 511 512def kill_and_get_minidump(pid, dump_directory, utility_path=None): 513 """ 514 Attempt to kill a process and leave behind a minidump describing its 515 execution state. 516 517 :param pid: The PID of the process to kill. 518 :param dump_directory: The directory where a minidump should be written on 519 Windows, where the dump will be written from outside the process. 520 521 On Windows a dump will be written using the MiniDumpWriteDump function 522 from DbgHelp.dll. On Linux and OS X the process will be sent a SIGABRT 523 signal to trigger minidump writing via a Breakpad signal handler. On other 524 platforms the process will simply be killed via SIGKILL. 525 526 If the process is hung in such a way that it cannot respond to SIGABRT 527 it may still be running after this function returns. In that case it 528 is the caller's responsibility to deal with killing it. 529 """ 530 needs_killing = True 531 if mozinfo.isWin: 532 write_minidump(pid, dump_directory, utility_path) 533 elif mozinfo.isLinux or mozinfo.isMac: 534 os.kill(pid, signal.SIGABRT) 535 needs_killing = False 536 if needs_killing: 537 kill_pid(pid) 538 539 540def cleanup_pending_crash_reports(): 541 """ 542 Delete any pending crash reports. 543 544 The presence of pending crash reports may be reported by the browser, 545 affecting test results; it is best to ensure that these are removed 546 before starting any browser tests. 547 548 Firefox stores pending crash reports in "<UAppData>/Crash Reports". 549 If the browser is not running, it cannot provide <UAppData>, so this 550 code tries to anticipate its value. 551 552 See dom/system/OSFileConstants.cpp for platform variations of <UAppData>. 553 """ 554 if mozinfo.isWin: 555 location = os.path.expanduser("~\\AppData\\Roaming\\Mozilla\\Firefox\\Crash Reports") 556 elif mozinfo.isMac: 557 location = os.path.expanduser("~/Library/Application Support/firefox/Crash Reports") 558 else: 559 location = os.path.expanduser("~/.mozilla/firefox/Crash Reports") 560 logger = get_logger() 561 if os.path.exists(location): 562 try: 563 mozfile.remove(location) 564 logger.info("Removed pending crash reports at '%s'" % location) 565 except Exception: 566 pass 567 568 569if __name__ == '__main__': 570 import argparse 571 parser = argparse.ArgumentParser() 572 parser.add_argument('--stackwalk-binary', '-b') 573 parser.add_argument('--dump-save-path', '-o') 574 parser.add_argument('--test-name', '-n') 575 parser.add_argument('dump_directory') 576 parser.add_argument('symbols_path') 577 args = parser.parse_args() 578 579 check_for_crashes(args.dump_directory, args.symbols_path, 580 stackwalk_binary=args.stackwalk_binary, 581 dump_save_path=args.dump_save_path, 582 test_name=args.test_name) 583