1#!/usr/bin/env python3 2 3#---------------------------------------------------------------------- 4# Be sure to add the python path that points to the LLDB shared library. 5# 6# To use this in the embedded python interpreter using "lldb": 7# 8# cd /path/containing/crashlog.py 9# lldb 10# (lldb) script import crashlog 11# "crashlog" command installed, type "crashlog --help" for detailed help 12# (lldb) crashlog ~/Library/Logs/DiagnosticReports/a.crash 13# 14# The benefit of running the crashlog command inside lldb in the 15# embedded python interpreter is when the command completes, there 16# will be a target with all of the files loaded at the locations 17# described in the crash log. Only the files that have stack frames 18# in the backtrace will be loaded unless the "--load-all" option 19# has been specified. This allows users to explore the program in the 20# state it was in right at crash time. 21# 22# On MacOSX csh, tcsh: 23# ( setenv PYTHONPATH /path/to/LLDB.framework/Resources/Python ; ./crashlog.py ~/Library/Logs/DiagnosticReports/a.crash ) 24# 25# On MacOSX sh, bash: 26# PYTHONPATH=/path/to/LLDB.framework/Resources/Python ./crashlog.py ~/Library/Logs/DiagnosticReports/a.crash 27#---------------------------------------------------------------------- 28 29import abc 30import concurrent.futures 31import contextlib 32import datetime 33import json 34import optparse 35import os 36import platform 37import plistlib 38import re 39import shlex 40import string 41import subprocess 42import sys 43import threading 44import time 45import uuid 46 47 48print_lock = threading.RLock() 49 50try: 51 # First try for LLDB in case PYTHONPATH is already correctly setup. 52 import lldb 53except ImportError: 54 # Ask the command line driver for the path to the lldb module. Copy over 55 # the environment so that SDKROOT is propagated to xcrun. 56 command = ['xcrun', 'lldb', '-P'] if platform.system() == 'Darwin' else ['lldb', '-P'] 57 # Extend the PYTHONPATH if the path exists and isn't already there. 58 lldb_python_path = subprocess.check_output(command).decode("utf-8").strip() 59 if os.path.exists(lldb_python_path) and not sys.path.__contains__(lldb_python_path): 60 sys.path.append(lldb_python_path) 61 # Try importing LLDB again. 62 try: 63 import lldb 64 except ImportError: 65 print("error: couldn't locate the 'lldb' module, please set PYTHONPATH correctly") 66 sys.exit(1) 67 68from lldb.utils import symbolication 69 70def read_plist(s): 71 if sys.version_info.major == 3: 72 return plistlib.loads(s) 73 else: 74 return plistlib.readPlistFromString(s) 75 76class CrashLog(symbolication.Symbolicator): 77 class Thread: 78 """Class that represents a thread in a darwin crash log""" 79 80 def __init__(self, index, app_specific_backtrace): 81 self.index = index 82 self.id = index 83 self.frames = list() 84 self.idents = list() 85 self.registers = dict() 86 self.reason = None 87 self.name = None 88 self.queue = None 89 self.crashed = False 90 self.app_specific_backtrace = app_specific_backtrace 91 92 def dump(self, prefix): 93 if self.app_specific_backtrace: 94 print("%Application Specific Backtrace[%u] %s" % (prefix, self.index, self.reason)) 95 else: 96 print("%sThread[%u] %s" % (prefix, self.index, self.reason)) 97 if self.frames: 98 print("%s Frames:" % (prefix)) 99 for frame in self.frames: 100 frame.dump(prefix + ' ') 101 if self.registers: 102 print("%s Registers:" % (prefix)) 103 for reg in self.registers.keys(): 104 print("%s %-8s = %#16.16x" % (prefix, reg, self.registers[reg])) 105 106 def dump_symbolicated(self, crash_log, options): 107 this_thread_crashed = self.app_specific_backtrace 108 if not this_thread_crashed: 109 this_thread_crashed = self.did_crash() 110 if options.crashed_only and this_thread_crashed == False: 111 return 112 113 print("%s" % self) 114 display_frame_idx = -1 115 for frame_idx, frame in enumerate(self.frames): 116 disassemble = ( 117 this_thread_crashed or options.disassemble_all_threads) and frame_idx < options.disassemble_depth 118 119 # Except for the zeroth frame, we should subtract 1 from every 120 # frame pc to get the previous line entry. 121 pc = frame.pc & crash_log.addr_mask 122 pc = pc if frame_idx == 0 or pc == 0 else pc - 1 123 symbolicated_frame_addresses = crash_log.symbolicate(pc, options.verbose) 124 125 if symbolicated_frame_addresses: 126 symbolicated_frame_address_idx = 0 127 for symbolicated_frame_address in symbolicated_frame_addresses: 128 display_frame_idx += 1 129 print('[%3u] %s' % (frame_idx, symbolicated_frame_address)) 130 if (options.source_all or self.did_crash( 131 )) and display_frame_idx < options.source_frames and options.source_context: 132 source_context = options.source_context 133 line_entry = symbolicated_frame_address.get_symbol_context().line_entry 134 if line_entry.IsValid(): 135 strm = lldb.SBStream() 136 if line_entry: 137 crash_log.debugger.GetSourceManager().DisplaySourceLinesWithLineNumbers( 138 line_entry.file, line_entry.line, source_context, source_context, "->", strm) 139 source_text = strm.GetData() 140 if source_text: 141 # Indent the source a bit 142 indent_str = ' ' 143 join_str = '\n' + indent_str 144 print('%s%s' % (indent_str, join_str.join(source_text.split('\n')))) 145 if symbolicated_frame_address_idx == 0: 146 if disassemble: 147 instructions = symbolicated_frame_address.get_instructions() 148 if instructions: 149 print() 150 symbolication.disassemble_instructions( 151 crash_log.get_target(), 152 instructions, 153 frame.pc, 154 options.disassemble_before, 155 options.disassemble_after, 156 frame.index > 0) 157 print() 158 symbolicated_frame_address_idx += 1 159 else: 160 print(frame) 161 if self.registers: 162 print() 163 for reg in self.registers.keys(): 164 print(" %-8s = %#16.16x" % (reg, self.registers[reg])) 165 elif self.crashed: 166 print() 167 print("No thread state (register information) available") 168 169 def add_ident(self, ident): 170 if ident not in self.idents: 171 self.idents.append(ident) 172 173 def did_crash(self): 174 return self.reason is not None 175 176 def __str__(self): 177 if self.app_specific_backtrace: 178 s = "Application Specific Backtrace[%u]" % self.index 179 else: 180 s = "Thread[%u]" % self.index 181 if self.reason: 182 s += ' %s' % self.reason 183 return s 184 185 class Frame: 186 """Class that represents a stack frame in a thread in a darwin crash log""" 187 188 def __init__(self, index, pc, description): 189 self.pc = pc 190 self.description = description 191 self.index = index 192 193 def __str__(self): 194 if self.description: 195 return "[%3u] 0x%16.16x %s" % ( 196 self.index, self.pc, self.description) 197 else: 198 return "[%3u] 0x%16.16x" % (self.index, self.pc) 199 200 def dump(self, prefix): 201 print("%s%s" % (prefix, str(self))) 202 203 class DarwinImage(symbolication.Image): 204 """Class that represents a binary images in a darwin crash log""" 205 dsymForUUIDBinary = '/usr/local/bin/dsymForUUID' 206 if not os.path.exists(dsymForUUIDBinary): 207 try: 208 dsymForUUIDBinary = subprocess.check_output('which dsymForUUID', 209 shell=True).decode("utf-8").rstrip('\n') 210 except: 211 dsymForUUIDBinary = "" 212 213 dwarfdump_uuid_regex = re.compile( 214 'UUID: ([-0-9a-fA-F]+) \(([^\(]+)\) .*') 215 216 def __init__( 217 self, 218 text_addr_lo, 219 text_addr_hi, 220 identifier, 221 version, 222 uuid, 223 path, 224 verbose): 225 symbolication.Image.__init__(self, path, uuid) 226 self.add_section( 227 symbolication.Section( 228 text_addr_lo, 229 text_addr_hi, 230 "__TEXT")) 231 self.identifier = identifier 232 self.version = version 233 self.verbose = verbose 234 235 def show_symbol_progress(self): 236 """ 237 Hide progress output and errors from system frameworks as they are plentiful. 238 """ 239 if self.verbose: 240 return True 241 return not (self.path.startswith("/System/Library/") or 242 self.path.startswith("/usr/lib/")) 243 244 245 def find_matching_slice(self): 246 dwarfdump_cmd_output = subprocess.check_output( 247 'dwarfdump --uuid "%s"' % self.path, shell=True).decode("utf-8") 248 self_uuid = self.get_uuid() 249 for line in dwarfdump_cmd_output.splitlines(): 250 match = self.dwarfdump_uuid_regex.search(line) 251 if match: 252 dwarf_uuid_str = match.group(1) 253 dwarf_uuid = uuid.UUID(dwarf_uuid_str) 254 if self_uuid == dwarf_uuid: 255 self.resolved_path = self.path 256 self.arch = match.group(2) 257 return True 258 if not self.resolved_path: 259 self.unavailable = True 260 if self.show_symbol_progress(): 261 print(("error\n error: unable to locate '%s' with UUID %s" 262 % (self.path, self.get_normalized_uuid_string()))) 263 return False 264 265 def locate_module_and_debug_symbols(self): 266 # Don't load a module twice... 267 if self.resolved: 268 return True 269 # Mark this as resolved so we don't keep trying 270 self.resolved = True 271 uuid_str = self.get_normalized_uuid_string() 272 if self.show_symbol_progress(): 273 with print_lock: 274 print('Getting symbols for %s %s...' % (uuid_str, self.path)) 275 if os.path.exists(self.dsymForUUIDBinary): 276 dsym_for_uuid_command = '%s %s' % ( 277 self.dsymForUUIDBinary, uuid_str) 278 s = subprocess.check_output(dsym_for_uuid_command, shell=True) 279 if s: 280 try: 281 plist_root = read_plist(s) 282 except: 283 with print_lock: 284 print(("Got exception: ", sys.exc_info()[1], " handling dsymForUUID output: \n", s)) 285 raise 286 if plist_root: 287 plist = plist_root[uuid_str] 288 if plist: 289 if 'DBGArchitecture' in plist: 290 self.arch = plist['DBGArchitecture'] 291 if 'DBGDSYMPath' in plist: 292 self.symfile = os.path.realpath( 293 plist['DBGDSYMPath']) 294 if 'DBGSymbolRichExecutable' in plist: 295 self.path = os.path.expanduser( 296 plist['DBGSymbolRichExecutable']) 297 self.resolved_path = self.path 298 if not self.resolved_path and os.path.exists(self.path): 299 if not self.find_matching_slice(): 300 return False 301 if not self.resolved_path and not os.path.exists(self.path): 302 try: 303 mdfind_results = subprocess.check_output( 304 ["/usr/bin/mdfind", 305 "com_apple_xcode_dsym_uuids == %s" % uuid_str]).decode("utf-8").splitlines() 306 found_matching_slice = False 307 for dsym in mdfind_results: 308 dwarf_dir = os.path.join(dsym, 'Contents/Resources/DWARF') 309 if not os.path.exists(dwarf_dir): 310 # Not a dSYM bundle, probably an Xcode archive. 311 continue 312 with print_lock: 313 print('falling back to binary inside "%s"' % dsym) 314 self.symfile = dsym 315 for filename in os.listdir(dwarf_dir): 316 self.path = os.path.join(dwarf_dir, filename) 317 if self.find_matching_slice(): 318 found_matching_slice = True 319 break 320 if found_matching_slice: 321 break 322 except: 323 pass 324 if (self.resolved_path and os.path.exists(self.resolved_path)) or ( 325 self.path and os.path.exists(self.path)): 326 with print_lock: 327 print('Resolved symbols for %s %s...' % (uuid_str, self.path)) 328 return True 329 else: 330 self.unavailable = True 331 return False 332 333 def __init__(self, debugger, path, verbose): 334 """CrashLog constructor that take a path to a darwin crash log file""" 335 symbolication.Symbolicator.__init__(self, debugger) 336 self.path = os.path.expanduser(path) 337 self.info_lines = list() 338 self.system_profile = list() 339 self.threads = list() 340 self.backtraces = list() # For application specific backtraces 341 self.idents = list() # A list of the required identifiers for doing all stack backtraces 342 self.errors = list() 343 self.exception = dict() 344 self.crashed_thread_idx = -1 345 self.version = -1 346 self.target = None 347 self.verbose = verbose 348 349 def dump(self): 350 print("Crash Log File: %s" % (self.path)) 351 if self.backtraces: 352 print("\nApplication Specific Backtraces:") 353 for thread in self.backtraces: 354 thread.dump(' ') 355 print("\nThreads:") 356 for thread in self.threads: 357 thread.dump(' ') 358 print("\nImages:") 359 for image in self.images: 360 image.dump(' ') 361 362 def set_main_image(self, identifier): 363 for i, image in enumerate(self.images): 364 if image.identifier == identifier: 365 self.images.insert(0, self.images.pop(i)) 366 break 367 368 def find_image_with_identifier(self, identifier): 369 for image in self.images: 370 if image.identifier == identifier: 371 return image 372 regex_text = '^.*\.%s$' % (re.escape(identifier)) 373 regex = re.compile(regex_text) 374 for image in self.images: 375 if regex.match(image.identifier): 376 return image 377 return None 378 379 def create_target(self): 380 if self.target is None: 381 self.target = symbolication.Symbolicator.create_target(self) 382 if self.target: 383 return self.target 384 # We weren't able to open the main executable as, but we can still 385 # symbolicate 386 print('crashlog.create_target()...2') 387 if self.idents: 388 for ident in self.idents: 389 image = self.find_image_with_identifier(ident) 390 if image: 391 self.target = image.create_target(self.debugger) 392 if self.target: 393 return self.target # success 394 print('crashlog.create_target()...3') 395 for image in self.images: 396 self.target = image.create_target(self.debugger) 397 if self.target: 398 return self.target # success 399 print('crashlog.create_target()...4') 400 print('error: Unable to locate any executables from the crash log.') 401 print(' Try loading the executable into lldb before running crashlog') 402 print(' and/or make sure the .dSYM bundles can be found by Spotlight.') 403 return self.target 404 405 def get_target(self): 406 return self.target 407 408 409class CrashLogFormatException(Exception): 410 pass 411 412 413class CrashLogParseException(Exception): 414 pass 415 416class InteractiveCrashLogException(Exception): 417 pass 418 419class CrashLogParser: 420 @staticmethod 421 def create(debugger, path, verbose): 422 data = JSONCrashLogParser.is_valid_json(path) 423 if data: 424 parser = JSONCrashLogParser(debugger, path, verbose) 425 parser.data = data 426 return parser 427 else: 428 return TextCrashLogParser(debugger, path, verbose) 429 430 def __init__(self, debugger, path, verbose): 431 self.path = os.path.expanduser(path) 432 self.verbose = verbose 433 self.crashlog = CrashLog(debugger, self.path, self.verbose) 434 435 @abc.abstractmethod 436 def parse(self): 437 pass 438 439 440class JSONCrashLogParser(CrashLogParser): 441 @staticmethod 442 def is_valid_json(path): 443 def parse_json(buffer): 444 try: 445 return json.loads(buffer) 446 except: 447 # The first line can contain meta data. Try stripping it and 448 # try again. 449 head, _, tail = buffer.partition('\n') 450 return json.loads(tail) 451 452 with open(path, 'r', encoding='utf-8') as f: 453 buffer = f.read() 454 try: 455 return parse_json(buffer) 456 except: 457 return None 458 459 def parse(self): 460 try: 461 self.parse_process_info(self.data) 462 self.parse_images(self.data['usedImages']) 463 self.parse_main_image(self.data) 464 self.parse_threads(self.data['threads']) 465 if 'asi' in self.data: 466 self.crashlog.asi = self.data['asi'] 467 if 'asiBacktraces' in self.data: 468 self.parse_app_specific_backtraces(self.data['asiBacktraces']) 469 if 'lastExceptionBacktrace' in self.data: 470 self.crashlog.asb = self.data['lastExceptionBacktrace'] 471 self.parse_errors(self.data) 472 thread = self.crashlog.threads[self.crashlog.crashed_thread_idx] 473 reason = self.parse_crash_reason(self.data['exception']) 474 if thread.reason: 475 thread.reason = '{} {}'.format(thread.reason, reason) 476 else: 477 thread.reason = reason 478 except (KeyError, ValueError, TypeError) as e: 479 raise CrashLogParseException( 480 'Failed to parse JSON crashlog: {}: {}'.format( 481 type(e).__name__, e)) 482 483 return self.crashlog 484 485 def get_used_image(self, idx): 486 return self.data['usedImages'][idx] 487 488 def parse_process_info(self, json_data): 489 self.crashlog.process_id = json_data['pid'] 490 self.crashlog.process_identifier = json_data['procName'] 491 492 def parse_crash_reason(self, json_exception): 493 self.crashlog.exception = json_exception 494 exception_type = json_exception['type'] 495 exception_signal = " " 496 if 'signal' in json_exception: 497 exception_signal += "({})".format(json_exception['signal']) 498 499 if 'codes' in json_exception: 500 exception_extra = " ({})".format(json_exception['codes']) 501 elif 'subtype' in json_exception: 502 exception_extra = " ({})".format(json_exception['subtype']) 503 else: 504 exception_extra = "" 505 return "{}{}{}".format(exception_type, exception_signal, 506 exception_extra) 507 508 def parse_images(self, json_images): 509 idx = 0 510 for json_image in json_images: 511 img_uuid = uuid.UUID(json_image['uuid']) 512 low = int(json_image['base']) 513 high = int(0) 514 name = json_image['name'] if 'name' in json_image else '' 515 path = json_image['path'] if 'path' in json_image else '' 516 version = '' 517 darwin_image = self.crashlog.DarwinImage(low, high, name, version, 518 img_uuid, path, 519 self.verbose) 520 self.crashlog.images.append(darwin_image) 521 idx += 1 522 523 def parse_main_image(self, json_data): 524 if 'procName' in json_data: 525 proc_name = json_data['procName'] 526 self.crashlog.set_main_image(proc_name) 527 528 def parse_frames(self, thread, json_frames): 529 idx = 0 530 for json_frame in json_frames: 531 image_id = int(json_frame['imageIndex']) 532 json_image = self.get_used_image(image_id) 533 ident = json_image['name'] if 'name' in json_image else '' 534 thread.add_ident(ident) 535 if ident not in self.crashlog.idents: 536 self.crashlog.idents.append(ident) 537 538 frame_offset = int(json_frame['imageOffset']) 539 image_addr = self.get_used_image(image_id)['base'] 540 pc = image_addr + frame_offset 541 thread.frames.append(self.crashlog.Frame(idx, pc, frame_offset)) 542 543 # on arm64 systems, if it jump through a null function pointer, 544 # we end up at address 0 and the crash reporter unwinder 545 # misses the frame that actually faulted. 546 # But $lr can tell us where the last BL/BLR instruction used 547 # was at, so insert that address as the caller stack frame. 548 if idx == 0 and pc == 0 and "lr" in thread.registers: 549 pc = thread.registers["lr"] 550 for image in self.data['usedImages']: 551 text_lo = image['base'] 552 text_hi = text_lo + image['size'] 553 if text_lo <= pc < text_hi: 554 idx += 1 555 frame_offset = pc - text_lo 556 thread.frames.append(self.crashlog.Frame(idx, pc, frame_offset)) 557 break 558 559 idx += 1 560 561 def parse_threads(self, json_threads): 562 idx = 0 563 for json_thread in json_threads: 564 thread = self.crashlog.Thread(idx, False) 565 if 'name' in json_thread: 566 thread.name = json_thread['name'] 567 thread.reason = json_thread['name'] 568 if 'id' in json_thread: 569 thread.id = int(json_thread['id']) 570 if json_thread.get('triggered', False): 571 self.crashlog.crashed_thread_idx = idx 572 thread.crashed = True 573 if 'threadState' in json_thread: 574 thread.registers = self.parse_thread_registers( 575 json_thread['threadState']) 576 if 'queue' in json_thread: 577 thread.queue = json_thread.get('queue') 578 self.parse_frames(thread, json_thread.get('frames', [])) 579 self.crashlog.threads.append(thread) 580 idx += 1 581 582 def parse_asi_backtrace(self, thread, bt): 583 for line in bt.split('\n'): 584 frame_match = TextCrashLogParser.frame_regex.search(line) 585 if not frame_match: 586 print("error: can't parse application specific backtrace.") 587 return False 588 589 (frame_id, frame_img_name, frame_addr, 590 frame_ofs) = frame_match.groups() 591 592 thread.add_ident(frame_img_name) 593 if frame_img_name not in self.crashlog.idents: 594 self.crashlog.idents.append(frame_img_name) 595 thread.frames.append(self.crashlog.Frame(int(frame_id), int( 596 frame_addr, 0), frame_ofs)) 597 598 return True 599 600 def parse_app_specific_backtraces(self, json_app_specific_bts): 601 for idx, backtrace in enumerate(json_app_specific_bts): 602 thread = self.crashlog.Thread(idx, True) 603 thread.queue = "Application Specific Backtrace" 604 if self.parse_asi_backtrace(thread, backtrace): 605 self.crashlog.threads.append(thread) 606 607 def parse_thread_registers(self, json_thread_state, prefix=None): 608 registers = dict() 609 for key, state in json_thread_state.items(): 610 if key == "rosetta": 611 registers.update(self.parse_thread_registers(state)) 612 continue 613 if key == "x": 614 gpr_dict = { str(idx) : reg for idx,reg in enumerate(state) } 615 registers.update(self.parse_thread_registers(gpr_dict, key)) 616 continue 617 try: 618 value = int(state['value']) 619 registers["{}{}".format(prefix or '',key)] = value 620 except (KeyError, ValueError, TypeError): 621 pass 622 return registers 623 624 def parse_errors(self, json_data): 625 if 'reportNotes' in json_data: 626 self.crashlog.errors = json_data['reportNotes'] 627 628 629class CrashLogParseMode: 630 NORMAL = 0 631 THREAD = 1 632 IMAGES = 2 633 THREGS = 3 634 SYSTEM = 4 635 INSTRS = 5 636 637class TextCrashLogParser(CrashLogParser): 638 parent_process_regex = re.compile(r'^Parent Process:\s*(.*)\[(\d+)\]') 639 thread_state_regex = re.compile(r'^Thread \d+ crashed with') 640 thread_instrs_regex = re.compile(r'^Thread \d+ instruction stream') 641 thread_regex = re.compile(r'^Thread (\d+).*:') 642 app_backtrace_regex = re.compile(r'^Application Specific Backtrace (\d+).*:') 643 version = r'\(.+\)|(?:arm|x86_)[0-9a-z]+' 644 frame_regex = re.compile(r'^(\d+)\s+' # id 645 r'(.+?)\s+' # img_name 646 r'(?:' +version+ r'\s+)?' # img_version 647 r'(0x[0-9a-fA-F]{4,})' # addr (4 chars or more) 648 r'(?: +(.*))?' # offs 649 ) 650 null_frame_regex = re.compile(r'^\d+\s+\?\?\?\s+0{4,} +') 651 image_regex_uuid = re.compile(r'(0x[0-9a-fA-F]+)' # img_lo 652 r'\s+-\s+' # - 653 r'(0x[0-9a-fA-F]+)\s+' # img_hi 654 r'[+]?(.+?)\s+' # img_name 655 r'(?:(' +version+ r')\s+)?' # img_version 656 r'(?:<([-0-9a-fA-F]+)>\s+)?' # img_uuid 657 r'(\?+|/.*)' # img_path 658 ) 659 exception_type_regex = re.compile(r'^Exception Type:\s+(EXC_[A-Z_]+)(?:\s+\((.*)\))?') 660 exception_codes_regex = re.compile(r'^Exception Codes:\s+(0x[0-9a-fA-F]+),\s*(0x[0-9a-fA-F]+)') 661 exception_extra_regex = re.compile(r'^Exception\s+.*:\s+(.*)') 662 663 def __init__(self, debugger, path, verbose): 664 super().__init__(debugger, path, verbose) 665 self.thread = None 666 self.app_specific_backtrace = False 667 self.parse_mode = CrashLogParseMode.NORMAL 668 self.parsers = { 669 CrashLogParseMode.NORMAL : self.parse_normal, 670 CrashLogParseMode.THREAD : self.parse_thread, 671 CrashLogParseMode.IMAGES : self.parse_images, 672 CrashLogParseMode.THREGS : self.parse_thread_registers, 673 CrashLogParseMode.SYSTEM : self.parse_system, 674 CrashLogParseMode.INSTRS : self.parse_instructions, 675 } 676 677 def parse(self): 678 with open(self.path,'r', encoding='utf-8') as f: 679 lines = f.read().splitlines() 680 681 for line in lines: 682 line_len = len(line) 683 if line_len == 0: 684 if self.thread: 685 if self.parse_mode == CrashLogParseMode.THREAD: 686 if self.thread.index == self.crashlog.crashed_thread_idx: 687 self.thread.reason = '' 688 if hasattr(self.crashlog, 'thread_exception'): 689 self.thread.reason += self.crashlog.thread_exception 690 if hasattr(self.crashlog, 'thread_exception_data'): 691 self.thread.reason += " (%s)" % self.crashlog.thread_exception_data 692 if self.app_specific_backtrace: 693 self.crashlog.backtraces.append(self.thread) 694 else: 695 self.crashlog.threads.append(self.thread) 696 self.thread = None 697 else: 698 # only append an extra empty line if the previous line 699 # in the info_lines wasn't empty 700 if len(self.crashlog.info_lines) > 0 and len(self.crashlog.info_lines[-1]): 701 self.crashlog.info_lines.append(line) 702 self.parse_mode = CrashLogParseMode.NORMAL 703 else: 704 self.parsers[self.parse_mode](line) 705 706 return self.crashlog 707 708 def parse_exception(self, line): 709 if not line.startswith('Exception'): 710 return 711 if line.startswith('Exception Type:'): 712 self.crashlog.thread_exception = line[15:].strip() 713 exception_type_match = self.exception_type_regex.search(line) 714 if exception_type_match: 715 exc_type, exc_signal = exception_type_match.groups() 716 self.crashlog.exception['type'] = exc_type 717 if exc_signal: 718 self.crashlog.exception['signal'] = exc_signal 719 elif line.startswith('Exception Subtype:'): 720 self.crashlog.thread_exception_subtype = line[18:].strip() 721 if 'type' in self.crashlog.exception: 722 self.crashlog.exception['subtype'] = self.crashlog.thread_exception_subtype 723 elif line.startswith('Exception Codes:'): 724 self.crashlog.thread_exception_data = line[16:].strip() 725 if 'type' not in self.crashlog.exception: 726 return 727 exception_codes_match = self.exception_codes_regex.search(line) 728 if exception_codes_match: 729 self.crashlog.exception['codes'] = self.crashlog.thread_exception_data 730 code, subcode = exception_codes_match.groups() 731 self.crashlog.exception['rawCodes'] = [int(code, base=16), 732 int(subcode, base=16)] 733 else: 734 if 'type' not in self.crashlog.exception: 735 return 736 exception_extra_match = self.exception_extra_regex.search(line) 737 if exception_extra_match: 738 self.crashlog.exception['message'] = exception_extra_match.group(1) 739 740 def parse_normal(self, line): 741 if line.startswith('Process:'): 742 (self.crashlog.process_name, pid_with_brackets) = line[ 743 8:].strip().split(' [') 744 self.crashlog.process_id = pid_with_brackets.strip('[]') 745 elif line.startswith('Identifier:'): 746 self.crashlog.process_identifier = line[11:].strip() 747 elif line.startswith('Version:'): 748 version_string = line[8:].strip() 749 matched_pair = re.search("(.+)\((.+)\)", version_string) 750 if matched_pair: 751 self.crashlog.process_version = matched_pair.group(1) 752 self.crashlog.process_compatability_version = matched_pair.group( 753 2) 754 else: 755 self.crashlog.process = version_string 756 self.crashlog.process_compatability_version = version_string 757 elif self.parent_process_regex.search(line): 758 parent_process_match = self.parent_process_regex.search( 759 line) 760 self.crashlog.parent_process_name = parent_process_match.group(1) 761 self.crashlog.parent_process_id = parent_process_match.group(2) 762 elif line.startswith('Exception'): 763 self.parse_exception(line) 764 return 765 elif line.startswith('Crashed Thread:'): 766 self.crashlog.crashed_thread_idx = int(line[15:].strip().split()[0]) 767 return 768 elif line.startswith('Triggered by Thread:'): # iOS 769 self.crashlog.crashed_thread_idx = int(line[20:].strip().split()[0]) 770 return 771 elif line.startswith('Report Version:'): 772 self.crashlog.version = int(line[15:].strip()) 773 return 774 elif line.startswith('System Profile:'): 775 self.parse_mode = CrashLogParseMode.SYSTEM 776 return 777 elif (line.startswith('Interval Since Last Report:') or 778 line.startswith('Crashes Since Last Report:') or 779 line.startswith('Per-App Interval Since Last Report:') or 780 line.startswith('Per-App Crashes Since Last Report:') or 781 line.startswith('Sleep/Wake UUID:') or 782 line.startswith('Anonymous UUID:')): 783 # ignore these 784 return 785 elif line.startswith('Thread'): 786 thread_state_match = self.thread_state_regex.search(line) 787 if thread_state_match: 788 self.app_specific_backtrace = False 789 thread_state_match = self.thread_regex.search(line) 790 thread_idx = int(thread_state_match.group(1)) 791 self.parse_mode = CrashLogParseMode.THREGS 792 self.thread = self.crashlog.threads[thread_idx] 793 return 794 thread_insts_match = self.thread_instrs_regex.search(line) 795 if thread_insts_match: 796 self.parse_mode = CrashLogParseMode.INSTRS 797 return 798 thread_match = self.thread_regex.search(line) 799 if thread_match: 800 self.app_specific_backtrace = False 801 self.parse_mode = CrashLogParseMode.THREAD 802 thread_idx = int(thread_match.group(1)) 803 self.thread = self.crashlog.Thread(thread_idx, False) 804 return 805 return 806 elif line.startswith('Binary Images:'): 807 self.parse_mode = CrashLogParseMode.IMAGES 808 return 809 elif line.startswith('Application Specific Backtrace'): 810 app_backtrace_match = self.app_backtrace_regex.search(line) 811 if app_backtrace_match: 812 self.parse_mode = CrashLogParseMode.THREAD 813 self.app_specific_backtrace = True 814 idx = int(app_backtrace_match.group(1)) 815 self.thread = self.crashlog.Thread(idx, True) 816 elif line.startswith('Last Exception Backtrace:'): # iOS 817 self.parse_mode = CrashLogParseMode.THREAD 818 self.app_specific_backtrace = True 819 idx = 1 820 self.thread = self.crashlog.Thread(idx, True) 821 self.crashlog.info_lines.append(line.strip()) 822 823 def parse_thread(self, line): 824 if line.startswith('Thread'): 825 return 826 if self.null_frame_regex.search(line): 827 print('warning: thread parser ignored null-frame: "%s"' % line) 828 return 829 frame_match = self.frame_regex.search(line) 830 if frame_match: 831 (frame_id, frame_img_name, frame_addr, 832 frame_ofs) = frame_match.groups() 833 ident = frame_img_name 834 self.thread.add_ident(ident) 835 if ident not in self.crashlog.idents: 836 self.crashlog.idents.append(ident) 837 self.thread.frames.append(self.crashlog.Frame(int(frame_id), int( 838 frame_addr, 0), frame_ofs)) 839 else: 840 print('error: frame regex failed for line: "%s"' % line) 841 842 def parse_images(self, line): 843 image_match = self.image_regex_uuid.search(line) 844 if image_match: 845 (img_lo, img_hi, img_name, img_version, 846 img_uuid, img_path) = image_match.groups() 847 image = self.crashlog.DarwinImage(int(img_lo, 0), int(img_hi, 0), 848 img_name.strip(), 849 img_version.strip() 850 if img_version else "", 851 uuid.UUID(img_uuid), img_path, 852 self.verbose) 853 self.crashlog.images.append(image) 854 else: 855 print("error: image regex failed for: %s" % line) 856 857 858 def parse_thread_registers(self, line): 859 # "r12: 0x00007fff6b5939c8 r13: 0x0000000007000006 r14: 0x0000000000002a03 r15: 0x0000000000000c00" 860 reg_values = re.findall('([a-z0-9]+): (0x[0-9a-f]+)', line, re.I) 861 for reg, value in reg_values: 862 self.thread.registers[reg] = int(value, 16) 863 864 def parse_system(self, line): 865 self.crashlog.system_profile.append(line) 866 867 def parse_instructions(self, line): 868 pass 869 870 871def usage(): 872 print("Usage: lldb-symbolicate.py [-n name] executable-image") 873 sys.exit(0) 874 875 876def save_crashlog(debugger, command, exe_ctx, result, dict): 877 usage = "usage: %prog [options] <output-path>" 878 description = '''Export the state of current target into a crashlog file''' 879 parser = optparse.OptionParser( 880 description=description, 881 prog='save_crashlog', 882 usage=usage) 883 parser.add_option( 884 '-v', 885 '--verbose', 886 action='store_true', 887 dest='verbose', 888 help='display verbose debug info', 889 default=False) 890 try: 891 (options, args) = parser.parse_args(shlex.split(command)) 892 except: 893 result.PutCString("error: invalid options") 894 return 895 if len(args) != 1: 896 result.PutCString( 897 "error: invalid arguments, a single output file is the only valid argument") 898 return 899 out_file = open(args[0], 'w', encoding='utf-8') 900 if not out_file: 901 result.PutCString( 902 "error: failed to open file '%s' for writing...", 903 args[0]) 904 return 905 target = exe_ctx.target 906 if target: 907 identifier = target.executable.basename 908 process = exe_ctx.process 909 if process: 910 pid = process.id 911 if pid != lldb.LLDB_INVALID_PROCESS_ID: 912 out_file.write( 913 'Process: %s [%u]\n' % 914 (identifier, pid)) 915 out_file.write('Path: %s\n' % (target.executable.fullpath)) 916 out_file.write('Identifier: %s\n' % (identifier)) 917 out_file.write('\nDate/Time: %s\n' % 918 (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))) 919 out_file.write( 920 'OS Version: Mac OS X %s (%s)\n' % 921 (platform.mac_ver()[0], subprocess.check_output('sysctl -n kern.osversion', shell=True).decode("utf-8"))) 922 out_file.write('Report Version: 9\n') 923 for thread_idx in range(process.num_threads): 924 thread = process.thread[thread_idx] 925 out_file.write('\nThread %u:\n' % (thread_idx)) 926 for (frame_idx, frame) in enumerate(thread.frames): 927 frame_pc = frame.pc 928 frame_offset = 0 929 if frame.function: 930 block = frame.GetFrameBlock() 931 block_range = block.range[frame.addr] 932 if block_range: 933 block_start_addr = block_range[0] 934 frame_offset = frame_pc - block_start_addr.GetLoadAddress(target) 935 else: 936 frame_offset = frame_pc - frame.function.addr.GetLoadAddress(target) 937 elif frame.symbol: 938 frame_offset = frame_pc - frame.symbol.addr.GetLoadAddress(target) 939 out_file.write( 940 '%-3u %-32s 0x%16.16x %s' % 941 (frame_idx, frame.module.file.basename, frame_pc, frame.name)) 942 if frame_offset > 0: 943 out_file.write(' + %u' % (frame_offset)) 944 line_entry = frame.line_entry 945 if line_entry: 946 if options.verbose: 947 # This will output the fullpath + line + column 948 out_file.write(' %s' % (line_entry)) 949 else: 950 out_file.write( 951 ' %s:%u' % 952 (line_entry.file.basename, line_entry.line)) 953 column = line_entry.column 954 if column: 955 out_file.write(':%u' % (column)) 956 out_file.write('\n') 957 958 out_file.write('\nBinary Images:\n') 959 for module in target.modules: 960 text_segment = module.section['__TEXT'] 961 if text_segment: 962 text_segment_load_addr = text_segment.GetLoadAddress(target) 963 if text_segment_load_addr != lldb.LLDB_INVALID_ADDRESS: 964 text_segment_end_load_addr = text_segment_load_addr + text_segment.size 965 identifier = module.file.basename 966 module_version = '???' 967 module_version_array = module.GetVersion() 968 if module_version_array: 969 module_version = '.'.join( 970 map(str, module_version_array)) 971 out_file.write( 972 ' 0x%16.16x - 0x%16.16x %s (%s - ???) <%s> %s\n' % 973 (text_segment_load_addr, 974 text_segment_end_load_addr, 975 identifier, 976 module_version, 977 module.GetUUIDString(), 978 module.file.fullpath)) 979 out_file.close() 980 else: 981 result.PutCString("error: invalid target") 982 983 984class Symbolicate: 985 def __init__(self, debugger, internal_dict): 986 pass 987 988 def __call__(self, debugger, command, exe_ctx, result): 989 SymbolicateCrashLogs(debugger, shlex.split(command), result) 990 991 def get_short_help(self): 992 return "Symbolicate one or more darwin crash log files." 993 994 def get_long_help(self): 995 option_parser = CrashLogOptionParser() 996 return option_parser.format_help() 997 998 999def SymbolicateCrashLog(crash_log, options): 1000 if options.debug: 1001 crash_log.dump() 1002 if not crash_log.images: 1003 print('error: no images in crash log') 1004 return 1005 1006 if options.dump_image_list: 1007 print("Binary Images:") 1008 for image in crash_log.images: 1009 if options.verbose: 1010 print(image.debug_dump()) 1011 else: 1012 print(image) 1013 1014 target = crash_log.create_target() 1015 if not target: 1016 return 1017 exe_module = target.GetModuleAtIndex(0) 1018 images_to_load = list() 1019 loaded_images = list() 1020 if options.load_all_images: 1021 # --load-all option was specified, load everything up 1022 for image in crash_log.images: 1023 images_to_load.append(image) 1024 else: 1025 # Only load the images found in stack frames for the crashed threads 1026 if options.crashed_only: 1027 for thread in crash_log.threads: 1028 if thread.did_crash(): 1029 for ident in thread.idents: 1030 images = crash_log.find_images_with_identifier(ident) 1031 if images: 1032 for image in images: 1033 images_to_load.append(image) 1034 else: 1035 print('error: can\'t find image for identifier "%s"' % ident) 1036 else: 1037 for ident in crash_log.idents: 1038 images = crash_log.find_images_with_identifier(ident) 1039 if images: 1040 for image in images: 1041 images_to_load.append(image) 1042 else: 1043 print('error: can\'t find image for identifier "%s"' % ident) 1044 1045 futures = [] 1046 with concurrent.futures.ThreadPoolExecutor() as executor: 1047 def add_module(image, target): 1048 return image, image.add_module(target) 1049 1050 for image in images_to_load: 1051 futures.append(executor.submit(add_module, image=image, target=target)) 1052 1053 for future in concurrent.futures.as_completed(futures): 1054 image, err = future.result() 1055 if err: 1056 print(err) 1057 else: 1058 loaded_images.append(image) 1059 1060 if crash_log.backtraces: 1061 for thread in crash_log.backtraces: 1062 thread.dump_symbolicated(crash_log, options) 1063 print() 1064 1065 for thread in crash_log.threads: 1066 thread.dump_symbolicated(crash_log, options) 1067 print() 1068 1069 if crash_log.errors: 1070 print("Errors:") 1071 for error in crash_log.errors: 1072 print(error) 1073 1074def load_crashlog_in_scripted_process(debugger, crash_log_file, options, result): 1075 crashlog_path = os.path.expanduser(crash_log_file) 1076 if not os.path.exists(crashlog_path): 1077 raise InteractiveCrashLogException("crashlog file %s does not exist" % crashlog_path) 1078 1079 crashlog = CrashLogParser.create(debugger, crashlog_path, False).parse() 1080 1081 target = lldb.SBTarget() 1082 # 1. Try to use the user-provided target 1083 if options.target_path: 1084 target = debugger.CreateTarget(options.target_path) 1085 if not target: 1086 raise InteractiveCrashLogException("couldn't create target provided by the user (%s)" % options.target_path) 1087 1088 # 2. If the user didn't provide a target, try to create a target using the symbolicator 1089 if not target or not target.IsValid(): 1090 target = crashlog.create_target() 1091 # 3. If that didn't work, and a target is already loaded, use it 1092 if (target is None or not target.IsValid()) and debugger.GetNumTargets() > 0: 1093 target = debugger.GetTargetAtIndex(0) 1094 # 4. Fail 1095 if target is None or not target.IsValid(): 1096 raise InteractiveCrashLogException("couldn't create target") 1097 1098 ci = debugger.GetCommandInterpreter() 1099 if not ci: 1100 raise InteractiveCrashLogException("couldn't get command interpreter") 1101 1102 ci.HandleCommand('script from lldb.macosx import crashlog_scripted_process', result) 1103 if not result.Succeeded(): 1104 raise InteractiveCrashLogException("couldn't import crashlog scripted process module") 1105 1106 structured_data = lldb.SBStructuredData() 1107 structured_data.SetFromJSON(json.dumps({ "file_path" : crashlog_path, 1108 "load_all_images": options.load_all_images })) 1109 launch_info = lldb.SBLaunchInfo(None) 1110 launch_info.SetProcessPluginName("ScriptedProcess") 1111 launch_info.SetScriptedProcessClassName("crashlog_scripted_process.CrashLogScriptedProcess") 1112 launch_info.SetScriptedProcessDictionary(structured_data) 1113 error = lldb.SBError() 1114 process = target.Launch(launch_info, error) 1115 1116 if not process or error.Fail(): 1117 raise InteractiveCrashLogException("couldn't launch Scripted Process", error) 1118 1119 if not options.skip_status: 1120 @contextlib.contextmanager 1121 def synchronous(debugger): 1122 async_state = debugger.GetAsync() 1123 debugger.SetAsync(False) 1124 try: 1125 yield 1126 finally: 1127 debugger.SetAsync(async_state) 1128 1129 with synchronous(debugger): 1130 run_options = lldb.SBCommandInterpreterRunOptions() 1131 run_options.SetStopOnError(True) 1132 run_options.SetStopOnCrash(True) 1133 run_options.SetEchoCommands(True) 1134 1135 commands_stream = lldb.SBStream() 1136 commands_stream.Print("process status --verbose\n") 1137 commands_stream.Print("thread backtrace --extended true\n") 1138 error = debugger.SetInputString(commands_stream.GetData()) 1139 if error.Success(): 1140 debugger.RunCommandInterpreter(True, False, run_options, 0, False, True) 1141 1142def CreateSymbolicateCrashLogOptions( 1143 command_name, 1144 description, 1145 add_interactive_options): 1146 usage = "usage: %prog [options] <FILE> [FILE ...]" 1147 option_parser = optparse.OptionParser( 1148 description=description, prog='crashlog', usage=usage) 1149 option_parser.add_option( 1150 '--version', 1151 '-V', 1152 dest='version', 1153 action='store_true', 1154 help='Show crashlog version', 1155 default=False) 1156 option_parser.add_option( 1157 '--verbose', 1158 '-v', 1159 action='store_true', 1160 dest='verbose', 1161 help='display verbose debug info', 1162 default=False) 1163 option_parser.add_option( 1164 '--debug', 1165 '-g', 1166 action='store_true', 1167 dest='debug', 1168 help='display verbose debug logging', 1169 default=False) 1170 option_parser.add_option( 1171 '--load-all', 1172 '-a', 1173 action='store_true', 1174 dest='load_all_images', 1175 help='load all executable images, not just the images found in the ' 1176 'crashed stack frames, loads stackframes for all the threads in ' 1177 'interactive mode.', 1178 default=False) 1179 option_parser.add_option( 1180 '--images', 1181 action='store_true', 1182 dest='dump_image_list', 1183 help='show image list', 1184 default=False) 1185 option_parser.add_option( 1186 '--debug-delay', 1187 type='int', 1188 dest='debug_delay', 1189 metavar='NSEC', 1190 help='pause for NSEC seconds for debugger', 1191 default=0) 1192 option_parser.add_option( 1193 '--crashed-only', 1194 '-c', 1195 action='store_true', 1196 dest='crashed_only', 1197 help='only symbolicate the crashed thread', 1198 default=False) 1199 option_parser.add_option( 1200 '--disasm-depth', 1201 '-d', 1202 type='int', 1203 dest='disassemble_depth', 1204 help='set the depth in stack frames that should be disassembled (default is 1)', 1205 default=1) 1206 option_parser.add_option( 1207 '--disasm-all', 1208 '-D', 1209 action='store_true', 1210 dest='disassemble_all_threads', 1211 help='enabled disassembly of frames on all threads (not just the crashed thread)', 1212 default=False) 1213 option_parser.add_option( 1214 '--disasm-before', 1215 '-B', 1216 type='int', 1217 dest='disassemble_before', 1218 help='the number of instructions to disassemble before the frame PC', 1219 default=4) 1220 option_parser.add_option( 1221 '--disasm-after', 1222 '-A', 1223 type='int', 1224 dest='disassemble_after', 1225 help='the number of instructions to disassemble after the frame PC', 1226 default=4) 1227 option_parser.add_option( 1228 '--source-context', 1229 '-C', 1230 type='int', 1231 metavar='NLINES', 1232 dest='source_context', 1233 help='show NLINES source lines of source context (default = 4)', 1234 default=4) 1235 option_parser.add_option( 1236 '--source-frames', 1237 type='int', 1238 metavar='NFRAMES', 1239 dest='source_frames', 1240 help='show source for NFRAMES (default = 4)', 1241 default=4) 1242 option_parser.add_option( 1243 '--source-all', 1244 action='store_true', 1245 dest='source_all', 1246 help='show source for all threads, not just the crashed thread', 1247 default=False) 1248 if add_interactive_options: 1249 option_parser.add_option( 1250 '-i', 1251 '--interactive', 1252 action='store_true', 1253 help='parse a crash log and load it in a ScriptedProcess', 1254 default=False) 1255 option_parser.add_option( 1256 '-b', 1257 '--batch', 1258 action='store_true', 1259 help='dump symbolicated stackframes without creating a debug session', 1260 default=True) 1261 option_parser.add_option( 1262 '--target', 1263 '-t', 1264 dest='target_path', 1265 help='the target binary path that should be used for interactive crashlog (optional)', 1266 default=None) 1267 option_parser.add_option( 1268 '--skip-status', 1269 '-s', 1270 dest='skip_status', 1271 action='store_true', 1272 help='prevent the interactive crashlog to dump the process status and thread backtrace at launch', 1273 default=False) 1274 return option_parser 1275 1276 1277def CrashLogOptionParser(): 1278 description = '''Symbolicate one or more darwin crash log files to provide source file and line information, 1279inlined stack frames back to the concrete functions, and disassemble the location of the crash 1280for the first frame of the crashed thread. 1281If this script is imported into the LLDB command interpreter, a "crashlog" command will be added to the interpreter 1282for use at the LLDB command line. After a crash log has been parsed and symbolicated, a target will have been 1283created that has all of the shared libraries loaded at the load addresses found in the crash log file. This allows 1284you to explore the program as if it were stopped at the locations described in the crash log and functions can 1285be disassembled and lookups can be performed using the addresses found in the crash log.''' 1286 return CreateSymbolicateCrashLogOptions('crashlog', description, True) 1287 1288def SymbolicateCrashLogs(debugger, command_args, result): 1289 option_parser = CrashLogOptionParser() 1290 1291 if not len(command_args): 1292 option_parser.print_help() 1293 return 1294 1295 try: 1296 (options, args) = option_parser.parse_args(command_args) 1297 except: 1298 return 1299 1300 if options.version: 1301 print(debugger.GetVersionString()) 1302 return 1303 1304 if options.debug: 1305 print('command_args = %s' % command_args) 1306 print('options', options) 1307 print('args', args) 1308 1309 if options.debug_delay > 0: 1310 print("Waiting %u seconds for debugger to attach..." % options.debug_delay) 1311 time.sleep(options.debug_delay) 1312 error = lldb.SBError() 1313 1314 def should_run_in_interactive_mode(options, ci): 1315 if options.interactive: 1316 return True 1317 elif options.batch: 1318 return False 1319 # elif ci and ci.IsInteractive(): 1320 # return True 1321 else: 1322 return False 1323 1324 ci = debugger.GetCommandInterpreter() 1325 1326 if args: 1327 for crash_log_file in args: 1328 if should_run_in_interactive_mode(options, ci): 1329 try: 1330 load_crashlog_in_scripted_process(debugger, crash_log_file, 1331 options, result) 1332 except InteractiveCrashLogException as e: 1333 result.SetError(str(e)) 1334 else: 1335 crash_log = CrashLogParser.create(debugger, crash_log_file, options.verbose).parse() 1336 SymbolicateCrashLog(crash_log, options) 1337 1338if __name__ == '__main__': 1339 # Create a new debugger instance 1340 debugger = lldb.SBDebugger.Create() 1341 result = lldb.SBCommandReturnObject() 1342 SymbolicateCrashLogs(debugger, sys.argv[1:], result) 1343 lldb.SBDebugger.Destroy(debugger) 1344 1345def __lldb_init_module(debugger, internal_dict): 1346 debugger.HandleCommand( 1347 'command script add -o -c lldb.macosx.crashlog.Symbolicate crashlog') 1348 debugger.HandleCommand( 1349 'command script add -o -f lldb.macosx.crashlog.save_crashlog save_crashlog') 1350 print('"crashlog" and "save_crashlog" commands have been installed, use ' 1351 'the "--help" options on these commands for detailed help.') 1352