1#!/bin/sh 2# 3# Copyright (c) 2009 Mikolaj Golub. 4# All rights reserved. 5# 6# Some parts of the code (vmcore and kernel autodetection) were taken 7# from FreeBSD crashinfo(8) script. 8# 9# Redistribution and use in source and binary forms, with or without 10# modification, are permitted provided that the following conditions 11# are met: 12# 1. Redistributions of source code must retain the above copyright 13# notice, this list of conditions and the following disclaimer. 14# 2. Redistributions in binary form must reproduce the above copyright 15# notice, this list of conditions and the following disclaimer in the 16# documentation and/or other materials provided with the distribution. 17# 3. Neither the name of the author nor the names of any co-contributors 18# may be used to endorse or promote products derived from this software 19# without specific prior written permission. 20# 21# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 22# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 25# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 27# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 28# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 30# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31# SUCH DAMAGE. 32# 33# $Id$ 34 35# This script creates tar archive that contains all files needed for 36# debugging FreeBSD kernel crash (vmcore, kernel, loaded modules, 37# sources that appear in backtrace). This is useful for debugging a 38# crash on another host or sending it to developer. 39# 40# Created tar archive contains also a script that when being run 41# inside unpacked archive will give kgdb(1) session with crash core 42# loaded in it. The script should be run with root privileges because 43# it does chroot(8) before starting kgdb(1). 44# 45# WARNING: As core file presents a content of kernel virtual memory at 46# the moment of the crash it might contain information you would not 47# like to become public. So think twice before giving the access to 48# the core to other person. 49# 50 51set -ue 52 53# 54# Global vars 55# 56 57PROGNAME=`basename $0` 58CRASHDIR=/var/crash 59DUMPNR= 60VMCORE= 61KERNEL= 62TARFILE= 63TMPDIR= 64CRASH= 65 66# 67# Functions 68# 69 70# Print minihelp. 71 72usage() 73{ 74 75 echo 76 echo "usage: $PROGNAME [options] [<crash.tar.gz>]" 77 echo 78 echo "Options:" 79 echo 80 echo " -h print this help and exit" 81 echo " -d <crashdir> path to crash directory" 82 echo " -n <dumpnr> dump number" 83 echo " -k <kernel> path to kernel" 84 echo " -c <core> path to core file" 85 echo 86 echo " <crash.tar.gz> path to tar file where files will be stored" 87 echo 88} 89 90# Create directory we will use for storing temporary files and 91# data. Set trap to remove it on exit. 92 93mk_tmpdir () 94{ 95 96 if ! TMPDIR=`mktemp -dt $PROGNAME`; then 97 echo "Can't create tmp directory" >&2 98 exit 1 99 fi 100 101 chmod 0700 "$TMPDIR" 102 103 trap "rm -Rf '$TMPDIR'" INT QUIT EXIT 104 105} 106 107# Find kernel. 108 109find_kernel() 110{ 111 local ivers k kvers 112 113 ivers=$(awk ' 114 /Version String/ { 115 print 116 nextline=1 117 next 118 } 119 // { 120 if (nextline) { 121 print 122 nextline=0 123 } 124 }' $INFO) 125 126 # Look for a matching kernel version. 127 for k in `sysctl -n kern.bootfile` $(ls -t /boot/*/kernel); do 128 129 kvers=$(echo 'printf " Version String: %s", version' | \ 130 gdb -x /dev/stdin -batch $k 2>/dev/null) 131 132 if [ "$ivers" = "$kvers" ]; then 133 KERNEL=$k 134 break 135 fi 136 137 done 138} 139 140# Run kgdb and generate all info we need looking for files we want to 141# archive. 142 143run_kgdb() 144{ 145 local nthr i 146 147 # We run kgdb redirecting its output to fifo and read this output 148 # to make a decision about the next command to run. 149 150 mkfifo $TMPDIR/fifo 151 152 { 153 # Just generate some known output. 154 echo "show version" 155 156 # On start kgdb outputs modules it loads, like this one: 157 # "Loaded symbols for /boot/kernel/ng_socket.ko". 158 # Parse these lines to generate the list of modules. 159 160 sed -lnEe 's|^Loaded symbols for (/[^ ].*\.ko)(\.symbols)?$|\1|p; 161 /\(kgdb\)/q' > $TMPDIR/modules 162 163 # Find the number of threads from "info threads" command 164 # output. The thread with the max number is printed first. 165 echo "info threads" 166 nthr=`sed -nEe '/^.* [0-9]+ Thread [0-9]+.*$/{ 167 s/^.* ([0-9]+) Thread [0-9]+.*$/\1/p 168 q 169 }'` 170 171 # Output backtrace of every thread and parse output looking 172 # for source files. If a source path is not full run 173 # "maintenance info symtabs file" 174 175 i=$nthr 176 while [ "$i" -gt 0 ]; do 177 echo thread apply $i backtrace 178 i=$((i - 1)) 179 done 180 # Just generate some known output so we know where we is. 181 echo "show version" 182 183 > $TMPDIR/sources.nonunique 184 sed -lnEe 's|^.* at +([^:]*):[0-9]+$|\1|p; /GNU gdb.*FreeBSD/q' | 185 awk '{ 186 if (/^\//) 187 print >> "'$TMPDIR/sources.nonunique'" 188 else 189 print "maintenance info symtabs " $0 190 }' 191 # Just generate some known output so we know where we is. 192 echo "show version" 193 194 # Parse output of "maintenance info symtabs file" commands. 195 # It looks like this one: 196 # { symtab vm_page.h ((struct symtab *) 0x2d11d8d0) 197 # dirname /usr/src/sys/vm 198 # fullname (null) 199 # blockvector ((struct blockvector *) 0x2d11d690) 200 # debugformat unknown 201 # } 202 awk '$1 == "{" && $2 == "symtab" {file=$3} 203 file && $1 == "dirname" {print $2 "/" file; file = ""} 204 $1 == "}" {file = ""} 205 /GNU gdb.*FreeBSD/ {exit}' >> $TMPDIR/sources.nonunique 206 207 sort -u $TMPDIR/sources.nonunique > $TMPDIR/sources 208 209 # Find srcbase. 210 echo "break vn_open" 211 srcbase=`sed -nEe '/^.*Breakpoint 1 at .* file .*\/kern\/vfs_vnops.c,.*$/{ 212 s|^.*Breakpoint 1 at .* file (.*)/kern/vfs_vnops.c,.*$|\1|p 213 q 214 }'` 215 echo ${srcbase#/} > $TMPDIR/srcbase 216 find $srcbase/`uname -m`/include -type f -name '*.h' >> $TMPDIR/sources 217 218 # Send quit and wait for gdb to close fifo on its side (to avoid sigpipe). 219 echo "quit" 220 cat > /dev/null 221 222 } < $TMPDIR/fifo | 223 kgdb $KERNEL $VMCORE 2>/dev/null | tee $TMPDIR/kgdb.out > $TMPDIR/fifo 224} 225 226# 227# Main 228# 229 230while getopts "hd:n:k:c:" opt; do 231 232 case "$opt" in 233 234 h) 235 usage 236 exit 0 237 ;; 238 d) 239 CRASHDIR=$OPTARG 240 ;; 241 n) 242 DUMPNR=$OPTARG 243 ;; 244 k) 245 KERNEL=$OPTARG 246 ;; 247 c) 248 VMCORE=$OPTARG 249 ;; 250 \?) 251 usage >&2 252 exit 1 253 ;; 254 esac 255done 256 257mk_tmpdir 258 259if [ -n "$DUMPNR" -a -n "$VMCORE" ]; then 260 echo "-n and -c options are mutually exclusive" >&2 261 usage >&2 262 exit 1 263fi 264 265 266if [ -n "$VMCORE" ]; then 267 CRASHDIR=`dirname $VMCORE` 268fi 269 270if [ ! -x $CRASHDIR ]; then 271 echo "No access to crash directory $CRASHDIR" >&2 272 exit 1 273fi 274 275if [ -n "$VMCORE" ]; then 276 DUMPNR=$(expr $(basename $VMCORE) : 'vmcore\.\([0-9]*\)$') || : 277 if [ -z "$DUMPNR" ]; then 278 echo "Unable to determine dump number from vmcore file $VMCORE." >&2 279 exit 1 280 fi 281else 282 # If we don't have an explicit dump number, operate on the most 283 # recent dump. 284 if [ -z "$DUMPNR" ]; then 285 if ! [ -r $CRASHDIR/bounds ]; then 286 echo "No crash dumps in $CRASHDIR." >&2 287 exit 1 288 fi 289 next=`cat $CRASHDIR/bounds` 290 if [ -z "$next" ] || [ "$next" -eq 0 ]; then 291 echo "No crash dumps in $CRASHDIR." >&2 292 exit 1 293 fi 294 DUMPNR=$(($next - 1)) 295 fi 296fi 297 298shift $((OPTIND - 1)) 299 300if [ $# -gt 1 ]; then 301 usage >&2 302 exit 1 303elif [ $# -eq 1 ]; then 304 TARFILE=$1 305else 306 TARFILE=$CRASHDIR/crash.$DUMPNR.tar.gz 307fi 308 309if ! echo $TARFILE | grep -q '\.tar\.gz$'; then 310 echo "crash tarfile name should have '.tar.gz' extension" >&2 311 usage >&2 312 exit 1 313fi 314 315CRASH=`basename $TARFILE .tar.gz` 316VMCORE=$CRASHDIR/vmcore.$DUMPNR 317INFO=$CRASHDIR/info.$DUMPNR 318 319if [ ! -e $VMCORE ]; then 320 echo "$VMCORE not found" >&2 321 exit 1 322fi 323 324if [ ! -r $VMCORE ]; then 325 echo "$VMCORE not readable" >&2 326 exit 1 327fi 328 329if [ ! -e $INFO ]; then 330 echo "$INFO not found" >&2 331 exit 1 332fi 333 334if [ ! -r $INFO ]; then 335 echo "$INFO not readable" >&2 336 exit 1 337fi 338 339# If the user didn't specify a kernel, then try to find one. 340if [ -z "$KERNEL" ]; then 341 find_kernel 342 if [ -z "$KERNEL" ]; then 343 echo "Unable to find matching kernel for $VMCORE" >&2 344 exit 1 345 fi 346elif [ ! -e $KERNEL ]; then 347 echo "$KERNEL not found" >&2 348 exit 1 349fi 350 351run_kgdb; 352 353mkdir $TMPDIR/$CRASH 354 355mv $TMPDIR/kgdb.out $TMPDIR/$CRASH 356 357ln -s `cat $TMPDIR/srcbase`/`uname -m`/include "$TMPDIR/$CRASH/machine" 358 359{ 360 cat $TMPDIR/sources 361 362 echo $VMCORE 363 echo $INFO 364 365 for f in $KERNEL `cat $TMPDIR/modules`; do 366 echo $f 367 echo $f.symbols 368 done 369 370} | cpio -pvd "$TMPDIR/$CRASH" 2>/dev/null || : 371 372cat > "$TMPDIR/$CRASH/debug.sh" << EOF 373#!/bin/sh 374{ 375 echo /libexec/ld-elf.so.1 376 echo /usr/bin/kgdb 377 ldd /usr/bin/kgdb | 378 awk '\$3 ~ /\.so\./ {print \$3}' 379} | cpio -pvd . 380 381chroot . /usr/bin/kgdb '$KERNEL' '$VMCORE' 382 383EOF 384 385chmod a+x "$TMPDIR/$CRASH/debug.sh" 386 387cat > "$TMPDIR/$CRASH/README" <<EOF 388Run ./debug.sh under root to debug the crashdump. 389EOF 390 391echo "Archiving the crash to $TARFILE." 392 393touch "$TARFILE" 394chmod 0600 "$TARFILE" 395tar -C "$TMPDIR" -czf "$TARFILE" $CRASH 396