1 /*- 2 * Copyright (c) 2006 Peter Wemm 3 * Copyright (c) 2015 The FreeBSD Foundation 4 * All rights reserved. 5 * 6 * This software was developed by Andrew Turner under 7 * sponsorship from the FreeBSD Foundation. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD$"); 34 35 #include "opt_watchdog.h" 36 37 #include <sys/param.h> 38 #include <sys/systm.h> 39 #include <sys/conf.h> 40 #include <sys/cons.h> 41 #include <sys/kernel.h> 42 #include <sys/kerneldump.h> 43 #include <sys/msgbuf.h> 44 #include <sys/watchdog.h> 45 #include <sys/vmmeter.h> 46 47 #include <vm/vm.h> 48 #include <vm/vm_param.h> 49 #include <vm/vm_page.h> 50 #include <vm/vm_phys.h> 51 #include <vm/pmap.h> 52 53 #include <machine/md_var.h> 54 #include <machine/pte.h> 55 #include <machine/minidump.h> 56 57 CTASSERT(sizeof(struct kerneldumpheader) == 512); 58 59 uint64_t *vm_page_dump; 60 int vm_page_dump_size; 61 62 static struct kerneldumpheader kdh; 63 64 /* Handle chunked writes. */ 65 static size_t fragsz; 66 static void *dump_va; 67 static size_t counter, progress, dumpsize; 68 69 static uint64_t tmpbuffer[Ln_ENTRIES]; 70 71 CTASSERT(sizeof(*vm_page_dump) == 8); 72 73 static int 74 is_dumpable(vm_paddr_t pa) 75 { 76 vm_page_t m; 77 int i; 78 79 if ((m = vm_phys_paddr_to_vm_page(pa)) != NULL) 80 return ((m->flags & PG_NODUMP) == 0); 81 for (i = 0; dump_avail[i] != 0 || dump_avail[i + 1] != 0; i += 2) { 82 if (pa >= dump_avail[i] && pa < dump_avail[i + 1]) 83 return (1); 84 } 85 return (0); 86 } 87 88 static int 89 blk_flush(struct dumperinfo *di) 90 { 91 int error; 92 93 if (fragsz == 0) 94 return (0); 95 96 error = dump_append(di, dump_va, 0, fragsz); 97 fragsz = 0; 98 return (error); 99 } 100 101 static struct { 102 int min_per; 103 int max_per; 104 int visited; 105 } progress_track[10] = { 106 { 0, 10, 0}, 107 { 10, 20, 0}, 108 { 20, 30, 0}, 109 { 30, 40, 0}, 110 { 40, 50, 0}, 111 { 50, 60, 0}, 112 { 60, 70, 0}, 113 { 70, 80, 0}, 114 { 80, 90, 0}, 115 { 90, 100, 0} 116 }; 117 118 static void 119 report_progress(size_t progress, size_t dumpsize) 120 { 121 int sofar, i; 122 123 sofar = 100 - ((progress * 100) / dumpsize); 124 for (i = 0; i < nitems(progress_track); i++) { 125 if (sofar < progress_track[i].min_per || 126 sofar > progress_track[i].max_per) 127 continue; 128 if (progress_track[i].visited) 129 return; 130 progress_track[i].visited = 1; 131 printf("..%d%%", sofar); 132 return; 133 } 134 } 135 136 static int 137 blk_write(struct dumperinfo *di, char *ptr, vm_paddr_t pa, size_t sz) 138 { 139 size_t len; 140 int error, c; 141 u_int maxdumpsz; 142 143 maxdumpsz = min(di->maxiosize, MAXDUMPPGS * PAGE_SIZE); 144 if (maxdumpsz == 0) /* seatbelt */ 145 maxdumpsz = PAGE_SIZE; 146 error = 0; 147 if ((sz % PAGE_SIZE) != 0) { 148 printf("size not page aligned\n"); 149 return (EINVAL); 150 } 151 if (ptr != NULL && pa != 0) { 152 printf("cant have both va and pa!\n"); 153 return (EINVAL); 154 } 155 if ((((uintptr_t)pa) % PAGE_SIZE) != 0) { 156 printf("address not page aligned %p\n", ptr); 157 return (EINVAL); 158 } 159 if (ptr != NULL) { 160 /* 161 * If we're doing a virtual dump, flush any 162 * pre-existing pa pages. 163 */ 164 error = blk_flush(di); 165 if (error) 166 return (error); 167 } 168 while (sz) { 169 len = maxdumpsz - fragsz; 170 if (len > sz) 171 len = sz; 172 counter += len; 173 progress -= len; 174 if (counter >> 22) { 175 report_progress(progress, dumpsize); 176 counter &= (1 << 22) - 1; 177 } 178 179 wdog_kern_pat(WD_LASTVAL); 180 181 if (ptr) { 182 error = dump_append(di, ptr, 0, len); 183 if (error) 184 return (error); 185 ptr += len; 186 sz -= len; 187 } else { 188 dump_va = (void *)PHYS_TO_DMAP(pa); 189 fragsz += len; 190 pa += len; 191 sz -= len; 192 error = blk_flush(di); 193 if (error) 194 return (error); 195 } 196 197 /* Check for user abort. */ 198 c = cncheckc(); 199 if (c == 0x03) 200 return (ECANCELED); 201 if (c != -1) 202 printf(" (CTRL-C to abort) "); 203 } 204 205 return (0); 206 } 207 208 int 209 minidumpsys(struct dumperinfo *di) 210 { 211 struct minidumphdr mdhdr; 212 pd_entry_t *l0, *l1, *l2; 213 pt_entry_t *l3; 214 vm_offset_t va; 215 vm_paddr_t pa; 216 uint64_t bits; 217 uint32_t pmapsize; 218 int bit, error, i, j, retry_count; 219 220 retry_count = 0; 221 retry: 222 retry_count++; 223 error = 0; 224 pmapsize = 0; 225 for (va = VM_MIN_KERNEL_ADDRESS; va < kernel_vm_end; va += L2_SIZE) { 226 pmapsize += PAGE_SIZE; 227 if (!pmap_get_tables(pmap_kernel(), va, &l0, &l1, &l2, &l3)) 228 continue; 229 230 if ((*l1 & ATTR_DESCR_MASK) == L1_BLOCK) { 231 pa = *l1 & ~ATTR_MASK; 232 for (i = 0; i < Ln_ENTRIES * Ln_ENTRIES; 233 i++, pa += PAGE_SIZE) 234 if (is_dumpable(pa)) 235 dump_add_page(pa); 236 pmapsize += (Ln_ENTRIES - 1) * PAGE_SIZE; 237 va += L1_SIZE - L2_SIZE; 238 } else if ((*l2 & ATTR_DESCR_MASK) == L2_BLOCK) { 239 pa = *l2 & ~ATTR_MASK; 240 for (i = 0; i < Ln_ENTRIES; i++, pa += PAGE_SIZE) { 241 if (is_dumpable(pa)) 242 dump_add_page(pa); 243 } 244 } else if ((*l2 & ATTR_DESCR_MASK) == L2_TABLE) { 245 for (i = 0; i < Ln_ENTRIES; i++) { 246 if ((l3[i] & ATTR_DESCR_MASK) != L3_PAGE) 247 continue; 248 pa = l3[i] & ~ATTR_MASK; 249 if (is_dumpable(pa)) 250 dump_add_page(pa); 251 } 252 } 253 } 254 255 /* Calculate dump size. */ 256 dumpsize = pmapsize; 257 dumpsize += round_page(msgbufp->msg_size); 258 dumpsize += round_page(vm_page_dump_size); 259 for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { 260 bits = vm_page_dump[i]; 261 while (bits) { 262 bit = ffsl(bits) - 1; 263 pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + 264 bit) * PAGE_SIZE; 265 /* Clear out undumpable pages now if needed */ 266 if (is_dumpable(pa)) 267 dumpsize += PAGE_SIZE; 268 else 269 dump_drop_page(pa); 270 bits &= ~(1ul << bit); 271 } 272 } 273 dumpsize += PAGE_SIZE; 274 275 progress = dumpsize; 276 277 /* Initialize mdhdr */ 278 bzero(&mdhdr, sizeof(mdhdr)); 279 strcpy(mdhdr.magic, MINIDUMP_MAGIC); 280 mdhdr.version = MINIDUMP_VERSION; 281 mdhdr.msgbufsize = msgbufp->msg_size; 282 mdhdr.bitmapsize = vm_page_dump_size; 283 mdhdr.pmapsize = pmapsize; 284 mdhdr.kernbase = VM_MIN_KERNEL_ADDRESS; 285 mdhdr.dmapphys = DMAP_MIN_PHYSADDR; 286 mdhdr.dmapbase = DMAP_MIN_ADDRESS; 287 mdhdr.dmapend = DMAP_MAX_ADDRESS; 288 289 dump_init_header(di, &kdh, KERNELDUMPMAGIC, KERNELDUMP_AARCH64_VERSION, 290 dumpsize); 291 292 error = dump_start(di, &kdh); 293 if (error != 0) 294 goto fail; 295 296 printf("Dumping %llu out of %ju MB:", (long long)dumpsize >> 20, 297 ptoa((uintmax_t)physmem) / 1048576); 298 299 /* Dump my header */ 300 bzero(&tmpbuffer, sizeof(tmpbuffer)); 301 bcopy(&mdhdr, &tmpbuffer, sizeof(mdhdr)); 302 error = blk_write(di, (char *)&tmpbuffer, 0, PAGE_SIZE); 303 if (error) 304 goto fail; 305 306 /* Dump msgbuf up front */ 307 error = blk_write(di, (char *)msgbufp->msg_ptr, 0, 308 round_page(msgbufp->msg_size)); 309 if (error) 310 goto fail; 311 312 /* Dump bitmap */ 313 error = blk_write(di, (char *)vm_page_dump, 0, 314 round_page(vm_page_dump_size)); 315 if (error) 316 goto fail; 317 318 /* Dump kernel page directory pages */ 319 bzero(&tmpbuffer, sizeof(tmpbuffer)); 320 for (va = VM_MIN_KERNEL_ADDRESS; va < kernel_vm_end; va += L2_SIZE) { 321 if (!pmap_get_tables(pmap_kernel(), va, &l0, &l1, &l2, &l3)) { 322 /* We always write a page, even if it is zero */ 323 error = blk_write(di, (char *)&tmpbuffer, 0, PAGE_SIZE); 324 if (error) 325 goto fail; 326 /* flush, in case we reuse tmpbuffer in the same block*/ 327 error = blk_flush(di); 328 if (error) 329 goto fail; 330 } else if ((*l1 & ATTR_DESCR_MASK) == L1_BLOCK) { 331 /* 332 * Handle a 1GB block mapping: write out 512 fake L2 333 * pages. 334 */ 335 pa = (*l1 & ~ATTR_MASK) | (va & L1_OFFSET); 336 337 for (i = 0; i < Ln_ENTRIES; i++) { 338 for (j = 0; j < Ln_ENTRIES; j++) { 339 tmpbuffer[j] = pa + i * L2_SIZE + 340 j * PAGE_SIZE | ATTR_DEFAULT | 341 L3_PAGE; 342 } 343 error = blk_write(di, (char *)&tmpbuffer, 0, 344 PAGE_SIZE); 345 if (error) 346 goto fail; 347 } 348 /* flush, in case we reuse tmpbuffer in the same block*/ 349 error = blk_flush(di); 350 if (error) 351 goto fail; 352 bzero(&tmpbuffer, sizeof(tmpbuffer)); 353 va += L1_SIZE - L2_SIZE; 354 } else if ((*l2 & ATTR_DESCR_MASK) == L2_BLOCK) { 355 pa = (*l2 & ~ATTR_MASK) | (va & L2_OFFSET); 356 357 /* Generate fake l3 entries based upon the l1 entry */ 358 for (i = 0; i < Ln_ENTRIES; i++) { 359 tmpbuffer[i] = pa + (i * PAGE_SIZE) | 360 ATTR_DEFAULT | L3_PAGE; 361 } 362 error = blk_write(di, (char *)&tmpbuffer, 0, PAGE_SIZE); 363 if (error) 364 goto fail; 365 /* flush, in case we reuse fakepd in the same block */ 366 error = blk_flush(di); 367 if (error) 368 goto fail; 369 bzero(&tmpbuffer, sizeof(tmpbuffer)); 370 continue; 371 } else { 372 pa = *l2 & ~ATTR_MASK; 373 374 error = blk_write(di, NULL, pa, PAGE_SIZE); 375 if (error) 376 goto fail; 377 } 378 } 379 380 /* Dump memory chunks */ 381 for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { 382 bits = vm_page_dump[i]; 383 while (bits) { 384 bit = ffsl(bits) - 1; 385 pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + 386 bit) * PAGE_SIZE; 387 error = blk_write(di, 0, pa, PAGE_SIZE); 388 if (error) 389 goto fail; 390 bits &= ~(1ul << bit); 391 } 392 } 393 394 error = blk_flush(di); 395 if (error) 396 goto fail; 397 398 error = dump_finish(di, &kdh); 399 if (error != 0) 400 goto fail; 401 402 printf("\nDump complete\n"); 403 return (0); 404 405 fail: 406 if (error < 0) 407 error = -error; 408 409 printf("\n"); 410 if (error == ENOSPC) { 411 printf("Dump map grown while dumping. "); 412 if (retry_count < 5) { 413 printf("Retrying...\n"); 414 goto retry; 415 } 416 printf("Dump failed.\n"); 417 } 418 else if (error == ECANCELED) 419 printf("Dump aborted\n"); 420 else if (error == E2BIG) { 421 printf("Dump failed. Partition too small (about %lluMB were " 422 "needed this time).\n", (long long)dumpsize >> 20); 423 } else 424 printf("** DUMP FAILED (ERROR %d) **\n", error); 425 return (error); 426 } 427 428 void 429 dump_add_page(vm_paddr_t pa) 430 { 431 int idx, bit; 432 433 pa >>= PAGE_SHIFT; 434 idx = pa >> 6; /* 2^6 = 64 */ 435 bit = pa & 63; 436 atomic_set_long(&vm_page_dump[idx], 1ul << bit); 437 } 438 439 void 440 dump_drop_page(vm_paddr_t pa) 441 { 442 int idx, bit; 443 444 pa >>= PAGE_SHIFT; 445 idx = pa >> 6; /* 2^6 = 64 */ 446 bit = pa & 63; 447 atomic_clear_long(&vm_page_dump[idx], 1ul << bit); 448 } 449