1 /*- 2 * Copyright (c) 2006 Peter Wemm 3 * Copyright (c) 2015 The FreeBSD Foundation 4 * All rights reserved. 5 * 6 * This software was developed by Andrew Turner under 7 * sponsorship from the FreeBSD Foundation. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 #include "opt_watchdog.h" 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/conf.h> 38 #include <sys/cons.h> 39 #include <sys/kernel.h> 40 #include <sys/kerneldump.h> 41 #include <sys/msgbuf.h> 42 #include <sys/watchdog.h> 43 #include <sys/vmmeter.h> 44 45 #include <vm/vm.h> 46 #include <vm/vm_param.h> 47 #include <vm/vm_page.h> 48 #include <vm/vm_phys.h> 49 #include <vm/vm_dumpset.h> 50 #include <vm/pmap.h> 51 52 #include <machine/atomic.h> 53 #include <machine/md_var.h> 54 #include <machine/pte.h> 55 #include <machine/minidump.h> 56 57 CTASSERT(sizeof(struct kerneldumpheader) == 512); 58 59 static struct kerneldumpheader kdh; 60 61 /* Handle chunked writes. */ 62 static size_t fragsz; 63 static void *dump_va; 64 static size_t dumpsize; 65 66 static uint64_t tmpbuffer[Ln_ENTRIES]; 67 68 static int 69 blk_flush(struct dumperinfo *di) 70 { 71 int error; 72 73 if (fragsz == 0) 74 return (0); 75 76 error = dump_append(di, dump_va, fragsz); 77 fragsz = 0; 78 return (error); 79 } 80 81 static int 82 blk_write(struct dumperinfo *di, char *ptr, vm_paddr_t pa, size_t sz) 83 { 84 size_t len; 85 int error, c; 86 u_int maxdumpsz; 87 88 maxdumpsz = min(di->maxiosize, MAXDUMPPGS * PAGE_SIZE); 89 if (maxdumpsz == 0) /* seatbelt */ 90 maxdumpsz = PAGE_SIZE; 91 error = 0; 92 if ((sz % PAGE_SIZE) != 0) { 93 printf("size not page aligned\n"); 94 return (EINVAL); 95 } 96 if (ptr != NULL && pa != 0) { 97 printf("cant have both va and pa!\n"); 98 return (EINVAL); 99 } 100 if ((((uintptr_t)pa) % PAGE_SIZE) != 0) { 101 printf("address not page aligned %p\n", ptr); 102 return (EINVAL); 103 } 104 if (ptr != NULL) { 105 /* 106 * If we're doing a virtual dump, flush any 107 * pre-existing pa pages. 108 */ 109 error = blk_flush(di); 110 if (error) 111 return (error); 112 } 113 while (sz) { 114 len = maxdumpsz - fragsz; 115 if (len > sz) 116 len = sz; 117 118 dumpsys_pb_progress(len); 119 wdog_kern_pat(WD_LASTVAL); 120 121 if (ptr) { 122 error = dump_append(di, ptr, len); 123 if (error) 124 return (error); 125 ptr += len; 126 sz -= len; 127 } else { 128 dump_va = (void *)PHYS_TO_DMAP(pa); 129 fragsz += len; 130 pa += len; 131 sz -= len; 132 error = blk_flush(di); 133 if (error) 134 return (error); 135 } 136 137 /* Check for user abort. */ 138 c = cncheckc(); 139 if (c == 0x03) 140 return (ECANCELED); 141 if (c != -1) 142 printf(" (CTRL-C to abort) "); 143 } 144 145 return (0); 146 } 147 148 int 149 cpu_minidumpsys(struct dumperinfo *di, const struct minidumpstate *state) 150 { 151 struct minidumphdr mdhdr; 152 struct msgbuf *mbp; 153 pd_entry_t *l0, *l1, l1e, *l2, l2e; 154 pt_entry_t *l3, l3e; 155 vm_offset_t va, kva_end; 156 vm_paddr_t pa; 157 uint32_t pmapsize; 158 int error, i, j, retry_count; 159 160 retry_count = 0; 161 retry: 162 retry_count++; 163 error = 0; 164 pmapsize = 0; 165 166 /* Snapshot the KVA upper bound in case it grows. */ 167 kva_end = kernel_vm_end; 168 169 /* 170 * Walk the kernel page table pages, setting the active entries in the 171 * dump bitmap. 172 * 173 * NB: for a live dump, we may be racing with updates to the page 174 * tables, so care must be taken to read each entry only once. 175 */ 176 for (va = VM_MIN_KERNEL_ADDRESS; va < kva_end; va += L2_SIZE) { 177 pmapsize += PAGE_SIZE; 178 if (!pmap_get_tables(pmap_kernel(), va, &l0, &l1, &l2, &l3)) 179 continue; 180 181 l1e = atomic_load_64(l1); 182 l2e = atomic_load_64(l2); 183 if ((l1e & ATTR_DESCR_MASK) == L1_BLOCK) { 184 pa = PTE_TO_PHYS(l1e); 185 for (i = 0; i < Ln_ENTRIES * Ln_ENTRIES; 186 i++, pa += PAGE_SIZE) 187 if (vm_phys_is_dumpable(pa)) 188 vm_page_dump_add(state->dump_bitset, 189 pa); 190 pmapsize += (Ln_ENTRIES - 1) * PAGE_SIZE; 191 va += L1_SIZE - L2_SIZE; 192 } else if ((l2e & ATTR_DESCR_MASK) == L2_BLOCK) { 193 pa = PTE_TO_PHYS(l2e); 194 for (i = 0; i < Ln_ENTRIES; i++, pa += PAGE_SIZE) { 195 if (vm_phys_is_dumpable(pa)) 196 vm_page_dump_add(state->dump_bitset, 197 pa); 198 } 199 } else if ((l2e & ATTR_DESCR_MASK) == L2_TABLE) { 200 for (i = 0; i < Ln_ENTRIES; i++) { 201 l3e = atomic_load_64(&l3[i]); 202 if ((l3e & ATTR_DESCR_MASK) != L3_PAGE) 203 continue; 204 pa = PTE_TO_PHYS(l3e); 205 if (PHYS_IN_DMAP_RANGE(pa) && 206 vm_phys_is_dumpable(pa)) 207 vm_page_dump_add(state->dump_bitset, 208 pa); 209 } 210 } 211 } 212 213 /* Calculate dump size. */ 214 mbp = state->msgbufp; 215 dumpsize = pmapsize; 216 dumpsize += round_page(mbp->msg_size); 217 dumpsize += round_page(sizeof(dump_avail)); 218 dumpsize += round_page(BITSET_SIZE(vm_page_dump_pages)); 219 VM_PAGE_DUMP_FOREACH(state->dump_bitset, pa) { 220 if (PHYS_IN_DMAP_RANGE(pa) && vm_phys_is_dumpable(pa)) 221 dumpsize += PAGE_SIZE; 222 else 223 vm_page_dump_drop(state->dump_bitset, pa); 224 } 225 dumpsize += PAGE_SIZE; 226 227 dumpsys_pb_init(dumpsize); 228 229 /* Initialize mdhdr */ 230 bzero(&mdhdr, sizeof(mdhdr)); 231 strcpy(mdhdr.magic, MINIDUMP_MAGIC); 232 mdhdr.version = MINIDUMP_VERSION; 233 mdhdr.msgbufsize = mbp->msg_size; 234 mdhdr.bitmapsize = round_page(BITSET_SIZE(vm_page_dump_pages)); 235 mdhdr.pmapsize = pmapsize; 236 mdhdr.kernbase = VM_MIN_KERNEL_ADDRESS; 237 mdhdr.dmapphys = DMAP_MIN_PHYSADDR; 238 mdhdr.dmapbase = DMAP_MIN_ADDRESS; 239 mdhdr.dmapend = DMAP_MAX_ADDRESS; 240 mdhdr.dumpavailsize = round_page(sizeof(dump_avail)); 241 #if PAGE_SIZE == PAGE_SIZE_4K 242 mdhdr.flags = MINIDUMP_FLAG_PS_4K; 243 #elif PAGE_SIZE == PAGE_SIZE_16K 244 mdhdr.flags = MINIDUMP_FLAG_PS_16K; 245 #else 246 #error Unsupported page size 247 #endif 248 249 dump_init_header(di, &kdh, KERNELDUMPMAGIC, KERNELDUMP_AARCH64_VERSION, 250 dumpsize); 251 252 error = dump_start(di, &kdh); 253 if (error != 0) 254 goto fail; 255 256 printf("Dumping %llu out of %ju MB:", (long long)dumpsize >> 20, 257 ptoa((uintmax_t)physmem) / 1048576); 258 259 /* Dump my header */ 260 bzero(&tmpbuffer, sizeof(tmpbuffer)); 261 bcopy(&mdhdr, &tmpbuffer, sizeof(mdhdr)); 262 error = blk_write(di, (char *)&tmpbuffer, 0, PAGE_SIZE); 263 if (error) 264 goto fail; 265 266 /* Dump msgbuf up front */ 267 error = blk_write(di, mbp->msg_ptr, 0, round_page(mbp->msg_size)); 268 if (error) 269 goto fail; 270 271 /* Dump dump_avail */ 272 _Static_assert(sizeof(dump_avail) <= sizeof(tmpbuffer), 273 "Large dump_avail not handled"); 274 bzero(tmpbuffer, sizeof(tmpbuffer)); 275 memcpy(tmpbuffer, dump_avail, sizeof(dump_avail)); 276 error = blk_write(di, (char *)&tmpbuffer, 0, PAGE_SIZE); 277 if (error) 278 goto fail; 279 280 /* Dump bitmap */ 281 error = blk_write(di, (char *)state->dump_bitset, 0, 282 round_page(BITSET_SIZE(vm_page_dump_pages))); 283 if (error) 284 goto fail; 285 286 /* Dump kernel page directory pages */ 287 bzero(&tmpbuffer, sizeof(tmpbuffer)); 288 for (va = VM_MIN_KERNEL_ADDRESS; va < kva_end; va += L2_SIZE) { 289 if (!pmap_get_tables(pmap_kernel(), va, &l0, &l1, &l2, &l3)) { 290 /* We always write a page, even if it is zero */ 291 error = blk_write(di, (char *)&tmpbuffer, 0, PAGE_SIZE); 292 if (error) 293 goto fail; 294 /* flush, in case we reuse tmpbuffer in the same block*/ 295 error = blk_flush(di); 296 if (error) 297 goto fail; 298 continue; 299 } 300 301 l1e = atomic_load_64(l1); 302 l2e = atomic_load_64(l2); 303 if ((l1e & ATTR_DESCR_MASK) == L1_BLOCK) { 304 /* 305 * Handle a 1GB block mapping: write out 512 fake L2 306 * pages. 307 */ 308 pa = PTE_TO_PHYS(l1e) | (va & L1_OFFSET); 309 310 for (i = 0; i < Ln_ENTRIES; i++) { 311 for (j = 0; j < Ln_ENTRIES; j++) { 312 tmpbuffer[j] = (pa + i * L2_SIZE + 313 j * PAGE_SIZE) | ATTR_DEFAULT | 314 L3_PAGE; 315 } 316 error = blk_write(di, (char *)&tmpbuffer, 0, 317 PAGE_SIZE); 318 if (error) 319 goto fail; 320 } 321 /* flush, in case we reuse tmpbuffer in the same block*/ 322 error = blk_flush(di); 323 if (error) 324 goto fail; 325 bzero(&tmpbuffer, sizeof(tmpbuffer)); 326 va += L1_SIZE - L2_SIZE; 327 } else if ((l2e & ATTR_DESCR_MASK) == L2_BLOCK) { 328 pa = PTE_TO_PHYS(l2e) | (va & L2_OFFSET); 329 330 /* Generate fake l3 entries based upon the l1 entry */ 331 for (i = 0; i < Ln_ENTRIES; i++) { 332 tmpbuffer[i] = (pa + i * PAGE_SIZE) | 333 ATTR_DEFAULT | L3_PAGE; 334 } 335 error = blk_write(di, (char *)&tmpbuffer, 0, PAGE_SIZE); 336 if (error) 337 goto fail; 338 /* flush, in case we reuse fakepd in the same block */ 339 error = blk_flush(di); 340 if (error) 341 goto fail; 342 bzero(&tmpbuffer, sizeof(tmpbuffer)); 343 continue; 344 } else { 345 pa = PTE_TO_PHYS(l2e); 346 347 /* 348 * We always write a page, even if it is zero. If pa 349 * is malformed, write the zeroed tmpbuffer. 350 */ 351 if (PHYS_IN_DMAP_RANGE(pa) && vm_phys_is_dumpable(pa)) 352 error = blk_write(di, NULL, pa, PAGE_SIZE); 353 else 354 error = blk_write(di, (char *)&tmpbuffer, 0, 355 PAGE_SIZE); 356 if (error) 357 goto fail; 358 } 359 } 360 361 /* Dump memory chunks */ 362 VM_PAGE_DUMP_FOREACH(state->dump_bitset, pa) { 363 error = blk_write(di, 0, pa, PAGE_SIZE); 364 if (error) 365 goto fail; 366 } 367 368 error = blk_flush(di); 369 if (error) 370 goto fail; 371 372 error = dump_finish(di, &kdh); 373 if (error != 0) 374 goto fail; 375 376 printf("\nDump complete\n"); 377 return (0); 378 379 fail: 380 if (error < 0) 381 error = -error; 382 383 printf("\n"); 384 if (error == ENOSPC) { 385 printf("Dump map grown while dumping. "); 386 if (retry_count < 5) { 387 printf("Retrying...\n"); 388 goto retry; 389 } 390 printf("Dump failed.\n"); 391 } 392 else if (error == ECANCELED) 393 printf("Dump aborted\n"); 394 else if (error == E2BIG) { 395 printf("Dump failed. Partition too small (about %lluMB were " 396 "needed this time).\n", (long long)dumpsize >> 20); 397 } else 398 printf("** DUMP FAILED (ERROR %d) **\n", error); 399 return (error); 400 } 401