1 /*- 2 * Copyright (c) 2006 Peter Wemm 3 * Copyright (c) 2015 The FreeBSD Foundation 4 * All rights reserved. 5 * 6 * This software was developed by Andrew Turner under 7 * sponsorship from the FreeBSD Foundation. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD$"); 34 35 #include "opt_watchdog.h" 36 37 #include <sys/param.h> 38 #include <sys/systm.h> 39 #include <sys/conf.h> 40 #include <sys/cons.h> 41 #include <sys/kernel.h> 42 #include <sys/kerneldump.h> 43 #include <sys/msgbuf.h> 44 #include <sys/watchdog.h> 45 #include <sys/vmmeter.h> 46 47 #include <vm/vm.h> 48 #include <vm/vm_param.h> 49 #include <vm/vm_page.h> 50 #include <vm/vm_phys.h> 51 #include <vm/vm_dumpset.h> 52 #include <vm/pmap.h> 53 54 #include <machine/atomic.h> 55 #include <machine/md_var.h> 56 #include <machine/pte.h> 57 #include <machine/minidump.h> 58 59 CTASSERT(sizeof(struct kerneldumpheader) == 512); 60 61 static struct kerneldumpheader kdh; 62 63 /* Handle chunked writes. */ 64 static size_t fragsz; 65 static void *dump_va; 66 static size_t dumpsize; 67 68 static uint64_t tmpbuffer[Ln_ENTRIES]; 69 70 static int 71 blk_flush(struct dumperinfo *di) 72 { 73 int error; 74 75 if (fragsz == 0) 76 return (0); 77 78 error = dump_append(di, dump_va, fragsz); 79 fragsz = 0; 80 return (error); 81 } 82 83 static int 84 blk_write(struct dumperinfo *di, char *ptr, vm_paddr_t pa, size_t sz) 85 { 86 size_t len; 87 int error, c; 88 u_int maxdumpsz; 89 90 maxdumpsz = min(di->maxiosize, MAXDUMPPGS * PAGE_SIZE); 91 if (maxdumpsz == 0) /* seatbelt */ 92 maxdumpsz = PAGE_SIZE; 93 error = 0; 94 if ((sz % PAGE_SIZE) != 0) { 95 printf("size not page aligned\n"); 96 return (EINVAL); 97 } 98 if (ptr != NULL && pa != 0) { 99 printf("cant have both va and pa!\n"); 100 return (EINVAL); 101 } 102 if ((((uintptr_t)pa) % PAGE_SIZE) != 0) { 103 printf("address not page aligned %p\n", ptr); 104 return (EINVAL); 105 } 106 if (ptr != NULL) { 107 /* 108 * If we're doing a virtual dump, flush any 109 * pre-existing pa pages. 110 */ 111 error = blk_flush(di); 112 if (error) 113 return (error); 114 } 115 while (sz) { 116 len = maxdumpsz - fragsz; 117 if (len > sz) 118 len = sz; 119 120 dumpsys_pb_progress(len); 121 wdog_kern_pat(WD_LASTVAL); 122 123 if (ptr) { 124 error = dump_append(di, ptr, len); 125 if (error) 126 return (error); 127 ptr += len; 128 sz -= len; 129 } else { 130 dump_va = (void *)PHYS_TO_DMAP(pa); 131 fragsz += len; 132 pa += len; 133 sz -= len; 134 error = blk_flush(di); 135 if (error) 136 return (error); 137 } 138 139 /* Check for user abort. */ 140 c = cncheckc(); 141 if (c == 0x03) 142 return (ECANCELED); 143 if (c != -1) 144 printf(" (CTRL-C to abort) "); 145 } 146 147 return (0); 148 } 149 150 int 151 cpu_minidumpsys(struct dumperinfo *di, const struct minidumpstate *state) 152 { 153 struct minidumphdr mdhdr; 154 struct msgbuf *mbp; 155 pd_entry_t *l0, *l1, l1e, *l2, l2e; 156 pt_entry_t *l3, l3e; 157 vm_offset_t va, kva_end; 158 vm_paddr_t pa; 159 uint32_t pmapsize; 160 int error, i, j, retry_count; 161 162 retry_count = 0; 163 retry: 164 retry_count++; 165 error = 0; 166 pmapsize = 0; 167 168 /* Snapshot the KVA upper bound in case it grows. */ 169 kva_end = kernel_vm_end; 170 171 /* 172 * Walk the kernel page table pages, setting the active entries in the 173 * dump bitmap. 174 * 175 * NB: for a live dump, we may be racing with updates to the page 176 * tables, so care must be taken to read each entry only once. 177 */ 178 for (va = VM_MIN_KERNEL_ADDRESS; va < kva_end; va += L2_SIZE) { 179 pmapsize += PAGE_SIZE; 180 if (!pmap_get_tables(pmap_kernel(), va, &l0, &l1, &l2, &l3)) 181 continue; 182 183 l1e = atomic_load_64(l1); 184 l2e = atomic_load_64(l2); 185 if ((l1e & ATTR_DESCR_MASK) == L1_BLOCK) { 186 pa = PTE_TO_PHYS(l1e); 187 for (i = 0; i < Ln_ENTRIES * Ln_ENTRIES; 188 i++, pa += PAGE_SIZE) 189 if (vm_phys_is_dumpable(pa)) 190 vm_page_dump_add(state->dump_bitset, 191 pa); 192 pmapsize += (Ln_ENTRIES - 1) * PAGE_SIZE; 193 va += L1_SIZE - L2_SIZE; 194 } else if ((l2e & ATTR_DESCR_MASK) == L2_BLOCK) { 195 pa = PTE_TO_PHYS(l2e); 196 for (i = 0; i < Ln_ENTRIES; i++, pa += PAGE_SIZE) { 197 if (vm_phys_is_dumpable(pa)) 198 vm_page_dump_add(state->dump_bitset, 199 pa); 200 } 201 } else if ((l2e & ATTR_DESCR_MASK) == L2_TABLE) { 202 for (i = 0; i < Ln_ENTRIES; i++) { 203 l3e = atomic_load_64(&l3[i]); 204 if ((l3e & ATTR_DESCR_MASK) != L3_PAGE) 205 continue; 206 pa = PTE_TO_PHYS(l3e); 207 if (PHYS_IN_DMAP(pa) && vm_phys_is_dumpable(pa)) 208 vm_page_dump_add(state->dump_bitset, 209 pa); 210 } 211 } 212 } 213 214 /* Calculate dump size. */ 215 mbp = state->msgbufp; 216 dumpsize = pmapsize; 217 dumpsize += round_page(mbp->msg_size); 218 dumpsize += round_page(sizeof(dump_avail)); 219 dumpsize += round_page(BITSET_SIZE(vm_page_dump_pages)); 220 VM_PAGE_DUMP_FOREACH(state->dump_bitset, pa) { 221 if (PHYS_IN_DMAP(pa) && vm_phys_is_dumpable(pa)) 222 dumpsize += PAGE_SIZE; 223 else 224 vm_page_dump_drop(state->dump_bitset, pa); 225 } 226 dumpsize += PAGE_SIZE; 227 228 dumpsys_pb_init(dumpsize); 229 230 /* Initialize mdhdr */ 231 bzero(&mdhdr, sizeof(mdhdr)); 232 strcpy(mdhdr.magic, MINIDUMP_MAGIC); 233 mdhdr.version = MINIDUMP_VERSION; 234 mdhdr.msgbufsize = mbp->msg_size; 235 mdhdr.bitmapsize = round_page(BITSET_SIZE(vm_page_dump_pages)); 236 mdhdr.pmapsize = pmapsize; 237 mdhdr.kernbase = VM_MIN_KERNEL_ADDRESS; 238 mdhdr.dmapphys = DMAP_MIN_PHYSADDR; 239 mdhdr.dmapbase = DMAP_MIN_ADDRESS; 240 mdhdr.dmapend = DMAP_MAX_ADDRESS; 241 mdhdr.dumpavailsize = round_page(sizeof(dump_avail)); 242 #if PAGE_SIZE == PAGE_SIZE_4K 243 mdhdr.flags = MINIDUMP_FLAG_PS_4K; 244 #elif PAGE_SIZE == PAGE_SIZE_16K 245 mdhdr.flags = MINIDUMP_FLAG_PS_16K; 246 #else 247 #error Unsupported page size 248 #endif 249 250 dump_init_header(di, &kdh, KERNELDUMPMAGIC, KERNELDUMP_AARCH64_VERSION, 251 dumpsize); 252 253 error = dump_start(di, &kdh); 254 if (error != 0) 255 goto fail; 256 257 printf("Dumping %llu out of %ju MB:", (long long)dumpsize >> 20, 258 ptoa((uintmax_t)physmem) / 1048576); 259 260 /* Dump my header */ 261 bzero(&tmpbuffer, sizeof(tmpbuffer)); 262 bcopy(&mdhdr, &tmpbuffer, sizeof(mdhdr)); 263 error = blk_write(di, (char *)&tmpbuffer, 0, PAGE_SIZE); 264 if (error) 265 goto fail; 266 267 /* Dump msgbuf up front */ 268 error = blk_write(di, mbp->msg_ptr, 0, round_page(mbp->msg_size)); 269 if (error) 270 goto fail; 271 272 /* Dump dump_avail */ 273 _Static_assert(sizeof(dump_avail) <= sizeof(tmpbuffer), 274 "Large dump_avail not handled"); 275 bzero(tmpbuffer, sizeof(tmpbuffer)); 276 memcpy(tmpbuffer, dump_avail, sizeof(dump_avail)); 277 error = blk_write(di, (char *)&tmpbuffer, 0, PAGE_SIZE); 278 if (error) 279 goto fail; 280 281 /* Dump bitmap */ 282 error = blk_write(di, (char *)state->dump_bitset, 0, 283 round_page(BITSET_SIZE(vm_page_dump_pages))); 284 if (error) 285 goto fail; 286 287 /* Dump kernel page directory pages */ 288 bzero(&tmpbuffer, sizeof(tmpbuffer)); 289 for (va = VM_MIN_KERNEL_ADDRESS; va < kva_end; va += L2_SIZE) { 290 if (!pmap_get_tables(pmap_kernel(), va, &l0, &l1, &l2, &l3)) { 291 /* We always write a page, even if it is zero */ 292 error = blk_write(di, (char *)&tmpbuffer, 0, PAGE_SIZE); 293 if (error) 294 goto fail; 295 /* flush, in case we reuse tmpbuffer in the same block*/ 296 error = blk_flush(di); 297 if (error) 298 goto fail; 299 continue; 300 } 301 302 l1e = atomic_load_64(l1); 303 l2e = atomic_load_64(l2); 304 if ((l1e & ATTR_DESCR_MASK) == L1_BLOCK) { 305 /* 306 * Handle a 1GB block mapping: write out 512 fake L2 307 * pages. 308 */ 309 pa = PTE_TO_PHYS(l1e) | (va & L1_OFFSET); 310 311 for (i = 0; i < Ln_ENTRIES; i++) { 312 for (j = 0; j < Ln_ENTRIES; j++) { 313 tmpbuffer[j] = (pa + i * L2_SIZE + 314 j * PAGE_SIZE) | ATTR_DEFAULT | 315 L3_PAGE; 316 } 317 error = blk_write(di, (char *)&tmpbuffer, 0, 318 PAGE_SIZE); 319 if (error) 320 goto fail; 321 } 322 /* flush, in case we reuse tmpbuffer in the same block*/ 323 error = blk_flush(di); 324 if (error) 325 goto fail; 326 bzero(&tmpbuffer, sizeof(tmpbuffer)); 327 va += L1_SIZE - L2_SIZE; 328 } else if ((l2e & ATTR_DESCR_MASK) == L2_BLOCK) { 329 pa = PTE_TO_PHYS(l2e) | (va & L2_OFFSET); 330 331 /* Generate fake l3 entries based upon the l1 entry */ 332 for (i = 0; i < Ln_ENTRIES; i++) { 333 tmpbuffer[i] = (pa + i * PAGE_SIZE) | 334 ATTR_DEFAULT | L3_PAGE; 335 } 336 error = blk_write(di, (char *)&tmpbuffer, 0, PAGE_SIZE); 337 if (error) 338 goto fail; 339 /* flush, in case we reuse fakepd in the same block */ 340 error = blk_flush(di); 341 if (error) 342 goto fail; 343 bzero(&tmpbuffer, sizeof(tmpbuffer)); 344 continue; 345 } else { 346 pa = PTE_TO_PHYS(l2e); 347 348 /* 349 * We always write a page, even if it is zero. If pa 350 * is malformed, write the zeroed tmpbuffer. 351 */ 352 if (PHYS_IN_DMAP(pa) && vm_phys_is_dumpable(pa)) 353 error = blk_write(di, NULL, pa, PAGE_SIZE); 354 else 355 error = blk_write(di, (char *)&tmpbuffer, 0, 356 PAGE_SIZE); 357 if (error) 358 goto fail; 359 } 360 } 361 362 /* Dump memory chunks */ 363 VM_PAGE_DUMP_FOREACH(state->dump_bitset, pa) { 364 error = blk_write(di, 0, pa, PAGE_SIZE); 365 if (error) 366 goto fail; 367 } 368 369 error = blk_flush(di); 370 if (error) 371 goto fail; 372 373 error = dump_finish(di, &kdh); 374 if (error != 0) 375 goto fail; 376 377 printf("\nDump complete\n"); 378 return (0); 379 380 fail: 381 if (error < 0) 382 error = -error; 383 384 printf("\n"); 385 if (error == ENOSPC) { 386 printf("Dump map grown while dumping. "); 387 if (retry_count < 5) { 388 printf("Retrying...\n"); 389 goto retry; 390 } 391 printf("Dump failed.\n"); 392 } 393 else if (error == ECANCELED) 394 printf("Dump aborted\n"); 395 else if (error == E2BIG) { 396 printf("Dump failed. Partition too small (about %lluMB were " 397 "needed this time).\n", (long long)dumpsize >> 20); 398 } else 399 printf("** DUMP FAILED (ERROR %d) **\n", error); 400 return (error); 401 } 402