1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2014 Ian Lepore <ian@freebsd.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #ifdef _KERNEL 33 #include "opt_acpi.h" 34 #include "opt_ddb.h" 35 #endif 36 37 /* 38 * Routines for describing and initializing anything related to physical memory. 39 */ 40 41 #include <sys/param.h> 42 #include <sys/systm.h> 43 #include <sys/bus.h> 44 #include <sys/kernel.h> 45 #include <sys/module.h> 46 #include <sys/physmem.h> 47 48 #ifdef _KERNEL 49 #include <vm/vm.h> 50 #include <vm/vm_param.h> 51 #include <vm/vm_page.h> 52 #include <vm/vm_phys.h> 53 #include <vm/vm_dumpset.h> 54 55 #include <machine/md_var.h> 56 #include <machine/resource.h> 57 #else 58 #include <stdarg.h> 59 #include <stdio.h> 60 #include <string.h> 61 #endif 62 63 /* 64 * These structures are used internally to keep track of regions of physical 65 * ram, and regions within the physical ram that need to be excluded. An 66 * exclusion region can be excluded from crash dumps, from the vm pool of pages 67 * that can be allocated, or both, depending on the exclusion flags associated 68 * with the region. 69 */ 70 #ifdef DEV_ACPI 71 #define MAX_HWCNT 32 /* ACPI needs more regions */ 72 #define MAX_EXCNT 32 73 #else 74 #define MAX_HWCNT 16 75 #define MAX_EXCNT 16 76 #endif 77 78 #if defined(__arm__) 79 #define MAX_PHYS_ADDR 0xFFFFFFFFull 80 #elif defined(__aarch64__) || defined(__amd64__) || defined(__riscv) 81 #define MAX_PHYS_ADDR 0xFFFFFFFFFFFFFFFFull 82 #endif 83 84 struct region { 85 vm_paddr_t addr; 86 vm_size_t size; 87 uint32_t flags; 88 }; 89 90 static struct region hwregions[MAX_HWCNT]; 91 static struct region exregions[MAX_EXCNT]; 92 93 static size_t hwcnt; 94 static size_t excnt; 95 96 /* 97 * realmem is the total number of hardware pages, excluded or not. 98 * Maxmem is one greater than the last physical page number. 99 */ 100 long realmem; 101 long Maxmem; 102 103 #ifndef _KERNEL 104 static void 105 panic(const char *fmt, ...) 106 { 107 va_list va; 108 109 va_start(va, fmt); 110 vfprintf(stderr, fmt, va); 111 fprintf(stderr, "\n"); 112 va_end(va); 113 __builtin_trap(); 114 } 115 #endif 116 117 /* 118 * Print the contents of the physical and excluded region tables using the 119 * provided printf-like output function (which will be either printf or 120 * db_printf). 121 */ 122 static void 123 physmem_dump_tables(int (*prfunc)(const char *, ...)) 124 { 125 size_t i; 126 int flags; 127 uintmax_t addr, size; 128 const unsigned int mbyte = 1024 * 1024; 129 130 prfunc("Physical memory chunk(s):\n"); 131 for (i = 0; i < hwcnt; ++i) { 132 addr = hwregions[i].addr; 133 size = hwregions[i].size; 134 prfunc(" 0x%08jx - 0x%08jx, %5ju MB (%7ju pages)\n", addr, 135 addr + size - 1, size / mbyte, size / PAGE_SIZE); 136 } 137 138 prfunc("Excluded memory regions:\n"); 139 for (i = 0; i < excnt; ++i) { 140 addr = exregions[i].addr; 141 size = exregions[i].size; 142 flags = exregions[i].flags; 143 prfunc(" 0x%08jx - 0x%08jx, %5ju MB (%7ju pages) %s %s\n", 144 addr, addr + size - 1, size / mbyte, size / PAGE_SIZE, 145 (flags & EXFLAG_NOALLOC) ? "NoAlloc" : "", 146 (flags & EXFLAG_NODUMP) ? "NoDump" : ""); 147 } 148 149 #ifdef DEBUG 150 prfunc("Avail lists:\n"); 151 for (i = 0; phys_avail[i] != 0; ++i) { 152 prfunc(" phys_avail[%d] 0x%08x\n", i, phys_avail[i]); 153 } 154 for (i = 0; dump_avail[i] != 0; ++i) { 155 prfunc(" dump_avail[%d] 0x%08x\n", i, dump_avail[i]); 156 } 157 #endif 158 } 159 160 /* 161 * Print the contents of the static mapping table. Used for bootverbose. 162 */ 163 void 164 physmem_print_tables(void) 165 { 166 167 physmem_dump_tables(printf); 168 } 169 170 /* 171 * Walk the list of hardware regions, processing it against the list of 172 * exclusions that contain the given exflags, and generating an "avail list". 173 * 174 * If maxphyssz is not zero it sets upper limit, in bytes, for the total 175 * "avail list" size. Walk stops once the limit is reached and the last region 176 * is cut short if necessary. 177 * 178 * Updates the value at *pavail with the sum of all pages in all hw regions. 179 * 180 * Returns the number of pages of non-excluded memory added to the avail list. 181 */ 182 static size_t 183 regions_to_avail(vm_paddr_t *avail, uint32_t exflags, size_t maxavail, 184 uint64_t maxphyssz, long *pavail, long *prealmem) 185 { 186 size_t acnt, exi, hwi; 187 uint64_t adj, end, start, xend, xstart; 188 long availmem, totalmem; 189 const struct region *exp, *hwp; 190 uint64_t availsz; 191 192 totalmem = 0; 193 availmem = 0; 194 availsz = 0; 195 acnt = 0; 196 for (hwi = 0, hwp = hwregions; hwi < hwcnt; ++hwi, ++hwp) { 197 adj = round_page(hwp->addr) - hwp->addr; 198 start = round_page(hwp->addr); 199 end = trunc_page(hwp->size + adj) + start; 200 totalmem += atop((vm_offset_t)(end - start)); 201 for (exi = 0, exp = exregions; exi < excnt; ++exi, ++exp) { 202 /* 203 * If the excluded region does not match given flags, 204 * continue checking with the next excluded region. 205 */ 206 if ((exp->flags & exflags) == 0) 207 continue; 208 xstart = exp->addr; 209 xend = exp->size + xstart; 210 /* 211 * If the excluded region ends before this hw region, 212 * continue checking with the next excluded region. 213 */ 214 if (xend <= start) 215 continue; 216 /* 217 * If the excluded region begins after this hw region 218 * we're done because both lists are sorted. 219 */ 220 if (xstart >= end) 221 break; 222 /* 223 * If the excluded region completely covers this hw 224 * region, shrink this hw region to zero size. 225 */ 226 if ((start >= xstart) && (end <= xend)) { 227 start = xend; 228 end = xend; 229 break; 230 } 231 /* 232 * If the excluded region falls wholly within this hw 233 * region without abutting or overlapping the beginning 234 * or end, create an available entry from the leading 235 * fragment, then adjust the start of this hw region to 236 * the end of the excluded region, and continue checking 237 * the next excluded region because another exclusion 238 * could affect the remainder of this hw region. 239 */ 240 if ((xstart > start) && (xend < end)) { 241 242 if ((maxphyssz != 0) && 243 (availsz + xstart - start > maxphyssz)) { 244 xstart = maxphyssz + start - availsz; 245 } 246 if (xstart <= start) 247 continue; 248 if (acnt > 0 && 249 avail[acnt - 1] == (vm_paddr_t)start) { 250 avail[acnt - 1] = (vm_paddr_t)xstart; 251 } else { 252 avail[acnt++] = (vm_paddr_t)start; 253 avail[acnt++] = (vm_paddr_t)xstart; 254 } 255 availsz += (xstart - start); 256 availmem += atop((vm_offset_t)(xstart - start)); 257 start = xend; 258 continue; 259 } 260 /* 261 * We know the excluded region overlaps either the start 262 * or end of this hardware region (but not both), trim 263 * the excluded portion off the appropriate end. 264 */ 265 if (xstart <= start) 266 start = xend; 267 else 268 end = xstart; 269 } 270 /* 271 * If the trimming actions above left a non-zero size, create an 272 * available entry for it. 273 */ 274 if (end > start) { 275 if ((maxphyssz != 0) && 276 (availsz + end - start > maxphyssz)) { 277 end = maxphyssz + start - availsz; 278 } 279 if (end <= start) 280 break; 281 282 if (acnt > 0 && avail[acnt - 1] == (vm_paddr_t)start) { 283 avail[acnt - 1] = (vm_paddr_t)end; 284 } else { 285 avail[acnt++] = (vm_paddr_t)start; 286 avail[acnt++] = (vm_paddr_t)end; 287 } 288 availsz += end - start; 289 availmem += atop((vm_offset_t)(end - start)); 290 } 291 if (acnt >= maxavail) 292 panic("Not enough space in the dump/phys_avail arrays"); 293 } 294 295 if (pavail != NULL) 296 *pavail = availmem; 297 if (prealmem != NULL) 298 *prealmem = totalmem; 299 return (acnt); 300 } 301 302 /* 303 * Check if the region at idx can be merged with the region above it. 304 */ 305 static size_t 306 merge_upper_regions(struct region *regions, size_t rcnt, size_t idx) 307 { 308 struct region *lower, *upper; 309 vm_paddr_t lend, uend; 310 size_t i, mergecnt, movecnt; 311 312 lower = ®ions[idx]; 313 lend = lower->addr + lower->size; 314 315 /* 316 * Continue merging in upper entries as long as we have entries to 317 * merge; the new block could have spanned more than one, although one 318 * is likely the common case. 319 */ 320 for (i = idx + 1; i < rcnt; i++) { 321 upper = ®ions[i]; 322 if (lend < upper->addr || lower->flags != upper->flags) 323 break; 324 325 uend = upper->addr + upper->size; 326 if (uend > lend) { 327 lower->size += uend - lend; 328 lend = lower->addr + lower->size; 329 } 330 331 if (uend >= lend) { 332 /* 333 * If we didn't move past the end of the upper region, 334 * then we don't need to bother checking for another 335 * merge because it would have been done already. Just 336 * increment i once more to maintain the invariant that 337 * i is one past the last entry merged. 338 */ 339 i++; 340 break; 341 } 342 } 343 344 /* 345 * We merged in the entries from [idx + 1, i); physically move the tail 346 * end at [i, rcnt) if we need to. 347 */ 348 mergecnt = i - (idx + 1); 349 if (mergecnt > 0) { 350 movecnt = rcnt - i; 351 if (movecnt == 0) { 352 /* Merged all the way to the end, just decrease rcnt. */ 353 rcnt = idx + 1; 354 } else { 355 memmove(®ions[idx + 1], ®ions[idx + mergecnt + 1], 356 movecnt * sizeof(*regions)); 357 rcnt -= mergecnt; 358 } 359 } 360 return (rcnt); 361 } 362 363 /* 364 * Insertion-sort a new entry into a regions list; sorted by start address. 365 */ 366 static size_t 367 insert_region(struct region *regions, size_t rcnt, vm_paddr_t addr, 368 vm_size_t size, uint32_t flags) 369 { 370 size_t i; 371 vm_paddr_t nend, rend; 372 struct region *ep, *rp; 373 374 nend = addr + size; 375 ep = regions + rcnt; 376 for (i = 0, rp = regions; i < rcnt; ++i, ++rp) { 377 if (flags == rp->flags) { 378 rend = rp->addr + rp->size; 379 if (addr <= rp->addr && nend >= rp->addr) { 380 /* 381 * New mapping overlaps at the beginning, shift 382 * for any difference in the beginning then 383 * shift if the new mapping extends past. 384 */ 385 rp->size += rp->addr - addr; 386 rp->addr = addr; 387 if (nend > rend) { 388 rp->size += nend - rend; 389 rcnt = merge_upper_regions(regions, 390 rcnt, i); 391 } 392 return (rcnt); 393 } else if (addr <= rend && nend > rp->addr) { 394 /* 395 * New mapping is either entirely contained 396 * within or it's overlapping at the end. 397 */ 398 if (nend > rend) { 399 rp->size += nend - rend; 400 rcnt = merge_upper_regions(regions, 401 rcnt, i); 402 } 403 return (rcnt); 404 } 405 } 406 if (addr < rp->addr) { 407 bcopy(rp, rp + 1, (ep - rp) * sizeof(*rp)); 408 break; 409 } 410 } 411 rp->addr = addr; 412 rp->size = size; 413 rp->flags = flags; 414 rcnt++; 415 416 return (rcnt); 417 } 418 419 /* 420 * Add a hardware memory region. 421 */ 422 void 423 physmem_hardware_region(uint64_t pa, uint64_t sz) 424 { 425 /* 426 * Filter out the page at PA 0x00000000. The VM can't handle it, as 427 * pmap_extract() == 0 means failure. 428 */ 429 if (pa == 0) { 430 if (sz <= PAGE_SIZE) 431 return; 432 pa = PAGE_SIZE; 433 sz -= PAGE_SIZE; 434 } else if (pa > MAX_PHYS_ADDR) { 435 /* This range is past usable memory, ignore it */ 436 return; 437 } 438 439 /* 440 * Also filter out the page at the end of the physical address space -- 441 * if addr is non-zero and addr+size is zero we wrapped to the next byte 442 * beyond what vm_paddr_t can express. That leads to a NULL pointer 443 * deref early in startup; work around it by leaving the last page out. 444 * 445 * XXX This just in: subtract out a whole megabyte, not just 1 page. 446 * Reducing the size by anything less than 1MB results in the NULL 447 * pointer deref in _vm_map_lock_read(). Better to give up a megabyte 448 * than leave some folks with an unusable system while we investigate. 449 */ 450 if ((pa + sz) > (MAX_PHYS_ADDR - 1024 * 1024)) { 451 sz = MAX_PHYS_ADDR - pa + 1; 452 if (sz <= 1024 * 1024) 453 return; 454 sz -= 1024 * 1024; 455 } 456 457 if (sz > 0 && hwcnt < nitems(hwregions)) 458 hwcnt = insert_region(hwregions, hwcnt, pa, sz, 0); 459 } 460 461 /* 462 * Add an exclusion region. 463 */ 464 void 465 physmem_exclude_region(vm_paddr_t pa, vm_size_t sz, uint32_t exflags) 466 { 467 vm_offset_t adj; 468 469 /* 470 * Truncate the starting address down to a page boundary, and round the 471 * ending page up to a page boundary. 472 */ 473 adj = pa - trunc_page(pa); 474 pa = trunc_page(pa); 475 sz = round_page(sz + adj); 476 477 if (excnt >= nitems(exregions)) 478 panic("failed to exclude region %#jx-%#jx", (uintmax_t)pa, 479 (uintmax_t)(pa + sz)); 480 excnt = insert_region(exregions, excnt, pa, sz, exflags); 481 } 482 483 size_t 484 physmem_avail(vm_paddr_t *avail, size_t maxavail) 485 { 486 487 return (regions_to_avail(avail, EXFLAG_NOALLOC, maxavail, 0, NULL, NULL)); 488 } 489 490 bool 491 physmem_excluded(vm_paddr_t pa, vm_size_t sz) 492 { 493 const struct region *exp; 494 size_t exi; 495 496 for (exi = 0, exp = exregions; exi < excnt; ++exi, ++exp) { 497 if (pa < exp->addr || pa + sz > exp->addr + exp->size) 498 continue; 499 return (true); 500 } 501 return (false); 502 } 503 504 #ifdef _KERNEL 505 /* 506 * Process all the regions added earlier into the global avail lists. 507 * 508 * Updates the kernel global 'physmem' with the number of physical pages 509 * available for use (all pages not in any exclusion region). 510 * 511 * Updates the kernel global 'Maxmem' with the page number one greater then the 512 * last page of physical memory in the system. 513 */ 514 void 515 physmem_init_kernel_globals(void) 516 { 517 size_t nextidx; 518 u_long hwphyssz; 519 520 hwphyssz = 0; 521 TUNABLE_ULONG_FETCH("hw.physmem", &hwphyssz); 522 523 regions_to_avail(dump_avail, EXFLAG_NODUMP, PHYS_AVAIL_ENTRIES, 524 hwphyssz, NULL, NULL); 525 nextidx = regions_to_avail(phys_avail, EXFLAG_NOALLOC, 526 PHYS_AVAIL_ENTRIES, hwphyssz, &physmem, &realmem); 527 if (nextidx == 0) 528 panic("No memory entries in phys_avail"); 529 Maxmem = atop(phys_avail[nextidx - 1]); 530 } 531 532 #ifdef DDB 533 #include <ddb/ddb.h> 534 535 DB_SHOW_COMMAND_FLAGS(physmem, db_show_physmem, DB_CMD_MEMSAFE) 536 { 537 538 physmem_dump_tables(db_printf); 539 } 540 541 #endif /* DDB */ 542 543 /* 544 * ram pseudo driver - this reserves I/O space resources corresponding to physical 545 * memory regions. 546 */ 547 548 static void 549 ram_identify(driver_t *driver, device_t parent) 550 { 551 552 if (resource_disabled("ram", 0)) 553 return; 554 if (BUS_ADD_CHILD(parent, 0, "ram", 0) == NULL) 555 panic("ram_identify"); 556 } 557 558 static int 559 ram_probe(device_t dev) 560 { 561 562 device_quiet(dev); 563 device_set_desc(dev, "System RAM"); 564 return (BUS_PROBE_SPECIFIC); 565 } 566 567 static int 568 ram_attach(device_t dev) 569 { 570 vm_paddr_t avail_list[PHYS_AVAIL_COUNT]; 571 rman_res_t start, end; 572 struct region *hwp; 573 int rid, i; 574 575 rid = 0; 576 577 /* Get the avail list. */ 578 bzero(avail_list, sizeof(avail_list)); 579 regions_to_avail(avail_list, EXFLAG_NOALLOC | EXFLAG_NODUMP, 580 PHYS_AVAIL_COUNT, 0, NULL, NULL); 581 582 /* Reserve all memory regions. */ 583 for (i = 0; avail_list[i + 1] != 0; i += 2) { 584 start = avail_list[i]; 585 end = avail_list[i + 1]; 586 587 if (bootverbose) 588 device_printf(dev, 589 "reserving memory region: %jx-%jx\n", 590 (uintmax_t)start, (uintmax_t)end); 591 592 if (bus_alloc_resource(dev, SYS_RES_MEMORY, &rid, start, end, 593 end - start, 0) == NULL) 594 panic("ram_attach: resource %d failed to attach", rid); 595 rid++; 596 } 597 598 /* Now, reserve the excluded memory regions. */ 599 for (i = 0, hwp = exregions; i < excnt; i++, hwp++) { 600 start = hwp->addr; 601 end = hwp->addr + hwp->size; 602 603 if (bootverbose) 604 device_printf(dev, 605 "reserving excluded region: %jx-%jx\n", 606 (uintmax_t)start, (uintmax_t)(end - 1)); 607 608 /* 609 * Best-effort attempt to reserve the range. This may fail, as 610 * sometimes the excluded ranges provided by the device tree 611 * will cover or overlap some I/O range. 612 */ 613 if (bus_alloc_resource(dev, SYS_RES_MEMORY, &rid, start, end, 614 end - start, 0) == NULL) { 615 if (bootverbose) 616 device_printf(dev, "failed to reserve region\n"); 617 continue; 618 } 619 rid++; 620 } 621 622 return (0); 623 } 624 625 static device_method_t ram_methods[] = { 626 /* Device interface */ 627 DEVMETHOD(device_identify, ram_identify), 628 DEVMETHOD(device_probe, ram_probe), 629 DEVMETHOD(device_attach, ram_attach), 630 631 DEVMETHOD_END 632 }; 633 634 DEFINE_CLASS_0(ram, ram_driver, ram_methods, /* no softc */ 1); 635 DRIVER_MODULE(ram, nexus, ram_driver, 0, 0); 636 #endif /* _KERNEL */ 637