1 /*- 2 * Copyright (c) 1999 Michael Smith <msmith@freebsd.org> 3 * Copyright (c) 2017 The FreeBSD Foundation 4 * All rights reserved. 5 * 6 * Portions of this software were developed by Konstantin Belousov 7 * under sponsorship from the FreeBSD Foundation. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include <sys/cdefs.h> 32 #include <sys/param.h> 33 #include <sys/kernel.h> 34 #include <sys/systm.h> 35 #include <sys/malloc.h> 36 #include <sys/memrange.h> 37 #include <sys/smp.h> 38 #include <sys/sysctl.h> 39 40 #include <vm/vm.h> 41 #include <vm/vm_param.h> 42 #include <vm/pmap.h> 43 44 #include <machine/cputypes.h> 45 #include <machine/md_var.h> 46 #include <machine/specialreg.h> 47 48 /* 49 * Pentium Pro+ memory range operations 50 * 51 * This code will probably be impenetrable without reference to the 52 * Intel Pentium Pro documentation or x86-64 programmers manual vol 2. 53 */ 54 55 static char *mem_owner_bios = "BIOS"; 56 57 #define MR686_FIXMTRR (1<<0) 58 59 #define mrwithin(mr, a) \ 60 (((a) >= (mr)->mr_base) && ((a) < ((mr)->mr_base + (mr)->mr_len))) 61 #define mroverlap(mra, mrb) \ 62 (mrwithin(mra, mrb->mr_base) || mrwithin(mrb, mra->mr_base)) 63 64 #define mrvalid(base, len) \ 65 ((!(base & ((1 << 12) - 1))) && /* base is multiple of 4k */ \ 66 ((len) >= (1 << 12)) && /* length is >= 4k */ \ 67 powerof2((len)) && /* ... and power of two */ \ 68 !((base) & ((len) - 1))) /* range is not discontiuous */ 69 70 #define mrcopyflags(curr, new) \ 71 (((curr) & ~MDF_ATTRMASK) | ((new) & MDF_ATTRMASK)) 72 73 static int mtrrs_disabled; 74 SYSCTL_INT(_machdep, OID_AUTO, disable_mtrrs, CTLFLAG_RDTUN, 75 &mtrrs_disabled, 0, 76 "Disable MTRRs."); 77 78 static void x86_mrinit(struct mem_range_softc *sc); 79 static int x86_mrset(struct mem_range_softc *sc, 80 struct mem_range_desc *mrd, int *arg); 81 static void x86_mrAPinit(struct mem_range_softc *sc); 82 static void x86_mrreinit(struct mem_range_softc *sc); 83 84 static struct mem_range_ops x86_mrops = { 85 x86_mrinit, 86 x86_mrset, 87 x86_mrAPinit, 88 x86_mrreinit 89 }; 90 91 /* XXX for AP startup hook */ 92 static u_int64_t mtrrcap, mtrrdef; 93 94 /* The bitmask for the PhysBase and PhysMask fields of the variable MTRRs. */ 95 static u_int64_t mtrr_physmask; 96 97 static struct mem_range_desc *mem_range_match(struct mem_range_softc *sc, 98 struct mem_range_desc *mrd); 99 static void x86_mrfetch(struct mem_range_softc *sc); 100 static int x86_mtrrtype(int flags); 101 static int x86_mrt2mtrr(int flags, int oldval); 102 static int x86_mtrrconflict(int flag1, int flag2); 103 static void x86_mrstore(struct mem_range_softc *sc); 104 static void x86_mrstoreone(void *arg); 105 static struct mem_range_desc *x86_mtrrfixsearch(struct mem_range_softc *sc, 106 u_int64_t addr); 107 static int x86_mrsetlow(struct mem_range_softc *sc, 108 struct mem_range_desc *mrd, int *arg); 109 static int x86_mrsetvariable(struct mem_range_softc *sc, 110 struct mem_range_desc *mrd, int *arg); 111 112 /* ia32 MTRR type to memory range type conversion */ 113 static int x86_mtrrtomrt[] = { 114 MDF_UNCACHEABLE, 115 MDF_WRITECOMBINE, 116 MDF_UNKNOWN, 117 MDF_UNKNOWN, 118 MDF_WRITETHROUGH, 119 MDF_WRITEPROTECT, 120 MDF_WRITEBACK 121 }; 122 123 #define MTRRTOMRTLEN nitems(x86_mtrrtomrt) 124 125 static int 126 x86_mtrr2mrt(int val) 127 { 128 129 if (val < 0 || val >= MTRRTOMRTLEN) 130 return (MDF_UNKNOWN); 131 return (x86_mtrrtomrt[val]); 132 } 133 134 /* 135 * x86 MTRR conflicts. Writeback and uncachable may overlap. 136 */ 137 static int 138 x86_mtrrconflict(int flag1, int flag2) 139 { 140 141 flag1 &= MDF_ATTRMASK; 142 flag2 &= MDF_ATTRMASK; 143 if ((flag1 & MDF_UNKNOWN) || (flag2 & MDF_UNKNOWN)) 144 return (1); 145 if (flag1 == flag2 || 146 (flag1 == MDF_WRITEBACK && flag2 == MDF_UNCACHEABLE) || 147 (flag2 == MDF_WRITEBACK && flag1 == MDF_UNCACHEABLE)) 148 return (0); 149 return (1); 150 } 151 152 /* 153 * Look for an exactly-matching range. 154 */ 155 static struct mem_range_desc * 156 mem_range_match(struct mem_range_softc *sc, struct mem_range_desc *mrd) 157 { 158 struct mem_range_desc *cand; 159 int i; 160 161 for (i = 0, cand = sc->mr_desc; i < sc->mr_ndesc; i++, cand++) 162 if ((cand->mr_base == mrd->mr_base) && 163 (cand->mr_len == mrd->mr_len)) 164 return (cand); 165 return (NULL); 166 } 167 168 /* 169 * Ensure that the direct map region does not contain any mappings 170 * that span MTRRs of different types. However, the fixed MTRRs can 171 * be ignored, because a large page mapping the first 1 MB of physical 172 * memory is a special case that the processor handles. Invalidate 173 * any old TLB entries that might hold inconsistent memory type 174 * information. 175 */ 176 static void 177 x86_mr_split_dmap(struct mem_range_softc *sc __unused) 178 { 179 #ifdef __amd64__ 180 struct mem_range_desc *mrd; 181 int i; 182 183 i = (sc->mr_cap & MR686_FIXMTRR) ? MTRR_N64K + MTRR_N16K + MTRR_N4K : 0; 184 mrd = sc->mr_desc + i; 185 for (; i < sc->mr_ndesc; i++, mrd++) { 186 if ((mrd->mr_flags & (MDF_ACTIVE | MDF_BOGUS)) == MDF_ACTIVE) 187 pmap_demote_DMAP(mrd->mr_base, mrd->mr_len, TRUE); 188 } 189 #endif 190 } 191 192 /* 193 * Fetch the current mtrr settings from the current CPU (assumed to 194 * all be in sync in the SMP case). Note that if we are here, we 195 * assume that MTRRs are enabled, and we may or may not have fixed 196 * MTRRs. 197 */ 198 static void 199 x86_mrfetch(struct mem_range_softc *sc) 200 { 201 struct mem_range_desc *mrd; 202 u_int64_t msrv; 203 int i, j, msr; 204 205 mrd = sc->mr_desc; 206 207 /* Get fixed-range MTRRs. */ 208 if (sc->mr_cap & MR686_FIXMTRR) { 209 msr = MSR_MTRR64kBase; 210 for (i = 0; i < (MTRR_N64K / 8); i++, msr++) { 211 msrv = rdmsr(msr); 212 for (j = 0; j < 8; j++, mrd++) { 213 mrd->mr_flags = 214 (mrd->mr_flags & ~MDF_ATTRMASK) | 215 x86_mtrr2mrt(msrv & 0xff) | MDF_ACTIVE; 216 if (mrd->mr_owner[0] == 0) 217 strcpy(mrd->mr_owner, mem_owner_bios); 218 msrv = msrv >> 8; 219 } 220 } 221 msr = MSR_MTRR16kBase; 222 for (i = 0; i < MTRR_N16K / 8; i++, msr++) { 223 msrv = rdmsr(msr); 224 for (j = 0; j < 8; j++, mrd++) { 225 mrd->mr_flags = 226 (mrd->mr_flags & ~MDF_ATTRMASK) | 227 x86_mtrr2mrt(msrv & 0xff) | MDF_ACTIVE; 228 if (mrd->mr_owner[0] == 0) 229 strcpy(mrd->mr_owner, mem_owner_bios); 230 msrv = msrv >> 8; 231 } 232 } 233 msr = MSR_MTRR4kBase; 234 for (i = 0; i < MTRR_N4K / 8; i++, msr++) { 235 msrv = rdmsr(msr); 236 for (j = 0; j < 8; j++, mrd++) { 237 mrd->mr_flags = 238 (mrd->mr_flags & ~MDF_ATTRMASK) | 239 x86_mtrr2mrt(msrv & 0xff) | MDF_ACTIVE; 240 if (mrd->mr_owner[0] == 0) 241 strcpy(mrd->mr_owner, mem_owner_bios); 242 msrv = msrv >> 8; 243 } 244 } 245 } 246 247 /* Get remainder which must be variable MTRRs. */ 248 msr = MSR_MTRRVarBase; 249 for (; mrd - sc->mr_desc < sc->mr_ndesc; msr += 2, mrd++) { 250 msrv = rdmsr(msr); 251 mrd->mr_flags = (mrd->mr_flags & ~MDF_ATTRMASK) | 252 x86_mtrr2mrt(msrv & MTRR_PHYSBASE_TYPE); 253 mrd->mr_base = msrv & mtrr_physmask; 254 msrv = rdmsr(msr + 1); 255 mrd->mr_flags = (msrv & MTRR_PHYSMASK_VALID) ? 256 (mrd->mr_flags | MDF_ACTIVE) : 257 (mrd->mr_flags & ~MDF_ACTIVE); 258 259 /* Compute the range from the mask. Ick. */ 260 mrd->mr_len = (~(msrv & mtrr_physmask) & 261 (mtrr_physmask | 0xfff)) + 1; 262 if (!mrvalid(mrd->mr_base, mrd->mr_len)) 263 mrd->mr_flags |= MDF_BOGUS; 264 265 /* If unclaimed and active, must be the BIOS. */ 266 if ((mrd->mr_flags & MDF_ACTIVE) && (mrd->mr_owner[0] == 0)) 267 strcpy(mrd->mr_owner, mem_owner_bios); 268 } 269 } 270 271 /* 272 * Return the MTRR memory type matching a region's flags 273 */ 274 static int 275 x86_mtrrtype(int flags) 276 { 277 int i; 278 279 flags &= MDF_ATTRMASK; 280 281 for (i = 0; i < MTRRTOMRTLEN; i++) { 282 if (x86_mtrrtomrt[i] == MDF_UNKNOWN) 283 continue; 284 if (flags == x86_mtrrtomrt[i]) 285 return (i); 286 } 287 return (-1); 288 } 289 290 static int 291 x86_mrt2mtrr(int flags, int oldval) 292 { 293 int val; 294 295 if ((val = x86_mtrrtype(flags)) == -1) 296 return (oldval & 0xff); 297 return (val & 0xff); 298 } 299 300 /* 301 * Update running CPU(s) MTRRs to match the ranges in the descriptor 302 * list. 303 * 304 * Must be called with interrupts enabled. 305 */ 306 static void 307 x86_mrstore(struct mem_range_softc *sc) 308 { 309 310 smp_rendezvous(NULL, x86_mrstoreone, NULL, sc); 311 } 312 313 /* 314 * Update the current CPU's MTRRs with those represented in the 315 * descriptor list. Note that we do this wholesale rather than just 316 * stuffing one entry; this is simpler (but slower, of course). 317 */ 318 static void 319 x86_mrstoreone(void *arg) 320 { 321 struct mem_range_softc *sc = arg; 322 struct mem_range_desc *mrd; 323 u_int64_t omsrv, msrv; 324 int i, j, msr; 325 u_long cr0, cr4; 326 327 mrd = sc->mr_desc; 328 329 critical_enter(); 330 331 /* Disable PGE. */ 332 cr4 = rcr4(); 333 load_cr4(cr4 & ~CR4_PGE); 334 335 /* Disable caches (CD = 1, NW = 0). */ 336 cr0 = rcr0(); 337 load_cr0((cr0 & ~CR0_NW) | CR0_CD); 338 339 /* Flushes caches and TLBs. */ 340 wbinvd(); 341 invltlb(); 342 343 /* Disable MTRRs (E = 0). */ 344 wrmsr(MSR_MTRRdefType, rdmsr(MSR_MTRRdefType) & ~MTRR_DEF_ENABLE); 345 346 /* Set fixed-range MTRRs. */ 347 if (sc->mr_cap & MR686_FIXMTRR) { 348 msr = MSR_MTRR64kBase; 349 for (i = 0; i < MTRR_N64K / 8; i++, msr++) { 350 msrv = 0; 351 omsrv = rdmsr(msr); 352 for (j = 7; j >= 0; j--) { 353 msrv = msrv << 8; 354 msrv |= x86_mrt2mtrr((mrd + j)->mr_flags, 355 omsrv >> (j * 8)); 356 } 357 wrmsr(msr, msrv); 358 mrd += 8; 359 } 360 msr = MSR_MTRR16kBase; 361 for (i = 0; i < MTRR_N16K / 8; i++, msr++) { 362 msrv = 0; 363 omsrv = rdmsr(msr); 364 for (j = 7; j >= 0; j--) { 365 msrv = msrv << 8; 366 msrv |= x86_mrt2mtrr((mrd + j)->mr_flags, 367 omsrv >> (j * 8)); 368 } 369 wrmsr(msr, msrv); 370 mrd += 8; 371 } 372 msr = MSR_MTRR4kBase; 373 for (i = 0; i < MTRR_N4K / 8; i++, msr++) { 374 msrv = 0; 375 omsrv = rdmsr(msr); 376 for (j = 7; j >= 0; j--) { 377 msrv = msrv << 8; 378 msrv |= x86_mrt2mtrr((mrd + j)->mr_flags, 379 omsrv >> (j * 8)); 380 } 381 wrmsr(msr, msrv); 382 mrd += 8; 383 } 384 } 385 386 /* Set remainder which must be variable MTRRs. */ 387 msr = MSR_MTRRVarBase; 388 for (; mrd - sc->mr_desc < sc->mr_ndesc; msr += 2, mrd++) { 389 /* base/type register */ 390 omsrv = rdmsr(msr); 391 if (mrd->mr_flags & MDF_ACTIVE) { 392 msrv = mrd->mr_base & mtrr_physmask; 393 msrv |= x86_mrt2mtrr(mrd->mr_flags, omsrv); 394 } else { 395 msrv = 0; 396 } 397 wrmsr(msr, msrv); 398 399 /* mask/active register */ 400 if (mrd->mr_flags & MDF_ACTIVE) { 401 msrv = MTRR_PHYSMASK_VALID | 402 rounddown2(mtrr_physmask, mrd->mr_len); 403 } else { 404 msrv = 0; 405 } 406 wrmsr(msr + 1, msrv); 407 } 408 409 /* Flush caches and TLBs. */ 410 wbinvd(); 411 invltlb(); 412 413 /* Enable MTRRs. */ 414 wrmsr(MSR_MTRRdefType, rdmsr(MSR_MTRRdefType) | MTRR_DEF_ENABLE); 415 416 /* Restore caches and PGE. */ 417 load_cr0(cr0); 418 load_cr4(cr4); 419 420 critical_exit(); 421 } 422 423 /* 424 * Hunt for the fixed MTRR referencing (addr) 425 */ 426 static struct mem_range_desc * 427 x86_mtrrfixsearch(struct mem_range_softc *sc, u_int64_t addr) 428 { 429 struct mem_range_desc *mrd; 430 int i; 431 432 for (i = 0, mrd = sc->mr_desc; i < MTRR_N64K + MTRR_N16K + MTRR_N4K; 433 i++, mrd++) 434 if (addr >= mrd->mr_base && 435 addr < mrd->mr_base + mrd->mr_len) 436 return (mrd); 437 return (NULL); 438 } 439 440 /* 441 * Try to satisfy the given range request by manipulating the fixed 442 * MTRRs that cover low memory. 443 * 444 * Note that we try to be generous here; we'll bloat the range out to 445 * the next higher/lower boundary to avoid the consumer having to know 446 * too much about the mechanisms here. 447 * 448 * XXX note that this will have to be updated when we start supporting 449 * "busy" ranges. 450 */ 451 static int 452 x86_mrsetlow(struct mem_range_softc *sc, struct mem_range_desc *mrd, int *arg) 453 { 454 struct mem_range_desc *first_md, *last_md, *curr_md; 455 456 /* Range check. */ 457 if ((first_md = x86_mtrrfixsearch(sc, mrd->mr_base)) == NULL || 458 (last_md = x86_mtrrfixsearch(sc, mrd->mr_base + mrd->mr_len - 1)) 459 == NULL) 460 return (EINVAL); 461 462 /* Check that we aren't doing something risky. */ 463 if ((mrd->mr_flags & MDF_FORCE) == 0) { 464 for (curr_md = first_md; curr_md <= last_md; curr_md++) { 465 if ((curr_md->mr_flags & MDF_ATTRMASK) == MDF_UNKNOWN) 466 return (EACCES); 467 } 468 } 469 470 /* Set flags, clear set-by-firmware flag. */ 471 for (curr_md = first_md; curr_md <= last_md; curr_md++) { 472 curr_md->mr_flags = mrcopyflags(curr_md->mr_flags & 473 ~MDF_FIRMWARE, mrd->mr_flags); 474 bcopy(mrd->mr_owner, curr_md->mr_owner, sizeof(mrd->mr_owner)); 475 } 476 477 return (0); 478 } 479 480 /* 481 * Modify/add a variable MTRR to satisfy the request. 482 * 483 * XXX needs to be updated to properly support "busy" ranges. 484 */ 485 static int 486 x86_mrsetvariable(struct mem_range_softc *sc, struct mem_range_desc *mrd, 487 int *arg) 488 { 489 struct mem_range_desc *curr_md, *free_md; 490 int i; 491 492 /* 493 * Scan the currently active variable descriptors, look for 494 * one we exactly match (straight takeover) and for possible 495 * accidental overlaps. 496 * 497 * Keep track of the first empty variable descriptor in case 498 * we can't perform a takeover. 499 */ 500 i = (sc->mr_cap & MR686_FIXMTRR) ? MTRR_N64K + MTRR_N16K + MTRR_N4K : 0; 501 curr_md = sc->mr_desc + i; 502 free_md = NULL; 503 for (; i < sc->mr_ndesc; i++, curr_md++) { 504 if (curr_md->mr_flags & MDF_ACTIVE) { 505 /* Exact match? */ 506 if (curr_md->mr_base == mrd->mr_base && 507 curr_md->mr_len == mrd->mr_len) { 508 /* Whoops, owned by someone. */ 509 if (curr_md->mr_flags & MDF_BUSY) 510 return (EBUSY); 511 512 /* Check that we aren't doing something risky */ 513 if (!(mrd->mr_flags & MDF_FORCE) && 514 (curr_md->mr_flags & MDF_ATTRMASK) == 515 MDF_UNKNOWN) 516 return (EACCES); 517 518 /* Ok, just hijack this entry. */ 519 free_md = curr_md; 520 break; 521 } 522 523 /* Non-exact overlap? */ 524 if (mroverlap(curr_md, mrd)) { 525 /* Between conflicting region types? */ 526 if (x86_mtrrconflict(curr_md->mr_flags, 527 mrd->mr_flags)) 528 return (EINVAL); 529 } 530 } else if (free_md == NULL) { 531 free_md = curr_md; 532 } 533 } 534 535 /* Got somewhere to put it? */ 536 if (free_md == NULL) 537 return (ENOSPC); 538 539 /* Set up new descriptor. */ 540 free_md->mr_base = mrd->mr_base; 541 free_md->mr_len = mrd->mr_len; 542 free_md->mr_flags = mrcopyflags(MDF_ACTIVE, mrd->mr_flags); 543 bcopy(mrd->mr_owner, free_md->mr_owner, sizeof(mrd->mr_owner)); 544 return (0); 545 } 546 547 /* 548 * Handle requests to set memory range attributes by manipulating MTRRs. 549 */ 550 static int 551 x86_mrset(struct mem_range_softc *sc, struct mem_range_desc *mrd, int *arg) 552 { 553 struct mem_range_desc *targ; 554 int error; 555 556 switch (*arg) { 557 case MEMRANGE_SET_UPDATE: 558 /* 559 * Make sure that what's being asked for is even 560 * possible at all. 561 */ 562 if (!mrvalid(mrd->mr_base, mrd->mr_len) || 563 x86_mtrrtype(mrd->mr_flags) == -1) 564 return (EINVAL); 565 566 #define FIXTOP \ 567 ((MTRR_N64K * 0x10000) + (MTRR_N16K * 0x4000) + (MTRR_N4K * 0x1000)) 568 569 /* Are the "low memory" conditions applicable? */ 570 if ((sc->mr_cap & MR686_FIXMTRR) != 0 && 571 mrd->mr_base + mrd->mr_len <= FIXTOP) { 572 if ((error = x86_mrsetlow(sc, mrd, arg)) != 0) 573 return (error); 574 } else { 575 /* It's time to play with variable MTRRs. */ 576 if ((error = x86_mrsetvariable(sc, mrd, arg)) != 0) 577 return (error); 578 } 579 break; 580 581 case MEMRANGE_SET_REMOVE: 582 if ((targ = mem_range_match(sc, mrd)) == NULL) 583 return (ENOENT); 584 if (targ->mr_flags & MDF_FIXACTIVE) 585 return (EPERM); 586 if (targ->mr_flags & MDF_BUSY) 587 return (EBUSY); 588 targ->mr_flags &= ~MDF_ACTIVE; 589 targ->mr_owner[0] = 0; 590 break; 591 592 default: 593 return (EOPNOTSUPP); 594 } 595 596 x86_mr_split_dmap(sc); 597 598 /* Update the hardware. */ 599 x86_mrstore(sc); 600 601 /* Refetch to see where we're at. */ 602 x86_mrfetch(sc); 603 return (0); 604 } 605 606 /* 607 * Work out how many ranges we support, initialise storage for them, 608 * and fetch the initial settings. 609 */ 610 static void 611 x86_mrinit(struct mem_range_softc *sc) 612 { 613 struct mem_range_desc *mrd; 614 int i, nmdesc; 615 616 if (sc->mr_desc != NULL) 617 /* Already initialized. */ 618 return; 619 620 nmdesc = 0; 621 mtrrcap = rdmsr(MSR_MTRRcap); 622 mtrrdef = rdmsr(MSR_MTRRdefType); 623 624 /* For now, bail out if MTRRs are not enabled. */ 625 if (!(mtrrdef & MTRR_DEF_ENABLE)) { 626 if (bootverbose) 627 printf("CPU supports MTRRs but not enabled\n"); 628 return; 629 } 630 nmdesc = mtrrcap & MTRR_CAP_VCNT; 631 if (bootverbose) 632 printf("Pentium Pro MTRR support enabled\n"); 633 634 /* 635 * Determine the size of the PhysMask and PhysBase fields in 636 * the variable range MTRRs. 637 */ 638 mtrr_physmask = (((uint64_t)1 << cpu_maxphyaddr) - 1) & 639 ~(uint64_t)0xfff; 640 641 /* If fixed MTRRs supported and enabled. */ 642 if ((mtrrcap & MTRR_CAP_FIXED) && (mtrrdef & MTRR_DEF_FIXED_ENABLE)) { 643 sc->mr_cap = MR686_FIXMTRR; 644 nmdesc += MTRR_N64K + MTRR_N16K + MTRR_N4K; 645 } 646 647 sc->mr_desc = malloc(nmdesc * sizeof(struct mem_range_desc), M_MEMDESC, 648 M_WAITOK | M_ZERO); 649 sc->mr_ndesc = nmdesc; 650 651 mrd = sc->mr_desc; 652 653 /* Populate the fixed MTRR entries' base/length. */ 654 if (sc->mr_cap & MR686_FIXMTRR) { 655 for (i = 0; i < MTRR_N64K; i++, mrd++) { 656 mrd->mr_base = i * 0x10000; 657 mrd->mr_len = 0x10000; 658 mrd->mr_flags = MDF_FIXBASE | MDF_FIXLEN | 659 MDF_FIXACTIVE; 660 } 661 for (i = 0; i < MTRR_N16K; i++, mrd++) { 662 mrd->mr_base = i * 0x4000 + 0x80000; 663 mrd->mr_len = 0x4000; 664 mrd->mr_flags = MDF_FIXBASE | MDF_FIXLEN | 665 MDF_FIXACTIVE; 666 } 667 for (i = 0; i < MTRR_N4K; i++, mrd++) { 668 mrd->mr_base = i * 0x1000 + 0xc0000; 669 mrd->mr_len = 0x1000; 670 mrd->mr_flags = MDF_FIXBASE | MDF_FIXLEN | 671 MDF_FIXACTIVE; 672 } 673 } 674 675 /* 676 * Get current settings, anything set now is considered to 677 * have been set by the firmware. (XXX has something already 678 * played here?) 679 */ 680 x86_mrfetch(sc); 681 mrd = sc->mr_desc; 682 for (i = 0; i < sc->mr_ndesc; i++, mrd++) { 683 if (mrd->mr_flags & MDF_ACTIVE) 684 mrd->mr_flags |= MDF_FIRMWARE; 685 } 686 687 x86_mr_split_dmap(sc); 688 } 689 690 /* 691 * Initialise MTRRs on an AP after the BSP has run the init code. 692 */ 693 static void 694 x86_mrAPinit(struct mem_range_softc *sc) 695 { 696 697 x86_mrstoreone(sc); 698 wrmsr(MSR_MTRRdefType, mtrrdef); 699 } 700 701 /* 702 * Re-initialise running CPU(s) MTRRs to match the ranges in the descriptor 703 * list. 704 * 705 * Must be called with interrupts enabled. 706 */ 707 static void 708 x86_mrreinit(struct mem_range_softc *sc) 709 { 710 711 smp_rendezvous(NULL, (void (*)(void *))x86_mrAPinit, NULL, sc); 712 } 713 714 static void 715 x86_mem_drvinit(void *unused) 716 { 717 718 if (mtrrs_disabled) 719 return; 720 if (!(cpu_feature & CPUID_MTRR)) 721 return; 722 mem_range_softc.mr_op = &x86_mrops; 723 x86_mrinit(&mem_range_softc); 724 } 725 SYSINIT(x86memdev, SI_SUB_CPU, SI_ORDER_ANY, x86_mem_drvinit, NULL); 726