1 /*- 2 * Copyright (c) 1999 Michael Smith <msmith@freebsd.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <sys/param.h> 28 #include <sys/kernel.h> 29 #include <sys/systm.h> 30 #include <sys/malloc.h> 31 #include <sys/memrange.h> 32 #include <sys/sysctl.h> 33 #include <sys/thread.h> 34 35 #include <vm/vm.h> 36 #include <vm/vm_param.h> 37 #include <vm/pmap.h> 38 39 #include <sys/thread2.h> 40 41 #include <machine/cputypes.h> 42 #include <machine/md_var.h> 43 #include <machine/specialreg.h> 44 #include <machine/smp.h> 45 46 /* 47 * amd64 memory range operations 48 * 49 * This code will probably be impenetrable without reference to the 50 * Intel Pentium Pro documentation or x86-64 programmers manual vol 2. 51 */ 52 53 static char *mem_owner_bios = "BIOS"; 54 55 #define MR686_FIXMTRR (1<<0) 56 57 #define mrwithin(mr, a) \ 58 (((a) >= (mr)->mr_base) && ((a) < ((mr)->mr_base + (mr)->mr_len))) 59 #define mroverlap(mra, mrb) \ 60 (mrwithin(mra, mrb->mr_base) || mrwithin(mrb, mra->mr_base)) 61 62 #define mrvalid(base, len) \ 63 ((!(base & ((1 << 12) - 1))) && /* base is multiple of 4k */ \ 64 ((len) >= (1 << 12)) && /* length is >= 4k */ \ 65 powerof2((len)) && /* ... and power of two */ \ 66 !((base) & ((len) - 1))) /* range is not discontiuous */ 67 68 #define mrcopyflags(curr, new) \ 69 (((curr) & ~MDF_ATTRMASK) | ((new) & MDF_ATTRMASK)) 70 71 static int mtrrs_disabled; 72 TUNABLE_INT("machdep.disable_mtrrs", &mtrrs_disabled); 73 SYSCTL_INT(_machdep, OID_AUTO, disable_mtrrs, CTLFLAG_RD, 74 &mtrrs_disabled, 0, "Disable amd64 MTRRs."); 75 76 static void amd64_mrinit(struct mem_range_softc *sc); 77 static int amd64_mrset(struct mem_range_softc *sc, 78 struct mem_range_desc *mrd, int *arg); 79 static void amd64_mrAPinit(struct mem_range_softc *sc); 80 static void amd64_mrreinit(struct mem_range_softc *sc); 81 82 static struct mem_range_ops amd64_mrops = { 83 amd64_mrinit, 84 amd64_mrset, 85 amd64_mrAPinit, 86 amd64_mrreinit 87 }; 88 89 /* XXX for AP startup hook */ 90 static u_int64_t mtrrcap, mtrrdef; 91 92 /* The bitmask for the PhysBase and PhysMask fields of the variable MTRRs. */ 93 static u_int64_t mtrr_physmask; 94 95 static struct mem_range_desc *mem_range_match(struct mem_range_softc *sc, 96 struct mem_range_desc *mrd); 97 static void amd64_mrfetch(struct mem_range_softc *sc); 98 static int amd64_mtrrtype(int flags); 99 static int amd64_mrt2mtrr(int flags, int oldval); 100 static int amd64_mtrrconflict(int flag1, int flag2); 101 static void amd64_mrstore(struct mem_range_softc *sc); 102 static void amd64_mrstoreone(void *arg); 103 static struct mem_range_desc *amd64_mtrrfixsearch(struct mem_range_softc *sc, 104 u_int64_t addr); 105 static int amd64_mrsetlow(struct mem_range_softc *sc, 106 struct mem_range_desc *mrd, int *arg); 107 static int amd64_mrsetvariable(struct mem_range_softc *sc, 108 struct mem_range_desc *mrd, int *arg); 109 110 /* amd64 MTRR type to memory range type conversion */ 111 static int amd64_mtrrtomrt[] = { 112 MDF_UNCACHEABLE, 113 MDF_WRITECOMBINE, 114 MDF_UNKNOWN, 115 MDF_UNKNOWN, 116 MDF_WRITETHROUGH, 117 MDF_WRITEPROTECT, 118 MDF_WRITEBACK 119 }; 120 121 #define MTRRTOMRTLEN NELEM(amd64_mtrrtomrt) 122 123 static int 124 amd64_mtrr2mrt(int val) 125 { 126 127 if (val < 0 || val >= MTRRTOMRTLEN) 128 return (MDF_UNKNOWN); 129 return (amd64_mtrrtomrt[val]); 130 } 131 132 /* 133 * amd64 MTRR conflicts. Writeback and uncachable may overlap. 134 */ 135 static int 136 amd64_mtrrconflict(int flag1, int flag2) 137 { 138 139 flag1 &= MDF_ATTRMASK; 140 flag2 &= MDF_ATTRMASK; 141 if ((flag1 & MDF_UNKNOWN) || (flag2 & MDF_UNKNOWN)) 142 return (1); 143 if (flag1 == flag2 || 144 (flag1 == MDF_WRITEBACK && flag2 == MDF_UNCACHEABLE) || 145 (flag2 == MDF_WRITEBACK && flag1 == MDF_UNCACHEABLE)) 146 return (0); 147 return (1); 148 } 149 150 /* 151 * Look for an exactly-matching range. 152 */ 153 static struct mem_range_desc * 154 mem_range_match(struct mem_range_softc *sc, struct mem_range_desc *mrd) 155 { 156 struct mem_range_desc *cand; 157 int i; 158 159 for (i = 0, cand = sc->mr_desc; i < sc->mr_ndesc; i++, cand++) 160 if ((cand->mr_base == mrd->mr_base) && 161 (cand->mr_len == mrd->mr_len)) 162 return (cand); 163 return (NULL); 164 } 165 166 /* 167 * Fetch the current mtrr settings from the current CPU (assumed to 168 * all be in sync in the SMP case). Note that if we are here, we 169 * assume that MTRRs are enabled, and we may or may not have fixed 170 * MTRRs. 171 */ 172 static void 173 amd64_mrfetch(struct mem_range_softc *sc) 174 { 175 struct mem_range_desc *mrd; 176 u_int64_t msrv; 177 int i, j, msr; 178 179 mrd = sc->mr_desc; 180 181 /* Get fixed-range MTRRs. */ 182 if (sc->mr_cap & MR686_FIXMTRR) { 183 msr = MSR_MTRR64kBase; 184 for (i = 0; i < (MTRR_N64K / 8); i++, msr++) { 185 msrv = rdmsr(msr); 186 for (j = 0; j < 8; j++, mrd++) { 187 mrd->mr_flags = 188 (mrd->mr_flags & ~MDF_ATTRMASK) | 189 amd64_mtrr2mrt(msrv & 0xff) | MDF_ACTIVE; 190 if (mrd->mr_owner[0] == 0) 191 strcpy(mrd->mr_owner, mem_owner_bios); 192 msrv = msrv >> 8; 193 } 194 } 195 msr = MSR_MTRR16kBase; 196 for (i = 0; i < (MTRR_N16K / 8); i++, msr++) { 197 msrv = rdmsr(msr); 198 for (j = 0; j < 8; j++, mrd++) { 199 mrd->mr_flags = 200 (mrd->mr_flags & ~MDF_ATTRMASK) | 201 amd64_mtrr2mrt(msrv & 0xff) | MDF_ACTIVE; 202 if (mrd->mr_owner[0] == 0) 203 strcpy(mrd->mr_owner, mem_owner_bios); 204 msrv = msrv >> 8; 205 } 206 } 207 msr = MSR_MTRR4kBase; 208 for (i = 0; i < (MTRR_N4K / 8); i++, msr++) { 209 msrv = rdmsr(msr); 210 for (j = 0; j < 8; j++, mrd++) { 211 mrd->mr_flags = 212 (mrd->mr_flags & ~MDF_ATTRMASK) | 213 amd64_mtrr2mrt(msrv & 0xff) | MDF_ACTIVE; 214 if (mrd->mr_owner[0] == 0) 215 strcpy(mrd->mr_owner, mem_owner_bios); 216 msrv = msrv >> 8; 217 } 218 } 219 } 220 221 /* Get remainder which must be variable MTRRs. */ 222 msr = MSR_MTRRVarBase; 223 for (; (mrd - sc->mr_desc) < sc->mr_ndesc; msr += 2, mrd++) { 224 msrv = rdmsr(msr); 225 mrd->mr_flags = (mrd->mr_flags & ~MDF_ATTRMASK) | 226 amd64_mtrr2mrt(msrv & MTRR_PHYSBASE_TYPE); 227 mrd->mr_base = msrv & mtrr_physmask; 228 msrv = rdmsr(msr + 1); 229 mrd->mr_flags = (msrv & MTRR_PHYSMASK_VALID) ? 230 (mrd->mr_flags | MDF_ACTIVE) : 231 (mrd->mr_flags & ~MDF_ACTIVE); 232 233 /* Compute the range from the mask. Ick. */ 234 mrd->mr_len = (~(msrv & mtrr_physmask) & 235 (mtrr_physmask | 0xfffL)) + 1; 236 if (!mrvalid(mrd->mr_base, mrd->mr_len)) 237 mrd->mr_flags |= MDF_BOGUS; 238 239 /* If unclaimed and active, must be the BIOS. */ 240 if ((mrd->mr_flags & MDF_ACTIVE) && (mrd->mr_owner[0] == 0)) 241 strcpy(mrd->mr_owner, mem_owner_bios); 242 } 243 } 244 245 /* 246 * Return the MTRR memory type matching a region's flags 247 */ 248 static int 249 amd64_mtrrtype(int flags) 250 { 251 int i; 252 253 flags &= MDF_ATTRMASK; 254 255 for (i = 0; i < MTRRTOMRTLEN; i++) { 256 if (amd64_mtrrtomrt[i] == MDF_UNKNOWN) 257 continue; 258 if (flags == amd64_mtrrtomrt[i]) 259 return (i); 260 } 261 return (-1); 262 } 263 264 static int 265 amd64_mrt2mtrr(int flags, int oldval) 266 { 267 int val; 268 269 if ((val = amd64_mtrrtype(flags)) == -1) 270 return (oldval & 0xff); 271 return (val & 0xff); 272 } 273 274 /* 275 * Update running CPU(s) MTRRs to match the ranges in the descriptor 276 * list. 277 * 278 * XXX Must be called with interrupts enabled. 279 */ 280 static void 281 amd64_mrstore(struct mem_range_softc *sc) 282 { 283 #ifdef SMP 284 /* 285 * We should use ipi_all_but_self() to call other CPUs into a 286 * locking gate, then call a target function to do this work. 287 * The "proper" solution involves a generalised locking gate 288 * implementation, not ready yet. 289 */ 290 lwkt_send_ipiq_mask(smp_active_mask, (void *)amd64_mrstoreone, sc); 291 #else 292 crit_enter(); 293 amd64_mrstoreone(sc); 294 crit_exit(); 295 #endif 296 } 297 298 /* 299 * Update the current CPU's MTRRs with those represented in the 300 * descriptor list. Note that we do this wholesale rather than just 301 * stuffing one entry; this is simpler (but slower, of course). 302 */ 303 static void 304 amd64_mrstoreone(void *arg) 305 { 306 struct mem_range_softc *sc = arg; 307 struct mem_range_desc *mrd; 308 u_int64_t omsrv, msrv; 309 int i, j, msr; 310 u_long cr0, cr4; 311 312 mrd = sc->mr_desc; 313 314 crit_enter(); 315 316 /* Disable PGE. */ 317 cr4 = rcr4(); 318 load_cr4(cr4 & ~CR4_PGE); 319 320 /* Disable caches (CD = 1, NW = 0). */ 321 cr0 = rcr0(); 322 load_cr0((cr0 & ~CR0_NW) | CR0_CD); 323 324 /* Flushes caches and TLBs. */ 325 wbinvd(); 326 cpu_invltlb(); 327 328 /* Disable MTRRs (E = 0). */ 329 wrmsr(MSR_MTRRdefType, rdmsr(MSR_MTRRdefType) & ~MTRR_DEF_ENABLE); 330 331 /* Set fixed-range MTRRs. */ 332 if (sc->mr_cap & MR686_FIXMTRR) { 333 msr = MSR_MTRR64kBase; 334 for (i = 0; i < (MTRR_N64K / 8); i++, msr++) { 335 msrv = 0; 336 omsrv = rdmsr(msr); 337 for (j = 7; j >= 0; j--) { 338 msrv = msrv << 8; 339 msrv |= amd64_mrt2mtrr((mrd + j)->mr_flags, 340 omsrv >> (j * 8)); 341 } 342 wrmsr(msr, msrv); 343 mrd += 8; 344 } 345 msr = MSR_MTRR16kBase; 346 for (i = 0; i < (MTRR_N16K / 8); i++, msr++) { 347 msrv = 0; 348 omsrv = rdmsr(msr); 349 for (j = 7; j >= 0; j--) { 350 msrv = msrv << 8; 351 msrv |= amd64_mrt2mtrr((mrd + j)->mr_flags, 352 omsrv >> (j * 8)); 353 } 354 wrmsr(msr, msrv); 355 mrd += 8; 356 } 357 msr = MSR_MTRR4kBase; 358 for (i = 0; i < (MTRR_N4K / 8); i++, msr++) { 359 msrv = 0; 360 omsrv = rdmsr(msr); 361 for (j = 7; j >= 0; j--) { 362 msrv = msrv << 8; 363 msrv |= amd64_mrt2mtrr((mrd + j)->mr_flags, 364 omsrv >> (j * 8)); 365 } 366 wrmsr(msr, msrv); 367 mrd += 8; 368 } 369 } 370 371 /* Set remainder which must be variable MTRRs. */ 372 msr = MSR_MTRRVarBase; 373 for (; (mrd - sc->mr_desc) < sc->mr_ndesc; msr += 2, mrd++) { 374 /* base/type register */ 375 omsrv = rdmsr(msr); 376 if (mrd->mr_flags & MDF_ACTIVE) { 377 msrv = mrd->mr_base & mtrr_physmask; 378 msrv |= amd64_mrt2mtrr(mrd->mr_flags, omsrv); 379 } else { 380 msrv = 0; 381 } 382 wrmsr(msr, msrv); 383 384 /* mask/active register */ 385 if (mrd->mr_flags & MDF_ACTIVE) { 386 msrv = MTRR_PHYSMASK_VALID | 387 (~(mrd->mr_len - 1) & mtrr_physmask); 388 } else { 389 msrv = 0; 390 } 391 wrmsr(msr + 1, msrv); 392 } 393 394 /* Flush caches and TLBs. */ 395 wbinvd(); 396 cpu_invltlb(); 397 398 /* Enable MTRRs. */ 399 wrmsr(MSR_MTRRdefType, rdmsr(MSR_MTRRdefType) | MTRR_DEF_ENABLE); 400 401 /* Restore caches and PGE. */ 402 load_cr0(cr0); 403 load_cr4(cr4); 404 405 crit_exit(); 406 } 407 408 /* 409 * Hunt for the fixed MTRR referencing (addr) 410 */ 411 static struct mem_range_desc * 412 amd64_mtrrfixsearch(struct mem_range_softc *sc, u_int64_t addr) 413 { 414 struct mem_range_desc *mrd; 415 int i; 416 417 for (i = 0, mrd = sc->mr_desc; i < (MTRR_N64K + MTRR_N16K + MTRR_N4K); 418 i++, mrd++) 419 if ((addr >= mrd->mr_base) && 420 (addr < (mrd->mr_base + mrd->mr_len))) 421 return (mrd); 422 return (NULL); 423 } 424 425 /* 426 * Try to satisfy the given range request by manipulating the fixed 427 * MTRRs that cover low memory. 428 * 429 * Note that we try to be generous here; we'll bloat the range out to 430 * the next higher/lower boundary to avoid the consumer having to know 431 * too much about the mechanisms here. 432 * 433 * XXX note that this will have to be updated when we start supporting 434 * "busy" ranges. 435 */ 436 static int 437 amd64_mrsetlow(struct mem_range_softc *sc, struct mem_range_desc *mrd, int *arg) 438 { 439 struct mem_range_desc *first_md, *last_md, *curr_md; 440 441 /* Range check. */ 442 if (((first_md = amd64_mtrrfixsearch(sc, mrd->mr_base)) == NULL) || 443 ((last_md = amd64_mtrrfixsearch(sc, mrd->mr_base + mrd->mr_len - 1)) == NULL)) 444 return (EINVAL); 445 446 /* Check that we aren't doing something risky. */ 447 if (!(mrd->mr_flags & MDF_FORCE)) 448 for (curr_md = first_md; curr_md <= last_md; curr_md++) { 449 if ((curr_md->mr_flags & MDF_ATTRMASK) == MDF_UNKNOWN) 450 return (EACCES); 451 } 452 453 /* Set flags, clear set-by-firmware flag. */ 454 for (curr_md = first_md; curr_md <= last_md; curr_md++) { 455 curr_md->mr_flags = mrcopyflags(curr_md->mr_flags & 456 ~MDF_FIRMWARE, mrd->mr_flags); 457 bcopy(mrd->mr_owner, curr_md->mr_owner, sizeof(mrd->mr_owner)); 458 } 459 460 return (0); 461 } 462 463 /* 464 * Modify/add a variable MTRR to satisfy the request. 465 * 466 * XXX needs to be updated to properly support "busy" ranges. 467 */ 468 static int 469 amd64_mrsetvariable(struct mem_range_softc *sc, struct mem_range_desc *mrd, 470 int *arg) 471 { 472 struct mem_range_desc *curr_md, *free_md; 473 int i; 474 475 /* 476 * Scan the currently active variable descriptors, look for 477 * one we exactly match (straight takeover) and for possible 478 * accidental overlaps. 479 * 480 * Keep track of the first empty variable descriptor in case 481 * we can't perform a takeover. 482 */ 483 i = (sc->mr_cap & MR686_FIXMTRR) ? MTRR_N64K + MTRR_N16K + MTRR_N4K : 0; 484 curr_md = sc->mr_desc + i; 485 free_md = NULL; 486 for (; i < sc->mr_ndesc; i++, curr_md++) { 487 if (curr_md->mr_flags & MDF_ACTIVE) { 488 /* Exact match? */ 489 if ((curr_md->mr_base == mrd->mr_base) && 490 (curr_md->mr_len == mrd->mr_len)) { 491 492 /* Whoops, owned by someone. */ 493 if (curr_md->mr_flags & MDF_BUSY) 494 return (EBUSY); 495 496 /* Check that we aren't doing something risky */ 497 if (!(mrd->mr_flags & MDF_FORCE) && 498 ((curr_md->mr_flags & MDF_ATTRMASK) == 499 MDF_UNKNOWN)) 500 return (EACCES); 501 502 /* Ok, just hijack this entry. */ 503 free_md = curr_md; 504 break; 505 } 506 507 /* Non-exact overlap? */ 508 if (mroverlap(curr_md, mrd)) { 509 /* Between conflicting region types? */ 510 if (amd64_mtrrconflict(curr_md->mr_flags, 511 mrd->mr_flags)) 512 return (EINVAL); 513 } 514 } else if (free_md == NULL) { 515 free_md = curr_md; 516 } 517 } 518 519 /* Got somewhere to put it? */ 520 if (free_md == NULL) 521 return (ENOSPC); 522 523 /* Set up new descriptor. */ 524 free_md->mr_base = mrd->mr_base; 525 free_md->mr_len = mrd->mr_len; 526 free_md->mr_flags = mrcopyflags(MDF_ACTIVE, mrd->mr_flags); 527 bcopy(mrd->mr_owner, free_md->mr_owner, sizeof(mrd->mr_owner)); 528 return (0); 529 } 530 531 /* 532 * Handle requests to set memory range attributes by manipulating MTRRs. 533 */ 534 static int 535 amd64_mrset(struct mem_range_softc *sc, struct mem_range_desc *mrd, int *arg) 536 { 537 struct mem_range_desc *targ; 538 int error; 539 540 switch (*arg) { 541 case MEMRANGE_SET_UPDATE: 542 /* 543 * Make sure that what's being asked for is even 544 * possible at all. 545 */ 546 if (!mrvalid(mrd->mr_base, mrd->mr_len) || 547 amd64_mtrrtype(mrd->mr_flags) == -1) 548 return (EINVAL); 549 550 #define FIXTOP ((MTRR_N64K * 0x10000) + (MTRR_N16K * 0x4000) + (MTRR_N4K * 0x1000)) 551 552 /* Are the "low memory" conditions applicable? */ 553 if ((sc->mr_cap & MR686_FIXMTRR) && 554 ((mrd->mr_base + mrd->mr_len) <= FIXTOP)) { 555 if ((error = amd64_mrsetlow(sc, mrd, arg)) != 0) 556 return (error); 557 } else { 558 /* It's time to play with variable MTRRs. */ 559 if ((error = amd64_mrsetvariable(sc, mrd, arg)) != 0) 560 return (error); 561 } 562 break; 563 564 case MEMRANGE_SET_REMOVE: 565 if ((targ = mem_range_match(sc, mrd)) == NULL) 566 return (ENOENT); 567 if (targ->mr_flags & MDF_FIXACTIVE) 568 return (EPERM); 569 if (targ->mr_flags & MDF_BUSY) 570 return (EBUSY); 571 targ->mr_flags &= ~MDF_ACTIVE; 572 targ->mr_owner[0] = 0; 573 break; 574 575 default: 576 return (EOPNOTSUPP); 577 } 578 579 #if 0 580 /* XXX */ 581 /* 582 * Ensure that the direct map region does not contain any mappings 583 * that span MTRRs of different types. However, the fixed MTRRs can 584 * be ignored, because a large page mapping the first 1 MB of physical 585 * memory is a special case that the processor handles. The entire 586 * TLB will be invalidated by amd64_mrstore(), so pmap_demote_DMAP() 587 * needn't do it. 588 */ 589 int i; 590 591 i = (sc->mr_cap & MR686_FIXMTRR) ? MTRR_N64K + MTRR_N16K + MTRR_N4K : 0; 592 mrd = sc->mr_desc + i; 593 for (; i < sc->mr_ndesc; i++, mrd++) { 594 if ((mrd->mr_flags & (MDF_ACTIVE | MDF_BOGUS)) == MDF_ACTIVE) 595 pmap_demote_DMAP(mrd->mr_base, mrd->mr_len, FALSE); 596 } 597 #endif 598 599 /* Update the hardware. */ 600 amd64_mrstore(sc); 601 602 /* Refetch to see where we're at. */ 603 amd64_mrfetch(sc); 604 return (0); 605 } 606 607 /* 608 * Work out how many ranges we support, initialise storage for them, 609 * and fetch the initial settings. 610 */ 611 static void 612 amd64_mrinit(struct mem_range_softc *sc) 613 { 614 struct mem_range_desc *mrd; 615 u_int regs[4]; 616 int i, nmdesc = 0, pabits; 617 618 mtrrcap = rdmsr(MSR_MTRRcap); 619 mtrrdef = rdmsr(MSR_MTRRdefType); 620 621 /* For now, bail out if MTRRs are not enabled. */ 622 if (!(mtrrdef & MTRR_DEF_ENABLE)) { 623 if (bootverbose) 624 kprintf("CPU supports MTRRs but not enabled\n"); 625 return; 626 } 627 nmdesc = mtrrcap & MTRR_CAP_VCNT; 628 629 /* 630 * Determine the size of the PhysMask and PhysBase fields in 631 * the variable range MTRRs. If the extended CPUID 0x80000008 632 * is present, use that to figure out how many physical 633 * address bits the CPU supports. Otherwise, default to 36 634 * address bits. 635 */ 636 if (cpu_exthigh >= 0x80000008) { 637 do_cpuid(0x80000008, regs); 638 pabits = regs[0] & 0xff; 639 } else 640 pabits = 36; 641 mtrr_physmask = ((1UL << pabits) - 1) & ~0xfffUL; 642 643 /* If fixed MTRRs supported and enabled. */ 644 if ((mtrrcap & MTRR_CAP_FIXED) && (mtrrdef & MTRR_DEF_FIXED_ENABLE)) { 645 sc->mr_cap = MR686_FIXMTRR; 646 nmdesc += MTRR_N64K + MTRR_N16K + MTRR_N4K; 647 } 648 649 sc->mr_desc = kmalloc(nmdesc * sizeof(struct mem_range_desc), 650 M_MEMDESC, M_WAITOK | M_ZERO); 651 sc->mr_ndesc = nmdesc; 652 653 mrd = sc->mr_desc; 654 655 /* Populate the fixed MTRR entries' base/length. */ 656 if (sc->mr_cap & MR686_FIXMTRR) { 657 for (i = 0; i < MTRR_N64K; i++, mrd++) { 658 mrd->mr_base = i * 0x10000; 659 mrd->mr_len = 0x10000; 660 mrd->mr_flags = MDF_FIXBASE | MDF_FIXLEN | 661 MDF_FIXACTIVE; 662 } 663 for (i = 0; i < MTRR_N16K; i++, mrd++) { 664 mrd->mr_base = i * 0x4000 + 0x80000; 665 mrd->mr_len = 0x4000; 666 mrd->mr_flags = MDF_FIXBASE | MDF_FIXLEN | 667 MDF_FIXACTIVE; 668 } 669 for (i = 0; i < MTRR_N4K; i++, mrd++) { 670 mrd->mr_base = i * 0x1000 + 0xc0000; 671 mrd->mr_len = 0x1000; 672 mrd->mr_flags = MDF_FIXBASE | MDF_FIXLEN | 673 MDF_FIXACTIVE; 674 } 675 } 676 677 /* 678 * Get current settings, anything set now is considered to 679 * have been set by the firmware. (XXX has something already 680 * played here?) 681 */ 682 amd64_mrfetch(sc); 683 mrd = sc->mr_desc; 684 for (i = 0; i < sc->mr_ndesc; i++, mrd++) { 685 if (mrd->mr_flags & MDF_ACTIVE) 686 mrd->mr_flags |= MDF_FIRMWARE; 687 } 688 689 #if 0 690 /* 691 * Ensure that the direct map region does not contain any mappings 692 * that span MTRRs of different types. However, the fixed MTRRs can 693 * be ignored, because a large page mapping the first 1 MB of physical 694 * memory is a special case that the processor handles. Invalidate 695 * any old TLB entries that might hold inconsistent memory type 696 * information. 697 */ 698 i = (sc->mr_cap & MR686_FIXMTRR) ? MTRR_N64K + MTRR_N16K + MTRR_N4K : 0; 699 mrd = sc->mr_desc + i; 700 for (; i < sc->mr_ndesc; i++, mrd++) { 701 if ((mrd->mr_flags & (MDF_ACTIVE | MDF_BOGUS)) == MDF_ACTIVE) 702 pmap_demote_DMAP(mrd->mr_base, mrd->mr_len, TRUE); 703 } 704 #endif 705 } 706 707 /* 708 * Initialise MTRRs on an AP after the BSP has run the init code. 709 */ 710 static void 711 amd64_mrAPinit(struct mem_range_softc *sc) 712 { 713 amd64_mrstoreone(sc); 714 wrmsr(MSR_MTRRdefType, mtrrdef); 715 } 716 717 /* 718 * Re-initialise running CPU(s) MTRRs to match the ranges in the descriptor 719 * list. 720 * 721 * XXX Must be called with interrupts enabled. 722 */ 723 static void 724 amd64_mrreinit(struct mem_range_softc *sc) 725 { 726 #ifdef SMP 727 /* 728 * We should use ipi_all_but_self() to call other CPUs into a 729 * locking gate, then call a target function to do this work. 730 * The "proper" solution involves a generalised locking gate 731 * implementation, not ready yet. 732 */ 733 lwkt_send_ipiq_mask(smp_active_mask, (void *)amd64_mrAPinit, sc); 734 #else 735 crit_enter(); 736 amd64_mrAPinit(sc); 737 crit_exit(); 738 #endif 739 } 740 741 static void 742 amd64_mem_drvinit(void *unused) 743 { 744 745 if (mtrrs_disabled) 746 return; 747 if (!(cpu_feature & CPUID_MTRR)) 748 return; 749 if ((cpu_id & 0xf00) != 0x600 && (cpu_id & 0xf00) != 0xf00) 750 return; 751 switch (cpu_vendor_id) { 752 case CPU_VENDOR_INTEL: 753 case CPU_VENDOR_AMD: 754 case CPU_VENDOR_CENTAUR: 755 break; 756 default: 757 return; 758 } 759 mem_range_softc.mr_op = &amd64_mrops; 760 } 761 SYSINIT(amd64memdev, SI_SUB_DRIVERS, SI_ORDER_FIRST, amd64_mem_drvinit, NULL); 762