1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 1996, by Steve Passe 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. The name of the developer may NOT be used to endorse or promote products 13 * derived from this software without specific prior written permission. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_acpi.h" 32 #include "opt_apic.h" 33 #include "opt_cpu.h" 34 #include "opt_kstack_pages.h" 35 #include "opt_pmap.h" 36 #include "opt_sched.h" 37 #include "opt_smp.h" 38 39 #if !defined(lint) 40 #if !defined(SMP) 41 #error How did you get here? 42 #endif 43 44 #ifndef DEV_APIC 45 #error The apic device is required for SMP, add "device apic" to your config file. 46 #endif 47 #endif /* not lint */ 48 49 #include <sys/param.h> 50 #include <sys/systm.h> 51 #include <sys/bus.h> 52 #include <sys/cons.h> /* cngetc() */ 53 #include <sys/cpuset.h> 54 #include <sys/kdb.h> 55 #include <sys/kernel.h> 56 #include <sys/ktr.h> 57 #include <sys/lock.h> 58 #include <sys/malloc.h> 59 #include <sys/memrange.h> 60 #include <sys/mutex.h> 61 #include <sys/pcpu.h> 62 #include <sys/proc.h> 63 #include <sys/sched.h> 64 #include <sys/smp.h> 65 #include <sys/sysctl.h> 66 67 #include <vm/vm.h> 68 #include <vm/vm_param.h> 69 #include <vm/pmap.h> 70 #include <vm/vm_kern.h> 71 #include <vm/vm_extern.h> 72 73 #include <x86/apicreg.h> 74 #include <machine/clock.h> 75 #include <machine/cpu.h> 76 #include <machine/cputypes.h> 77 #include <x86/mca.h> 78 #include <machine/md_var.h> 79 #include <machine/pcb.h> 80 #include <machine/psl.h> 81 #include <machine/smp.h> 82 #include <machine/specialreg.h> 83 #include <x86/ucode.h> 84 85 #ifdef DEV_ACPI 86 #include <contrib/dev/acpica/include/acpi.h> 87 #include <dev/acpica/acpivar.h> 88 #endif 89 90 #define WARMBOOT_TARGET 0 91 #define WARMBOOT_OFF (PMAP_MAP_LOW + 0x0467) 92 #define WARMBOOT_SEG (PMAP_MAP_LOW + 0x0469) 93 94 #define CMOS_REG (0x70) 95 #define CMOS_DATA (0x71) 96 #define BIOS_RESET (0x0f) 97 #define BIOS_WARM (0x0a) 98 99 /* 100 * this code MUST be enabled here and in mpboot.s. 101 * it follows the very early stages of AP boot by placing values in CMOS ram. 102 * it NORMALLY will never be needed and thus the primitive method for enabling. 103 * 104 #define CHECK_POINTS 105 */ 106 107 #if defined(CHECK_POINTS) 108 #define CHECK_READ(A) (outb(CMOS_REG, (A)), inb(CMOS_DATA)) 109 #define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D))) 110 111 #define CHECK_INIT(D); \ 112 CHECK_WRITE(0x34, (D)); \ 113 CHECK_WRITE(0x35, (D)); \ 114 CHECK_WRITE(0x36, (D)); \ 115 CHECK_WRITE(0x37, (D)); \ 116 CHECK_WRITE(0x38, (D)); \ 117 CHECK_WRITE(0x39, (D)); 118 119 #define CHECK_PRINT(S); \ 120 printf("%s: %d, %d, %d, %d, %d, %d\n", \ 121 (S), \ 122 CHECK_READ(0x34), \ 123 CHECK_READ(0x35), \ 124 CHECK_READ(0x36), \ 125 CHECK_READ(0x37), \ 126 CHECK_READ(0x38), \ 127 CHECK_READ(0x39)); 128 129 #else /* CHECK_POINTS */ 130 131 #define CHECK_INIT(D) 132 #define CHECK_PRINT(S) 133 #define CHECK_WRITE(A, D) 134 135 #endif /* CHECK_POINTS */ 136 137 /* 138 * Local data and functions. 139 */ 140 141 static void install_ap_tramp(void); 142 static int start_all_aps(void); 143 static int start_ap(int apic_id); 144 145 static char *ap_copyout_buf; 146 static char *ap_tramp_stack_base; 147 148 unsigned int boot_address; 149 150 #define MiB(v) (v ## ULL << 20) 151 152 /* Allocate memory for the AP trampoline. */ 153 void 154 alloc_ap_trampoline(vm_paddr_t *physmap, unsigned int *physmap_idx) 155 { 156 unsigned int i; 157 bool allocated; 158 159 allocated = false; 160 for (i = *physmap_idx; i <= *physmap_idx; i -= 2) { 161 /* 162 * Find a memory region big enough and below the 1MB boundary 163 * for the trampoline code. 164 * NB: needs to be page aligned. 165 */ 166 if (physmap[i] >= MiB(1) || 167 (trunc_page(physmap[i + 1]) - round_page(physmap[i])) < 168 round_page(bootMP_size)) 169 continue; 170 171 allocated = true; 172 /* 173 * Try to steal from the end of the region to mimic previous 174 * behaviour, else fallback to steal from the start. 175 */ 176 if (physmap[i + 1] < MiB(1)) { 177 boot_address = trunc_page(physmap[i + 1]); 178 if ((physmap[i + 1] - boot_address) < bootMP_size) 179 boot_address -= round_page(bootMP_size); 180 physmap[i + 1] = boot_address; 181 } else { 182 boot_address = round_page(physmap[i]); 183 physmap[i] = boot_address + round_page(bootMP_size); 184 } 185 if (physmap[i] == physmap[i + 1] && *physmap_idx != 0) { 186 memmove(&physmap[i], &physmap[i + 2], 187 sizeof(*physmap) * (*physmap_idx - i + 2)); 188 *physmap_idx -= 2; 189 } 190 break; 191 } 192 193 if (!allocated) { 194 boot_address = basemem * 1024 - bootMP_size; 195 if (bootverbose) 196 printf( 197 "Cannot find enough space for the boot trampoline, placing it at %#x", 198 boot_address); 199 } 200 } 201 202 /* 203 * Initialize the IPI handlers and start up the AP's. 204 */ 205 void 206 cpu_mp_start(void) 207 { 208 int i; 209 210 /* Initialize the logical ID to APIC ID table. */ 211 for (i = 0; i < MAXCPU; i++) { 212 cpu_apic_ids[i] = -1; 213 } 214 215 /* Install an inter-CPU IPI for TLB invalidation */ 216 setidt(IPI_INVLTLB, IDTVEC(invltlb), 217 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 218 setidt(IPI_INVLPG, IDTVEC(invlpg), 219 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 220 setidt(IPI_INVLRNG, IDTVEC(invlrng), 221 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 222 223 /* Install an inter-CPU IPI for cache invalidation. */ 224 setidt(IPI_INVLCACHE, IDTVEC(invlcache), 225 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 226 227 /* Install an inter-CPU IPI for all-CPU rendezvous */ 228 setidt(IPI_RENDEZVOUS, IDTVEC(rendezvous), 229 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 230 231 /* Install generic inter-CPU IPI handler */ 232 setidt(IPI_BITMAP_VECTOR, IDTVEC(ipi_intr_bitmap_handler), 233 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 234 235 /* Install an inter-CPU IPI for CPU stop/restart */ 236 setidt(IPI_STOP, IDTVEC(cpustop), 237 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 238 239 /* Install an inter-CPU IPI for CPU suspend/resume */ 240 setidt(IPI_SUSPEND, IDTVEC(cpususpend), 241 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 242 243 /* Install an IPI for calling delayed SWI */ 244 setidt(IPI_SWI, IDTVEC(ipi_swi), 245 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 246 247 /* Set boot_cpu_id if needed. */ 248 if (boot_cpu_id == -1) { 249 boot_cpu_id = PCPU_GET(apic_id); 250 cpu_info[boot_cpu_id].cpu_bsp = 1; 251 } else 252 KASSERT(boot_cpu_id == PCPU_GET(apic_id), 253 ("BSP's APIC ID doesn't match boot_cpu_id")); 254 255 /* Probe logical/physical core configuration. */ 256 topo_probe(); 257 258 assign_cpu_ids(); 259 260 /* Start each Application Processor */ 261 start_all_aps(); 262 263 set_interrupt_apic_ids(); 264 265 #if defined(DEV_ACPI) && MAXMEMDOM > 1 266 acpi_pxm_set_cpu_locality(); 267 #endif 268 } 269 270 /* 271 * AP CPU's call this to initialize themselves. 272 */ 273 void 274 init_secondary(void) 275 { 276 struct pcpu *pc; 277 struct i386tss *common_tssp; 278 struct region_descriptor r_gdt, r_idt; 279 int gsel_tss, myid, x; 280 u_int cr0; 281 282 /* bootAP is set in start_ap() to our ID. */ 283 myid = bootAP; 284 285 /* Update microcode before doing anything else. */ 286 ucode_load_ap(myid); 287 288 /* Get per-cpu data */ 289 pc = &__pcpu[myid]; 290 291 /* prime data page for it to use */ 292 pcpu_init(pc, myid, sizeof(struct pcpu)); 293 dpcpu_init(dpcpu, myid); 294 pc->pc_apic_id = cpu_apic_ids[myid]; 295 pc->pc_prvspace = pc; 296 pc->pc_curthread = 0; 297 pc->pc_common_tssp = common_tssp = &(__pcpu[0].pc_common_tssp)[myid]; 298 299 fix_cpuid(); 300 301 gdt_segs[GPRIV_SEL].ssd_base = (int)pc; 302 gdt_segs[GPROC0_SEL].ssd_base = (int)common_tssp; 303 gdt_segs[GLDT_SEL].ssd_base = (int)ldt; 304 305 for (x = 0; x < NGDT; x++) { 306 ssdtosd(&gdt_segs[x], &gdt[myid * NGDT + x].sd); 307 } 308 309 r_gdt.rd_limit = NGDT * sizeof(gdt[0]) - 1; 310 r_gdt.rd_base = (int) &gdt[myid * NGDT]; 311 lgdt(&r_gdt); /* does magic intra-segment return */ 312 313 r_idt.rd_limit = sizeof(struct gate_descriptor) * NIDT - 1; 314 r_idt.rd_base = (int)idt; 315 lidt(&r_idt); 316 317 lldt(_default_ldt); 318 PCPU_SET(currentldt, _default_ldt); 319 320 PCPU_SET(trampstk, (uintptr_t)ap_tramp_stack_base + TRAMP_STACK_SZ - 321 VM86_STACK_SPACE); 322 323 gsel_tss = GSEL(GPROC0_SEL, SEL_KPL); 324 gdt[myid * NGDT + GPROC0_SEL].sd.sd_type = SDT_SYS386TSS; 325 common_tssp->tss_esp0 = PCPU_GET(trampstk); 326 common_tssp->tss_ss0 = GSEL(GDATA_SEL, SEL_KPL); 327 common_tssp->tss_ioopt = sizeof(struct i386tss) << 16; 328 PCPU_SET(tss_gdt, &gdt[myid * NGDT + GPROC0_SEL].sd); 329 PCPU_SET(common_tssd, *PCPU_GET(tss_gdt)); 330 ltr(gsel_tss); 331 332 PCPU_SET(fsgs_gdt, &gdt[myid * NGDT + GUFS_SEL].sd); 333 PCPU_SET(copyout_buf, ap_copyout_buf); 334 335 /* 336 * Set to a known state: 337 * Set by mpboot.s: CR0_PG, CR0_PE 338 * Set by cpu_setregs: CR0_NE, CR0_MP, CR0_TS, CR0_WP, CR0_AM 339 */ 340 cr0 = rcr0(); 341 cr0 &= ~(CR0_CD | CR0_NW | CR0_EM); 342 load_cr0(cr0); 343 CHECK_WRITE(0x38, 5); 344 345 /* signal our startup to the BSP. */ 346 mp_naps++; 347 CHECK_WRITE(0x39, 6); 348 349 /* Spin until the BSP releases the AP's. */ 350 while (atomic_load_acq_int(&aps_ready) == 0) 351 ia32_pause(); 352 353 /* BSP may have changed PTD while we were waiting */ 354 invltlb(); 355 356 #if defined(I586_CPU) && !defined(NO_F00F_HACK) 357 lidt(&r_idt); 358 #endif 359 360 init_secondary_tail(); 361 } 362 363 /* 364 * start each AP in our list 365 */ 366 #define TMPMAP_START 1 367 static int 368 start_all_aps(void) 369 { 370 u_char mpbiosreason; 371 u_int32_t mpbioswarmvec; 372 int apic_id, cpu; 373 374 mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN); 375 376 pmap_remap_lower(true); 377 378 /* install the AP 1st level boot code */ 379 install_ap_tramp(); 380 381 /* save the current value of the warm-start vector */ 382 mpbioswarmvec = *((u_int32_t *) WARMBOOT_OFF); 383 outb(CMOS_REG, BIOS_RESET); 384 mpbiosreason = inb(CMOS_DATA); 385 386 /* take advantage of the P==V mapping for PTD[0] for AP boot */ 387 388 /* start each AP */ 389 for (cpu = 1; cpu < mp_ncpus; cpu++) { 390 apic_id = cpu_apic_ids[cpu]; 391 392 /* allocate and set up a boot stack data page */ 393 bootstacks[cpu] = kmem_malloc(kstack_pages * PAGE_SIZE, 394 M_WAITOK | M_ZERO); 395 dpcpu = kmem_malloc(DPCPU_SIZE, M_WAITOK | M_ZERO); 396 /* setup a vector to our boot code */ 397 *((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET; 398 *((volatile u_short *) WARMBOOT_SEG) = (boot_address >> 4); 399 outb(CMOS_REG, BIOS_RESET); 400 outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */ 401 402 bootSTK = (char *)bootstacks[cpu] + kstack_pages * 403 PAGE_SIZE - 4; 404 bootAP = cpu; 405 406 ap_tramp_stack_base = pmap_trm_alloc(TRAMP_STACK_SZ, M_NOWAIT); 407 ap_copyout_buf = pmap_trm_alloc(TRAMP_COPYOUT_SZ, M_NOWAIT); 408 409 /* attempt to start the Application Processor */ 410 CHECK_INIT(99); /* setup checkpoints */ 411 if (!start_ap(apic_id)) { 412 printf("AP #%d (PHY# %d) failed!\n", cpu, apic_id); 413 CHECK_PRINT("trace"); /* show checkpoints */ 414 /* better panic as the AP may be running loose */ 415 printf("panic y/n? [y] "); 416 if (cngetc() != 'n') 417 panic("bye-bye"); 418 } 419 CHECK_PRINT("trace"); /* show checkpoints */ 420 421 CPU_SET(cpu, &all_cpus); /* record AP in CPU map */ 422 } 423 424 pmap_remap_lower(false); 425 426 /* restore the warmstart vector */ 427 *(u_int32_t *) WARMBOOT_OFF = mpbioswarmvec; 428 429 outb(CMOS_REG, BIOS_RESET); 430 outb(CMOS_DATA, mpbiosreason); 431 432 /* number of APs actually started */ 433 return mp_naps; 434 } 435 436 /* 437 * load the 1st level AP boot code into base memory. 438 */ 439 440 /* targets for relocation */ 441 extern void bigJump(void); 442 extern void bootCodeSeg(void); 443 extern void bootDataSeg(void); 444 extern void MPentry(void); 445 extern u_int MP_GDT; 446 extern u_int mp_gdtbase; 447 448 static void 449 install_ap_tramp(void) 450 { 451 int x; 452 int size = *(int *) ((u_long) & bootMP_size); 453 vm_offset_t va = boot_address; 454 u_char *src = (u_char *) ((u_long) bootMP); 455 u_char *dst = (u_char *) va; 456 u_int boot_base = (u_int) bootMP; 457 u_int8_t *dst8; 458 u_int16_t *dst16; 459 u_int32_t *dst32; 460 461 KASSERT (size <= PAGE_SIZE, 462 ("'size' do not fit into PAGE_SIZE, as expected.")); 463 pmap_kenter(va, boot_address); 464 pmap_invalidate_page (kernel_pmap, va); 465 for (x = 0; x < size; ++x) 466 *dst++ = *src++; 467 468 /* 469 * modify addresses in code we just moved to basemem. unfortunately we 470 * need fairly detailed info about mpboot.s for this to work. changes 471 * to mpboot.s might require changes here. 472 */ 473 474 /* boot code is located in KERNEL space */ 475 dst = (u_char *) va; 476 477 /* modify the lgdt arg */ 478 dst32 = (u_int32_t *) (dst + ((u_int) & mp_gdtbase - boot_base)); 479 *dst32 = boot_address + ((u_int) & MP_GDT - boot_base); 480 481 /* modify the ljmp target for MPentry() */ 482 dst32 = (u_int32_t *) (dst + ((u_int) bigJump - boot_base) + 1); 483 *dst32 = (u_int)MPentry; 484 485 /* modify the target for boot code segment */ 486 dst16 = (u_int16_t *) (dst + ((u_int) bootCodeSeg - boot_base)); 487 dst8 = (u_int8_t *) (dst16 + 1); 488 *dst16 = (u_int) boot_address & 0xffff; 489 *dst8 = ((u_int) boot_address >> 16) & 0xff; 490 491 /* modify the target for boot data segment */ 492 dst16 = (u_int16_t *) (dst + ((u_int) bootDataSeg - boot_base)); 493 dst8 = (u_int8_t *) (dst16 + 1); 494 *dst16 = (u_int) boot_address & 0xffff; 495 *dst8 = ((u_int) boot_address >> 16) & 0xff; 496 } 497 498 /* 499 * This function starts the AP (application processor) identified 500 * by the APIC ID 'physicalCpu'. It does quite a "song and dance" 501 * to accomplish this. This is necessary because of the nuances 502 * of the different hardware we might encounter. It isn't pretty, 503 * but it seems to work. 504 */ 505 static int 506 start_ap(int apic_id) 507 { 508 int vector, ms; 509 int cpus; 510 511 /* calculate the vector */ 512 vector = (boot_address >> 12) & 0xff; 513 514 /* used as a watchpoint to signal AP startup */ 515 cpus = mp_naps; 516 517 ipi_startup(apic_id, vector); 518 519 /* Wait up to 5 seconds for it to start. */ 520 for (ms = 0; ms < 5000; ms++) { 521 if (mp_naps > cpus) 522 return 1; /* return SUCCESS */ 523 DELAY(1000); 524 } 525 return 0; /* return FAILURE */ 526 } 527 528 /* 529 * Flush the TLB on other CPU's 530 */ 531 532 /* Variables needed for SMP tlb shootdown. */ 533 vm_offset_t smp_tlb_addr1, smp_tlb_addr2; 534 pmap_t smp_tlb_pmap; 535 volatile uint32_t smp_tlb_generation; 536 537 /* 538 * Used by pmap to request cache or TLB invalidation on local and 539 * remote processors. Mask provides the set of remote CPUs which are 540 * to be signalled with the invalidation IPI. Vector specifies which 541 * invalidation IPI is used. As an optimization, the curcpu_cb 542 * callback is invoked on the calling CPU while waiting for remote 543 * CPUs to complete the operation. 544 * 545 * The callback function is called unconditionally on the caller's 546 * underlying processor, even when this processor is not set in the 547 * mask. So, the callback function must be prepared to handle such 548 * spurious invocations. 549 */ 550 static void 551 smp_targeted_tlb_shootdown(cpuset_t mask, u_int vector, pmap_t pmap, 552 vm_offset_t addr1, vm_offset_t addr2, smp_invl_cb_t curcpu_cb) 553 { 554 cpuset_t other_cpus; 555 volatile uint32_t *p_cpudone; 556 uint32_t generation; 557 int cpu; 558 559 /* 560 * It is not necessary to signal other CPUs while booting or 561 * when in the debugger. 562 */ 563 if (kdb_active || KERNEL_PANICKED() || !smp_started) { 564 curcpu_cb(pmap, addr1, addr2); 565 return; 566 } 567 568 sched_pin(); 569 570 /* 571 * Check for other cpus. Return if none. 572 */ 573 if (CPU_ISFULLSET(&mask)) { 574 if (mp_ncpus <= 1) 575 goto nospinexit; 576 } else { 577 CPU_CLR(PCPU_GET(cpuid), &mask); 578 if (CPU_EMPTY(&mask)) 579 goto nospinexit; 580 } 581 582 KASSERT((read_eflags() & PSL_I) != 0, 583 ("smp_targeted_tlb_shootdown: interrupts disabled")); 584 mtx_lock_spin(&smp_ipi_mtx); 585 smp_tlb_addr1 = addr1; 586 smp_tlb_addr2 = addr2; 587 smp_tlb_pmap = pmap; 588 generation = ++smp_tlb_generation; 589 if (CPU_ISFULLSET(&mask)) { 590 ipi_all_but_self(vector); 591 other_cpus = all_cpus; 592 CPU_CLR(PCPU_GET(cpuid), &other_cpus); 593 } else { 594 other_cpus = mask; 595 ipi_selected(mask, vector); 596 } 597 curcpu_cb(pmap, addr1, addr2); 598 CPU_FOREACH_ISSET(cpu, &other_cpus) { 599 p_cpudone = &cpuid_to_pcpu[cpu]->pc_smp_tlb_done; 600 while (*p_cpudone != generation) 601 ia32_pause(); 602 } 603 mtx_unlock_spin(&smp_ipi_mtx); 604 sched_unpin(); 605 return; 606 607 nospinexit: 608 curcpu_cb(pmap, addr1, addr2); 609 sched_unpin(); 610 } 611 612 void 613 smp_masked_invltlb(cpuset_t mask, pmap_t pmap, smp_invl_cb_t curcpu_cb) 614 { 615 smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, pmap, 0, 0, curcpu_cb); 616 #ifdef COUNT_XINVLTLB_HITS 617 ipi_global++; 618 #endif 619 } 620 621 void 622 smp_masked_invlpg(cpuset_t mask, vm_offset_t addr, pmap_t pmap, 623 smp_invl_cb_t curcpu_cb) 624 { 625 smp_targeted_tlb_shootdown(mask, IPI_INVLPG, pmap, addr, 0, curcpu_cb); 626 #ifdef COUNT_XINVLTLB_HITS 627 ipi_page++; 628 #endif 629 } 630 631 void 632 smp_masked_invlpg_range(cpuset_t mask, vm_offset_t addr1, vm_offset_t addr2, 633 pmap_t pmap, smp_invl_cb_t curcpu_cb) 634 { 635 smp_targeted_tlb_shootdown(mask, IPI_INVLRNG, pmap, addr1, addr2, 636 curcpu_cb); 637 #ifdef COUNT_XINVLTLB_HITS 638 ipi_range++; 639 ipi_range_size += (addr2 - addr1) / PAGE_SIZE; 640 #endif 641 } 642 643 void 644 smp_cache_flush(smp_invl_cb_t curcpu_cb) 645 { 646 smp_targeted_tlb_shootdown(all_cpus, IPI_INVLCACHE, NULL, 0, 0, 647 curcpu_cb); 648 } 649 650 /* 651 * Handlers for TLB related IPIs 652 */ 653 void 654 invltlb_handler(void) 655 { 656 uint32_t generation; 657 658 trap_check_kstack(); 659 #ifdef COUNT_XINVLTLB_HITS 660 xhits_gbl[PCPU_GET(cpuid)]++; 661 #endif /* COUNT_XINVLTLB_HITS */ 662 #ifdef COUNT_IPIS 663 (*ipi_invltlb_counts[PCPU_GET(cpuid)])++; 664 #endif /* COUNT_IPIS */ 665 666 /* 667 * Reading the generation here allows greater parallelism 668 * since invalidating the TLB is a serializing operation. 669 */ 670 generation = smp_tlb_generation; 671 if (smp_tlb_pmap == kernel_pmap) 672 invltlb_glob(); 673 PCPU_SET(smp_tlb_done, generation); 674 } 675 676 void 677 invlpg_handler(void) 678 { 679 uint32_t generation; 680 681 trap_check_kstack(); 682 #ifdef COUNT_XINVLTLB_HITS 683 xhits_pg[PCPU_GET(cpuid)]++; 684 #endif /* COUNT_XINVLTLB_HITS */ 685 #ifdef COUNT_IPIS 686 (*ipi_invlpg_counts[PCPU_GET(cpuid)])++; 687 #endif /* COUNT_IPIS */ 688 689 generation = smp_tlb_generation; /* Overlap with serialization */ 690 if (smp_tlb_pmap == kernel_pmap) 691 invlpg(smp_tlb_addr1); 692 PCPU_SET(smp_tlb_done, generation); 693 } 694 695 void 696 invlrng_handler(void) 697 { 698 vm_offset_t addr, addr2; 699 uint32_t generation; 700 701 trap_check_kstack(); 702 #ifdef COUNT_XINVLTLB_HITS 703 xhits_rng[PCPU_GET(cpuid)]++; 704 #endif /* COUNT_XINVLTLB_HITS */ 705 #ifdef COUNT_IPIS 706 (*ipi_invlrng_counts[PCPU_GET(cpuid)])++; 707 #endif /* COUNT_IPIS */ 708 709 addr = smp_tlb_addr1; 710 addr2 = smp_tlb_addr2; 711 generation = smp_tlb_generation; /* Overlap with serialization */ 712 if (smp_tlb_pmap == kernel_pmap) { 713 do { 714 invlpg(addr); 715 addr += PAGE_SIZE; 716 } while (addr < addr2); 717 } 718 719 PCPU_SET(smp_tlb_done, generation); 720 } 721 722 void 723 invlcache_handler(void) 724 { 725 uint32_t generation; 726 727 trap_check_kstack(); 728 #ifdef COUNT_IPIS 729 (*ipi_invlcache_counts[PCPU_GET(cpuid)])++; 730 #endif /* COUNT_IPIS */ 731 732 /* 733 * Reading the generation here allows greater parallelism 734 * since wbinvd is a serializing instruction. Without the 735 * temporary, we'd wait for wbinvd to complete, then the read 736 * would execute, then the dependent write, which must then 737 * complete before return from interrupt. 738 */ 739 generation = smp_tlb_generation; 740 wbinvd(); 741 PCPU_SET(smp_tlb_done, generation); 742 } 743