1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <linux/kernel.h> 3 #include <linux/kvm_host.h> 4 #include <asm/asm-prototypes.h> 5 #include <asm/dbell.h> 6 #include <asm/kvm_ppc.h> 7 #include <asm/pmc.h> 8 #include <asm/ppc-opcode.h> 9 10 #include "book3s_hv.h" 11 12 static void freeze_pmu(unsigned long mmcr0, unsigned long mmcra) 13 { 14 if (!(mmcr0 & MMCR0_FC)) 15 goto do_freeze; 16 if (mmcra & MMCRA_SAMPLE_ENABLE) 17 goto do_freeze; 18 if (cpu_has_feature(CPU_FTR_ARCH_31)) { 19 if (!(mmcr0 & MMCR0_PMCCEXT)) 20 goto do_freeze; 21 if (!(mmcra & MMCRA_BHRB_DISABLE)) 22 goto do_freeze; 23 } 24 return; 25 26 do_freeze: 27 mmcr0 = MMCR0_FC; 28 mmcra = 0; 29 if (cpu_has_feature(CPU_FTR_ARCH_31)) { 30 mmcr0 |= MMCR0_PMCCEXT; 31 mmcra = MMCRA_BHRB_DISABLE; 32 } 33 34 mtspr(SPRN_MMCR0, mmcr0); 35 mtspr(SPRN_MMCRA, mmcra); 36 isync(); 37 } 38 39 void switch_pmu_to_guest(struct kvm_vcpu *vcpu, 40 struct p9_host_os_sprs *host_os_sprs) 41 { 42 struct lppaca *lp; 43 int load_pmu = 1; 44 45 lp = vcpu->arch.vpa.pinned_addr; 46 if (lp) 47 load_pmu = lp->pmcregs_in_use; 48 49 /* Save host */ 50 if (ppc_get_pmu_inuse()) { 51 /* 52 * It might be better to put PMU handling (at least for the 53 * host) in the perf subsystem because it knows more about what 54 * is being used. 55 */ 56 57 /* POWER9, POWER10 do not implement HPMC or SPMC */ 58 59 host_os_sprs->mmcr0 = mfspr(SPRN_MMCR0); 60 host_os_sprs->mmcra = mfspr(SPRN_MMCRA); 61 62 freeze_pmu(host_os_sprs->mmcr0, host_os_sprs->mmcra); 63 64 host_os_sprs->pmc1 = mfspr(SPRN_PMC1); 65 host_os_sprs->pmc2 = mfspr(SPRN_PMC2); 66 host_os_sprs->pmc3 = mfspr(SPRN_PMC3); 67 host_os_sprs->pmc4 = mfspr(SPRN_PMC4); 68 host_os_sprs->pmc5 = mfspr(SPRN_PMC5); 69 host_os_sprs->pmc6 = mfspr(SPRN_PMC6); 70 host_os_sprs->mmcr1 = mfspr(SPRN_MMCR1); 71 host_os_sprs->mmcr2 = mfspr(SPRN_MMCR2); 72 host_os_sprs->sdar = mfspr(SPRN_SDAR); 73 host_os_sprs->siar = mfspr(SPRN_SIAR); 74 host_os_sprs->sier1 = mfspr(SPRN_SIER); 75 76 if (cpu_has_feature(CPU_FTR_ARCH_31)) { 77 host_os_sprs->mmcr3 = mfspr(SPRN_MMCR3); 78 host_os_sprs->sier2 = mfspr(SPRN_SIER2); 79 host_os_sprs->sier3 = mfspr(SPRN_SIER3); 80 } 81 } 82 83 #ifdef CONFIG_PPC_PSERIES 84 /* After saving PMU, before loading guest PMU, flip pmcregs_in_use */ 85 if (kvmhv_on_pseries()) { 86 barrier(); 87 get_lppaca()->pmcregs_in_use = load_pmu; 88 barrier(); 89 } 90 #endif 91 92 /* 93 * Load guest. If the VPA said the PMCs are not in use but the guest 94 * tried to access them anyway, HFSCR[PM] will be set by the HFAC 95 * fault so we can make forward progress. 96 */ 97 if (load_pmu || (vcpu->arch.hfscr & HFSCR_PM)) { 98 mtspr(SPRN_PMC1, vcpu->arch.pmc[0]); 99 mtspr(SPRN_PMC2, vcpu->arch.pmc[1]); 100 mtspr(SPRN_PMC3, vcpu->arch.pmc[2]); 101 mtspr(SPRN_PMC4, vcpu->arch.pmc[3]); 102 mtspr(SPRN_PMC5, vcpu->arch.pmc[4]); 103 mtspr(SPRN_PMC6, vcpu->arch.pmc[5]); 104 mtspr(SPRN_MMCR1, vcpu->arch.mmcr[1]); 105 mtspr(SPRN_MMCR2, vcpu->arch.mmcr[2]); 106 mtspr(SPRN_SDAR, vcpu->arch.sdar); 107 mtspr(SPRN_SIAR, vcpu->arch.siar); 108 mtspr(SPRN_SIER, vcpu->arch.sier[0]); 109 110 if (cpu_has_feature(CPU_FTR_ARCH_31)) { 111 mtspr(SPRN_MMCR3, vcpu->arch.mmcr[3]); 112 mtspr(SPRN_SIER2, vcpu->arch.sier[1]); 113 mtspr(SPRN_SIER3, vcpu->arch.sier[2]); 114 } 115 116 /* Set MMCRA then MMCR0 last */ 117 mtspr(SPRN_MMCRA, vcpu->arch.mmcra); 118 mtspr(SPRN_MMCR0, vcpu->arch.mmcr[0]); 119 /* No isync necessary because we're starting counters */ 120 121 if (!vcpu->arch.nested && 122 (vcpu->arch.hfscr_permitted & HFSCR_PM)) 123 vcpu->arch.hfscr |= HFSCR_PM; 124 } 125 } 126 EXPORT_SYMBOL_GPL(switch_pmu_to_guest); 127 128 void switch_pmu_to_host(struct kvm_vcpu *vcpu, 129 struct p9_host_os_sprs *host_os_sprs) 130 { 131 struct lppaca *lp; 132 int save_pmu = 1; 133 134 lp = vcpu->arch.vpa.pinned_addr; 135 if (lp) 136 save_pmu = lp->pmcregs_in_use; 137 if (IS_ENABLED(CONFIG_KVM_BOOK3S_HV_NESTED_PMU_WORKAROUND)) { 138 /* 139 * Save pmu if this guest is capable of running nested guests. 140 * This is option is for old L1s that do not set their 141 * lppaca->pmcregs_in_use properly when entering their L2. 142 */ 143 save_pmu |= nesting_enabled(vcpu->kvm); 144 } 145 146 if (save_pmu) { 147 vcpu->arch.mmcr[0] = mfspr(SPRN_MMCR0); 148 vcpu->arch.mmcra = mfspr(SPRN_MMCRA); 149 150 freeze_pmu(vcpu->arch.mmcr[0], vcpu->arch.mmcra); 151 152 vcpu->arch.pmc[0] = mfspr(SPRN_PMC1); 153 vcpu->arch.pmc[1] = mfspr(SPRN_PMC2); 154 vcpu->arch.pmc[2] = mfspr(SPRN_PMC3); 155 vcpu->arch.pmc[3] = mfspr(SPRN_PMC4); 156 vcpu->arch.pmc[4] = mfspr(SPRN_PMC5); 157 vcpu->arch.pmc[5] = mfspr(SPRN_PMC6); 158 vcpu->arch.mmcr[1] = mfspr(SPRN_MMCR1); 159 vcpu->arch.mmcr[2] = mfspr(SPRN_MMCR2); 160 vcpu->arch.sdar = mfspr(SPRN_SDAR); 161 vcpu->arch.siar = mfspr(SPRN_SIAR); 162 vcpu->arch.sier[0] = mfspr(SPRN_SIER); 163 164 if (cpu_has_feature(CPU_FTR_ARCH_31)) { 165 vcpu->arch.mmcr[3] = mfspr(SPRN_MMCR3); 166 vcpu->arch.sier[1] = mfspr(SPRN_SIER2); 167 vcpu->arch.sier[2] = mfspr(SPRN_SIER3); 168 } 169 170 } else if (vcpu->arch.hfscr & HFSCR_PM) { 171 /* 172 * The guest accessed PMC SPRs without specifying they should 173 * be preserved, or it cleared pmcregs_in_use after the last 174 * access. Just ensure they are frozen. 175 */ 176 freeze_pmu(mfspr(SPRN_MMCR0), mfspr(SPRN_MMCRA)); 177 178 /* 179 * Demand-fault PMU register access in the guest. 180 * 181 * This is used to grab the guest's VPA pmcregs_in_use value 182 * and reflect it into the host's VPA in the case of a nested 183 * hypervisor. 184 * 185 * It also avoids having to zero-out SPRs after each guest 186 * exit to avoid side-channels when. 187 * 188 * This is cleared here when we exit the guest, so later HFSCR 189 * interrupt handling can add it back to run the guest with 190 * PM enabled next time. 191 */ 192 if (!vcpu->arch.nested) 193 vcpu->arch.hfscr &= ~HFSCR_PM; 194 } /* otherwise the PMU should still be frozen */ 195 196 #ifdef CONFIG_PPC_PSERIES 197 if (kvmhv_on_pseries()) { 198 barrier(); 199 get_lppaca()->pmcregs_in_use = ppc_get_pmu_inuse(); 200 barrier(); 201 } 202 #endif 203 204 if (ppc_get_pmu_inuse()) { 205 mtspr(SPRN_PMC1, host_os_sprs->pmc1); 206 mtspr(SPRN_PMC2, host_os_sprs->pmc2); 207 mtspr(SPRN_PMC3, host_os_sprs->pmc3); 208 mtspr(SPRN_PMC4, host_os_sprs->pmc4); 209 mtspr(SPRN_PMC5, host_os_sprs->pmc5); 210 mtspr(SPRN_PMC6, host_os_sprs->pmc6); 211 mtspr(SPRN_MMCR1, host_os_sprs->mmcr1); 212 mtspr(SPRN_MMCR2, host_os_sprs->mmcr2); 213 mtspr(SPRN_SDAR, host_os_sprs->sdar); 214 mtspr(SPRN_SIAR, host_os_sprs->siar); 215 mtspr(SPRN_SIER, host_os_sprs->sier1); 216 217 if (cpu_has_feature(CPU_FTR_ARCH_31)) { 218 mtspr(SPRN_MMCR3, host_os_sprs->mmcr3); 219 mtspr(SPRN_SIER2, host_os_sprs->sier2); 220 mtspr(SPRN_SIER3, host_os_sprs->sier3); 221 } 222 223 /* Set MMCRA then MMCR0 last */ 224 mtspr(SPRN_MMCRA, host_os_sprs->mmcra); 225 mtspr(SPRN_MMCR0, host_os_sprs->mmcr0); 226 isync(); 227 } 228 } 229 EXPORT_SYMBOL_GPL(switch_pmu_to_host); 230 231 static void load_spr_state(struct kvm_vcpu *vcpu, 232 struct p9_host_os_sprs *host_os_sprs) 233 { 234 /* TAR is very fast */ 235 mtspr(SPRN_TAR, vcpu->arch.tar); 236 237 #ifdef CONFIG_ALTIVEC 238 if (cpu_has_feature(CPU_FTR_ALTIVEC) && 239 current->thread.vrsave != vcpu->arch.vrsave) 240 mtspr(SPRN_VRSAVE, vcpu->arch.vrsave); 241 #endif 242 243 if (vcpu->arch.hfscr & HFSCR_EBB) { 244 if (current->thread.ebbhr != vcpu->arch.ebbhr) 245 mtspr(SPRN_EBBHR, vcpu->arch.ebbhr); 246 if (current->thread.ebbrr != vcpu->arch.ebbrr) 247 mtspr(SPRN_EBBRR, vcpu->arch.ebbrr); 248 if (current->thread.bescr != vcpu->arch.bescr) 249 mtspr(SPRN_BESCR, vcpu->arch.bescr); 250 } 251 252 if (cpu_has_feature(CPU_FTR_P9_TIDR) && 253 current->thread.tidr != vcpu->arch.tid) 254 mtspr(SPRN_TIDR, vcpu->arch.tid); 255 if (host_os_sprs->iamr != vcpu->arch.iamr) 256 mtspr(SPRN_IAMR, vcpu->arch.iamr); 257 if (host_os_sprs->amr != vcpu->arch.amr) 258 mtspr(SPRN_AMR, vcpu->arch.amr); 259 if (vcpu->arch.uamor != 0) 260 mtspr(SPRN_UAMOR, vcpu->arch.uamor); 261 if (current->thread.fscr != vcpu->arch.fscr) 262 mtspr(SPRN_FSCR, vcpu->arch.fscr); 263 if (current->thread.dscr != vcpu->arch.dscr) 264 mtspr(SPRN_DSCR, vcpu->arch.dscr); 265 if (vcpu->arch.pspb != 0) 266 mtspr(SPRN_PSPB, vcpu->arch.pspb); 267 268 /* 269 * DAR, DSISR, and for nested HV, SPRGs must be set with MSR[RI] 270 * clear (or hstate set appropriately to catch those registers 271 * being clobbered if we take a MCE or SRESET), so those are done 272 * later. 273 */ 274 275 if (!(vcpu->arch.ctrl & 1)) 276 mtspr(SPRN_CTRLT, 0); 277 } 278 279 static void store_spr_state(struct kvm_vcpu *vcpu) 280 { 281 vcpu->arch.tar = mfspr(SPRN_TAR); 282 283 #ifdef CONFIG_ALTIVEC 284 if (cpu_has_feature(CPU_FTR_ALTIVEC)) 285 vcpu->arch.vrsave = mfspr(SPRN_VRSAVE); 286 #endif 287 288 if (vcpu->arch.hfscr & HFSCR_EBB) { 289 vcpu->arch.ebbhr = mfspr(SPRN_EBBHR); 290 vcpu->arch.ebbrr = mfspr(SPRN_EBBRR); 291 vcpu->arch.bescr = mfspr(SPRN_BESCR); 292 } 293 294 if (cpu_has_feature(CPU_FTR_P9_TIDR)) 295 vcpu->arch.tid = mfspr(SPRN_TIDR); 296 vcpu->arch.iamr = mfspr(SPRN_IAMR); 297 vcpu->arch.amr = mfspr(SPRN_AMR); 298 vcpu->arch.uamor = mfspr(SPRN_UAMOR); 299 vcpu->arch.fscr = mfspr(SPRN_FSCR); 300 vcpu->arch.dscr = mfspr(SPRN_DSCR); 301 vcpu->arch.pspb = mfspr(SPRN_PSPB); 302 303 vcpu->arch.ctrl = mfspr(SPRN_CTRLF); 304 } 305 306 /* Returns true if current MSR and/or guest MSR may have changed */ 307 bool load_vcpu_state(struct kvm_vcpu *vcpu, 308 struct p9_host_os_sprs *host_os_sprs) 309 { 310 bool ret = false; 311 312 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 313 if (cpu_has_feature(CPU_FTR_TM) || 314 cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) { 315 unsigned long guest_msr = vcpu->arch.shregs.msr; 316 if (MSR_TM_ACTIVE(guest_msr)) { 317 kvmppc_restore_tm_hv(vcpu, guest_msr, true); 318 ret = true; 319 } else if (vcpu->arch.hfscr & HFSCR_TM) { 320 mtspr(SPRN_TEXASR, vcpu->arch.texasr); 321 mtspr(SPRN_TFHAR, vcpu->arch.tfhar); 322 mtspr(SPRN_TFIAR, vcpu->arch.tfiar); 323 } 324 } 325 #endif 326 327 load_spr_state(vcpu, host_os_sprs); 328 329 load_fp_state(&vcpu->arch.fp); 330 #ifdef CONFIG_ALTIVEC 331 load_vr_state(&vcpu->arch.vr); 332 #endif 333 334 return ret; 335 } 336 EXPORT_SYMBOL_GPL(load_vcpu_state); 337 338 void store_vcpu_state(struct kvm_vcpu *vcpu) 339 { 340 store_spr_state(vcpu); 341 342 store_fp_state(&vcpu->arch.fp); 343 #ifdef CONFIG_ALTIVEC 344 store_vr_state(&vcpu->arch.vr); 345 #endif 346 347 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 348 if (cpu_has_feature(CPU_FTR_TM) || 349 cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) { 350 unsigned long guest_msr = vcpu->arch.shregs.msr; 351 if (MSR_TM_ACTIVE(guest_msr)) { 352 kvmppc_save_tm_hv(vcpu, guest_msr, true); 353 } else if (vcpu->arch.hfscr & HFSCR_TM) { 354 vcpu->arch.texasr = mfspr(SPRN_TEXASR); 355 vcpu->arch.tfhar = mfspr(SPRN_TFHAR); 356 vcpu->arch.tfiar = mfspr(SPRN_TFIAR); 357 358 if (!vcpu->arch.nested) { 359 vcpu->arch.load_tm++; /* see load_ebb comment */ 360 if (!vcpu->arch.load_tm) 361 vcpu->arch.hfscr &= ~HFSCR_TM; 362 } 363 } 364 } 365 #endif 366 } 367 EXPORT_SYMBOL_GPL(store_vcpu_state); 368 369 void save_p9_host_os_sprs(struct p9_host_os_sprs *host_os_sprs) 370 { 371 host_os_sprs->iamr = mfspr(SPRN_IAMR); 372 host_os_sprs->amr = mfspr(SPRN_AMR); 373 } 374 EXPORT_SYMBOL_GPL(save_p9_host_os_sprs); 375 376 /* vcpu guest regs must already be saved */ 377 void restore_p9_host_os_sprs(struct kvm_vcpu *vcpu, 378 struct p9_host_os_sprs *host_os_sprs) 379 { 380 /* 381 * current->thread.xxx registers must all be restored to host 382 * values before a potential context switch, othrewise the context 383 * switch itself will overwrite current->thread.xxx with the values 384 * from the guest SPRs. 385 */ 386 387 mtspr(SPRN_SPRG_VDSO_WRITE, local_paca->sprg_vdso); 388 389 if (cpu_has_feature(CPU_FTR_P9_TIDR) && 390 current->thread.tidr != vcpu->arch.tid) 391 mtspr(SPRN_TIDR, current->thread.tidr); 392 if (host_os_sprs->iamr != vcpu->arch.iamr) 393 mtspr(SPRN_IAMR, host_os_sprs->iamr); 394 if (vcpu->arch.uamor != 0) 395 mtspr(SPRN_UAMOR, 0); 396 if (host_os_sprs->amr != vcpu->arch.amr) 397 mtspr(SPRN_AMR, host_os_sprs->amr); 398 if (current->thread.fscr != vcpu->arch.fscr) 399 mtspr(SPRN_FSCR, current->thread.fscr); 400 if (current->thread.dscr != vcpu->arch.dscr) 401 mtspr(SPRN_DSCR, current->thread.dscr); 402 if (vcpu->arch.pspb != 0) 403 mtspr(SPRN_PSPB, 0); 404 405 /* Save guest CTRL register, set runlatch to 1 */ 406 if (!(vcpu->arch.ctrl & 1)) 407 mtspr(SPRN_CTRLT, 1); 408 409 #ifdef CONFIG_ALTIVEC 410 if (cpu_has_feature(CPU_FTR_ALTIVEC) && 411 vcpu->arch.vrsave != current->thread.vrsave) 412 mtspr(SPRN_VRSAVE, current->thread.vrsave); 413 #endif 414 if (vcpu->arch.hfscr & HFSCR_EBB) { 415 if (vcpu->arch.bescr != current->thread.bescr) 416 mtspr(SPRN_BESCR, current->thread.bescr); 417 if (vcpu->arch.ebbhr != current->thread.ebbhr) 418 mtspr(SPRN_EBBHR, current->thread.ebbhr); 419 if (vcpu->arch.ebbrr != current->thread.ebbrr) 420 mtspr(SPRN_EBBRR, current->thread.ebbrr); 421 422 if (!vcpu->arch.nested) { 423 /* 424 * This is like load_fp in context switching, turn off 425 * the facility after it wraps the u8 to try avoiding 426 * saving and restoring the registers each partition 427 * switch. 428 */ 429 vcpu->arch.load_ebb++; 430 if (!vcpu->arch.load_ebb) 431 vcpu->arch.hfscr &= ~HFSCR_EBB; 432 } 433 } 434 435 if (vcpu->arch.tar != current->thread.tar) 436 mtspr(SPRN_TAR, current->thread.tar); 437 } 438 EXPORT_SYMBOL_GPL(restore_p9_host_os_sprs); 439 440 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 441 static void __start_timing(struct kvm_vcpu *vcpu, struct kvmhv_tb_accumulator *next) 442 { 443 struct kvmppc_vcore *vc = vcpu->arch.vcore; 444 u64 tb = mftb() - vc->tb_offset_applied; 445 446 vcpu->arch.cur_activity = next; 447 vcpu->arch.cur_tb_start = tb; 448 } 449 450 static void __accumulate_time(struct kvm_vcpu *vcpu, struct kvmhv_tb_accumulator *next) 451 { 452 struct kvmppc_vcore *vc = vcpu->arch.vcore; 453 struct kvmhv_tb_accumulator *curr; 454 u64 tb = mftb() - vc->tb_offset_applied; 455 u64 prev_tb; 456 u64 delta; 457 u64 seq; 458 459 curr = vcpu->arch.cur_activity; 460 vcpu->arch.cur_activity = next; 461 prev_tb = vcpu->arch.cur_tb_start; 462 vcpu->arch.cur_tb_start = tb; 463 464 if (!curr) 465 return; 466 467 delta = tb - prev_tb; 468 469 seq = curr->seqcount; 470 curr->seqcount = seq + 1; 471 smp_wmb(); 472 curr->tb_total += delta; 473 if (seq == 0 || delta < curr->tb_min) 474 curr->tb_min = delta; 475 if (delta > curr->tb_max) 476 curr->tb_max = delta; 477 smp_wmb(); 478 curr->seqcount = seq + 2; 479 } 480 481 #define start_timing(vcpu, next) __start_timing(vcpu, next) 482 #define end_timing(vcpu) __start_timing(vcpu, NULL) 483 #define accumulate_time(vcpu, next) __accumulate_time(vcpu, next) 484 #else 485 #define start_timing(vcpu, next) do {} while (0) 486 #define end_timing(vcpu) do {} while (0) 487 #define accumulate_time(vcpu, next) do {} while (0) 488 #endif 489 490 static inline u64 mfslbv(unsigned int idx) 491 { 492 u64 slbev; 493 494 asm volatile("slbmfev %0,%1" : "=r" (slbev) : "r" (idx)); 495 496 return slbev; 497 } 498 499 static inline u64 mfslbe(unsigned int idx) 500 { 501 u64 slbee; 502 503 asm volatile("slbmfee %0,%1" : "=r" (slbee) : "r" (idx)); 504 505 return slbee; 506 } 507 508 static inline void mtslb(u64 slbee, u64 slbev) 509 { 510 asm volatile("slbmte %0,%1" :: "r" (slbev), "r" (slbee)); 511 } 512 513 static inline void clear_slb_entry(unsigned int idx) 514 { 515 mtslb(idx, 0); 516 } 517 518 static inline void slb_clear_invalidate_partition(void) 519 { 520 clear_slb_entry(0); 521 asm volatile(PPC_SLBIA(6)); 522 } 523 524 /* 525 * Malicious or buggy radix guests may have inserted SLB entries 526 * (only 0..3 because radix always runs with UPRT=1), so these must 527 * be cleared here to avoid side-channels. slbmte is used rather 528 * than slbia, as it won't clear cached translations. 529 */ 530 static void radix_clear_slb(void) 531 { 532 int i; 533 534 for (i = 0; i < 4; i++) 535 clear_slb_entry(i); 536 } 537 538 static void switch_mmu_to_guest_radix(struct kvm *kvm, struct kvm_vcpu *vcpu, u64 lpcr) 539 { 540 struct kvm_nested_guest *nested = vcpu->arch.nested; 541 u32 lpid; 542 543 lpid = nested ? nested->shadow_lpid : kvm->arch.lpid; 544 545 /* 546 * Prior memory accesses to host PID Q3 must be completed before we 547 * start switching, and stores must be drained to avoid not-my-LPAR 548 * logic (see switch_mmu_to_host). 549 */ 550 asm volatile("hwsync" ::: "memory"); 551 isync(); 552 mtspr(SPRN_LPID, lpid); 553 mtspr(SPRN_LPCR, lpcr); 554 mtspr(SPRN_PID, vcpu->arch.pid); 555 /* 556 * isync not required here because we are HRFID'ing to guest before 557 * any guest context access, which is context synchronising. 558 */ 559 } 560 561 static void switch_mmu_to_guest_hpt(struct kvm *kvm, struct kvm_vcpu *vcpu, u64 lpcr) 562 { 563 u32 lpid; 564 int i; 565 566 lpid = kvm->arch.lpid; 567 568 /* 569 * See switch_mmu_to_guest_radix. ptesync should not be required here 570 * even if the host is in HPT mode because speculative accesses would 571 * not cause RC updates (we are in real mode). 572 */ 573 asm volatile("hwsync" ::: "memory"); 574 isync(); 575 mtspr(SPRN_LPID, lpid); 576 mtspr(SPRN_LPCR, lpcr); 577 mtspr(SPRN_PID, vcpu->arch.pid); 578 579 for (i = 0; i < vcpu->arch.slb_max; i++) 580 mtslb(vcpu->arch.slb[i].orige, vcpu->arch.slb[i].origv); 581 /* 582 * isync not required here, see switch_mmu_to_guest_radix. 583 */ 584 } 585 586 static void switch_mmu_to_host(struct kvm *kvm, u32 pid) 587 { 588 /* 589 * The guest has exited, so guest MMU context is no longer being 590 * non-speculatively accessed, but a hwsync is needed before the 591 * mtLPIDR / mtPIDR switch, in order to ensure all stores are drained, 592 * so the not-my-LPAR tlbie logic does not overlook them. 593 */ 594 asm volatile("hwsync" ::: "memory"); 595 isync(); 596 mtspr(SPRN_PID, pid); 597 mtspr(SPRN_LPID, kvm->arch.host_lpid); 598 mtspr(SPRN_LPCR, kvm->arch.host_lpcr); 599 /* 600 * isync is not required after the switch, because mtmsrd with L=0 601 * is performed after this switch, which is context synchronising. 602 */ 603 604 if (!radix_enabled()) 605 slb_restore_bolted_realmode(); 606 } 607 608 static void save_clear_host_mmu(struct kvm *kvm) 609 { 610 if (!radix_enabled()) { 611 /* 612 * Hash host could save and restore host SLB entries to 613 * reduce SLB fault overheads of VM exits, but for now the 614 * existing code clears all entries and restores just the 615 * bolted ones when switching back to host. 616 */ 617 slb_clear_invalidate_partition(); 618 } 619 } 620 621 static void save_clear_guest_mmu(struct kvm *kvm, struct kvm_vcpu *vcpu) 622 { 623 if (kvm_is_radix(kvm)) { 624 radix_clear_slb(); 625 } else { 626 int i; 627 int nr = 0; 628 629 /* 630 * This must run before switching to host (radix host can't 631 * access all SLBs). 632 */ 633 for (i = 0; i < vcpu->arch.slb_nr; i++) { 634 u64 slbee, slbev; 635 636 slbee = mfslbe(i); 637 if (slbee & SLB_ESID_V) { 638 slbev = mfslbv(i); 639 vcpu->arch.slb[nr].orige = slbee | i; 640 vcpu->arch.slb[nr].origv = slbev; 641 nr++; 642 } 643 } 644 vcpu->arch.slb_max = nr; 645 slb_clear_invalidate_partition(); 646 } 647 } 648 649 static void flush_guest_tlb(struct kvm *kvm) 650 { 651 unsigned long rb, set; 652 653 rb = PPC_BIT(52); /* IS = 2 */ 654 if (kvm_is_radix(kvm)) { 655 /* R=1 PRS=1 RIC=2 */ 656 asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1) 657 : : "r" (rb), "i" (1), "i" (1), "i" (2), 658 "r" (0) : "memory"); 659 for (set = 1; set < kvm->arch.tlb_sets; ++set) { 660 rb += PPC_BIT(51); /* increment set number */ 661 /* R=1 PRS=1 RIC=0 */ 662 asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1) 663 : : "r" (rb), "i" (1), "i" (1), "i" (0), 664 "r" (0) : "memory"); 665 } 666 asm volatile("ptesync": : :"memory"); 667 // POWER9 congruence-class TLBIEL leaves ERAT. Flush it now. 668 asm volatile(PPC_RADIX_INVALIDATE_ERAT_GUEST : : :"memory"); 669 } else { 670 for (set = 0; set < kvm->arch.tlb_sets; ++set) { 671 /* R=0 PRS=0 RIC=0 */ 672 asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1) 673 : : "r" (rb), "i" (0), "i" (0), "i" (0), 674 "r" (0) : "memory"); 675 rb += PPC_BIT(51); /* increment set number */ 676 } 677 asm volatile("ptesync": : :"memory"); 678 // POWER9 congruence-class TLBIEL leaves ERAT. Flush it now. 679 asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT : : :"memory"); 680 } 681 } 682 683 static void check_need_tlb_flush(struct kvm *kvm, int pcpu, 684 struct kvm_nested_guest *nested) 685 { 686 cpumask_t *need_tlb_flush; 687 bool all_set = true; 688 int i; 689 690 if (nested) 691 need_tlb_flush = &nested->need_tlb_flush; 692 else 693 need_tlb_flush = &kvm->arch.need_tlb_flush; 694 695 if (likely(!cpumask_test_cpu(pcpu, need_tlb_flush))) 696 return; 697 698 /* 699 * Individual threads can come in here, but the TLB is shared between 700 * the 4 threads in a core, hence invalidating on one thread 701 * invalidates for all, so only invalidate the first time (if all bits 702 * were set. The others must still execute a ptesync. 703 * 704 * If a race occurs and two threads do the TLB flush, that is not a 705 * problem, just sub-optimal. 706 */ 707 for (i = cpu_first_tlb_thread_sibling(pcpu); 708 i <= cpu_last_tlb_thread_sibling(pcpu); 709 i += cpu_tlb_thread_sibling_step()) { 710 if (!cpumask_test_cpu(i, need_tlb_flush)) { 711 all_set = false; 712 break; 713 } 714 } 715 if (all_set) 716 flush_guest_tlb(kvm); 717 else 718 asm volatile("ptesync" ::: "memory"); 719 720 /* Clear the bit after the TLB flush */ 721 cpumask_clear_cpu(pcpu, need_tlb_flush); 722 } 723 724 unsigned long kvmppc_msr_hard_disable_set_facilities(struct kvm_vcpu *vcpu, unsigned long msr) 725 { 726 unsigned long msr_needed = 0; 727 728 msr &= ~MSR_EE; 729 730 /* MSR bits may have been cleared by context switch so must recheck */ 731 if (IS_ENABLED(CONFIG_PPC_FPU)) 732 msr_needed |= MSR_FP; 733 if (cpu_has_feature(CPU_FTR_ALTIVEC)) 734 msr_needed |= MSR_VEC; 735 if (cpu_has_feature(CPU_FTR_VSX)) 736 msr_needed |= MSR_VSX; 737 if ((cpu_has_feature(CPU_FTR_TM) || 738 cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) && 739 (vcpu->arch.hfscr & HFSCR_TM)) 740 msr_needed |= MSR_TM; 741 742 /* 743 * This could be combined with MSR[RI] clearing, but that expands 744 * the unrecoverable window. It would be better to cover unrecoverable 745 * with KVM bad interrupt handling rather than use MSR[RI] at all. 746 * 747 * Much more difficult and less worthwhile to combine with IR/DR 748 * disable. 749 */ 750 if ((msr & msr_needed) != msr_needed) { 751 msr |= msr_needed; 752 __mtmsrd(msr, 0); 753 } else { 754 __hard_irq_disable(); 755 } 756 local_paca->irq_happened |= PACA_IRQ_HARD_DIS; 757 758 return msr; 759 } 760 EXPORT_SYMBOL_GPL(kvmppc_msr_hard_disable_set_facilities); 761 762 int kvmhv_vcpu_entry_p9(struct kvm_vcpu *vcpu, u64 time_limit, unsigned long lpcr, u64 *tb) 763 { 764 struct p9_host_os_sprs host_os_sprs; 765 struct kvm *kvm = vcpu->kvm; 766 struct kvm_nested_guest *nested = vcpu->arch.nested; 767 struct kvmppc_vcore *vc = vcpu->arch.vcore; 768 s64 hdec, dec; 769 u64 purr, spurr; 770 u64 *exsave; 771 int trap; 772 unsigned long msr; 773 unsigned long host_hfscr; 774 unsigned long host_ciabr; 775 unsigned long host_dawr0; 776 unsigned long host_dawrx0; 777 unsigned long host_psscr; 778 unsigned long host_hpsscr; 779 unsigned long host_pidr; 780 unsigned long host_dawr1; 781 unsigned long host_dawrx1; 782 unsigned long dpdes; 783 784 hdec = time_limit - *tb; 785 if (hdec < 0) 786 return BOOK3S_INTERRUPT_HV_DECREMENTER; 787 788 WARN_ON_ONCE(vcpu->arch.shregs.msr & MSR_HV); 789 WARN_ON_ONCE(!(vcpu->arch.shregs.msr & MSR_ME)); 790 791 start_timing(vcpu, &vcpu->arch.rm_entry); 792 793 vcpu->arch.ceded = 0; 794 795 /* Save MSR for restore, with EE clear. */ 796 msr = mfmsr() & ~MSR_EE; 797 798 host_hfscr = mfspr(SPRN_HFSCR); 799 host_ciabr = mfspr(SPRN_CIABR); 800 host_psscr = mfspr(SPRN_PSSCR_PR); 801 if (cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) 802 host_hpsscr = mfspr(SPRN_PSSCR); 803 host_pidr = mfspr(SPRN_PID); 804 805 if (dawr_enabled()) { 806 host_dawr0 = mfspr(SPRN_DAWR0); 807 host_dawrx0 = mfspr(SPRN_DAWRX0); 808 if (cpu_has_feature(CPU_FTR_DAWR1)) { 809 host_dawr1 = mfspr(SPRN_DAWR1); 810 host_dawrx1 = mfspr(SPRN_DAWRX1); 811 } 812 } 813 814 local_paca->kvm_hstate.host_purr = mfspr(SPRN_PURR); 815 local_paca->kvm_hstate.host_spurr = mfspr(SPRN_SPURR); 816 817 save_p9_host_os_sprs(&host_os_sprs); 818 819 msr = kvmppc_msr_hard_disable_set_facilities(vcpu, msr); 820 if (lazy_irq_pending()) { 821 trap = 0; 822 goto out; 823 } 824 825 if (unlikely(load_vcpu_state(vcpu, &host_os_sprs))) 826 msr = mfmsr(); /* MSR may have been updated */ 827 828 if (vc->tb_offset) { 829 u64 new_tb = *tb + vc->tb_offset; 830 mtspr(SPRN_TBU40, new_tb); 831 if ((mftb() & 0xffffff) < (new_tb & 0xffffff)) { 832 new_tb += 0x1000000; 833 mtspr(SPRN_TBU40, new_tb); 834 } 835 *tb = new_tb; 836 vc->tb_offset_applied = vc->tb_offset; 837 } 838 839 mtspr(SPRN_VTB, vc->vtb); 840 mtspr(SPRN_PURR, vcpu->arch.purr); 841 mtspr(SPRN_SPURR, vcpu->arch.spurr); 842 843 if (vc->pcr) 844 mtspr(SPRN_PCR, vc->pcr | PCR_MASK); 845 if (vcpu->arch.doorbell_request) { 846 vcpu->arch.doorbell_request = 0; 847 mtspr(SPRN_DPDES, 1); 848 } 849 850 if (dawr_enabled()) { 851 if (vcpu->arch.dawr0 != host_dawr0) 852 mtspr(SPRN_DAWR0, vcpu->arch.dawr0); 853 if (vcpu->arch.dawrx0 != host_dawrx0) 854 mtspr(SPRN_DAWRX0, vcpu->arch.dawrx0); 855 if (cpu_has_feature(CPU_FTR_DAWR1)) { 856 if (vcpu->arch.dawr1 != host_dawr1) 857 mtspr(SPRN_DAWR1, vcpu->arch.dawr1); 858 if (vcpu->arch.dawrx1 != host_dawrx1) 859 mtspr(SPRN_DAWRX1, vcpu->arch.dawrx1); 860 } 861 } 862 if (vcpu->arch.ciabr != host_ciabr) 863 mtspr(SPRN_CIABR, vcpu->arch.ciabr); 864 865 866 if (cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) { 867 mtspr(SPRN_PSSCR, vcpu->arch.psscr | PSSCR_EC | 868 (local_paca->kvm_hstate.fake_suspend << PSSCR_FAKE_SUSPEND_LG)); 869 } else { 870 if (vcpu->arch.psscr != host_psscr) 871 mtspr(SPRN_PSSCR_PR, vcpu->arch.psscr); 872 } 873 874 mtspr(SPRN_HFSCR, vcpu->arch.hfscr); 875 876 mtspr(SPRN_HSRR0, vcpu->arch.regs.nip); 877 mtspr(SPRN_HSRR1, (vcpu->arch.shregs.msr & ~MSR_HV) | MSR_ME); 878 879 /* 880 * On POWER9 DD2.1 and below, sometimes on a Hypervisor Data Storage 881 * Interrupt (HDSI) the HDSISR is not be updated at all. 882 * 883 * To work around this we put a canary value into the HDSISR before 884 * returning to a guest and then check for this canary when we take a 885 * HDSI. If we find the canary on a HDSI, we know the hardware didn't 886 * update the HDSISR. In this case we return to the guest to retake the 887 * HDSI which should correctly update the HDSISR the second time HDSI 888 * entry. 889 * 890 * The "radix prefetch bug" test can be used to test for this bug, as 891 * it also exists fo DD2.1 and below. 892 */ 893 if (cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG)) 894 mtspr(SPRN_HDSISR, HDSISR_CANARY); 895 896 mtspr(SPRN_SPRG0, vcpu->arch.shregs.sprg0); 897 mtspr(SPRN_SPRG1, vcpu->arch.shregs.sprg1); 898 mtspr(SPRN_SPRG2, vcpu->arch.shregs.sprg2); 899 mtspr(SPRN_SPRG3, vcpu->arch.shregs.sprg3); 900 901 /* 902 * It might be preferable to load_vcpu_state here, in order to get the 903 * GPR/FP register loads executing in parallel with the previous mtSPR 904 * instructions, but for now that can't be done because the TM handling 905 * in load_vcpu_state can change some SPRs and vcpu state (nip, msr). 906 * But TM could be split out if this would be a significant benefit. 907 */ 908 909 /* 910 * MSR[RI] does not need to be cleared (and is not, for radix guests 911 * with no prefetch bug), because in_guest is set. If we take a SRESET 912 * or MCE with in_guest set but still in HV mode, then 913 * kvmppc_p9_bad_interrupt handles the interrupt, which effectively 914 * clears MSR[RI] and doesn't return. 915 */ 916 WRITE_ONCE(local_paca->kvm_hstate.in_guest, KVM_GUEST_MODE_HV_P9); 917 barrier(); /* Open in_guest critical section */ 918 919 /* 920 * Hash host, hash guest, or radix guest with prefetch bug, all have 921 * to disable the MMU before switching to guest MMU state. 922 */ 923 if (!radix_enabled() || !kvm_is_radix(kvm) || 924 cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG)) 925 __mtmsrd(msr & ~(MSR_IR|MSR_DR|MSR_RI), 0); 926 927 save_clear_host_mmu(kvm); 928 929 if (kvm_is_radix(kvm)) 930 switch_mmu_to_guest_radix(kvm, vcpu, lpcr); 931 else 932 switch_mmu_to_guest_hpt(kvm, vcpu, lpcr); 933 934 /* TLBIEL uses LPID=LPIDR, so run this after setting guest LPID */ 935 check_need_tlb_flush(kvm, vc->pcpu, nested); 936 937 /* 938 * P9 suppresses the HDEC exception when LPCR[HDICE] = 0, 939 * so set guest LPCR (with HDICE) before writing HDEC. 940 */ 941 mtspr(SPRN_HDEC, hdec); 942 943 mtspr(SPRN_DEC, vcpu->arch.dec_expires - *tb); 944 945 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 946 tm_return_to_guest: 947 #endif 948 mtspr(SPRN_DAR, vcpu->arch.shregs.dar); 949 mtspr(SPRN_DSISR, vcpu->arch.shregs.dsisr); 950 mtspr(SPRN_SRR0, vcpu->arch.shregs.srr0); 951 mtspr(SPRN_SRR1, vcpu->arch.shregs.srr1); 952 953 accumulate_time(vcpu, &vcpu->arch.guest_time); 954 955 switch_pmu_to_guest(vcpu, &host_os_sprs); 956 kvmppc_p9_enter_guest(vcpu); 957 switch_pmu_to_host(vcpu, &host_os_sprs); 958 959 accumulate_time(vcpu, &vcpu->arch.rm_intr); 960 961 /* XXX: Could get these from r11/12 and paca exsave instead */ 962 vcpu->arch.shregs.srr0 = mfspr(SPRN_SRR0); 963 vcpu->arch.shregs.srr1 = mfspr(SPRN_SRR1); 964 vcpu->arch.shregs.dar = mfspr(SPRN_DAR); 965 vcpu->arch.shregs.dsisr = mfspr(SPRN_DSISR); 966 967 /* 0x2 bit for HSRR is only used by PR and P7/8 HV paths, clear it */ 968 trap = local_paca->kvm_hstate.scratch0 & ~0x2; 969 970 if (likely(trap > BOOK3S_INTERRUPT_MACHINE_CHECK)) 971 exsave = local_paca->exgen; 972 else if (trap == BOOK3S_INTERRUPT_SYSTEM_RESET) 973 exsave = local_paca->exnmi; 974 else /* trap == 0x200 */ 975 exsave = local_paca->exmc; 976 977 vcpu->arch.regs.gpr[1] = local_paca->kvm_hstate.scratch1; 978 vcpu->arch.regs.gpr[3] = local_paca->kvm_hstate.scratch2; 979 980 /* 981 * After reading machine check regs (DAR, DSISR, SRR0/1) and hstate 982 * scratch (which we need to move into exsave to make re-entrant vs 983 * SRESET/MCE), register state is protected from reentrancy. However 984 * timebase, MMU, among other state is still set to guest, so don't 985 * enable MSR[RI] here. It gets enabled at the end, after in_guest 986 * is cleared. 987 * 988 * It is possible an NMI could come in here, which is why it is 989 * important to save the above state early so it can be debugged. 990 */ 991 992 vcpu->arch.regs.gpr[9] = exsave[EX_R9/sizeof(u64)]; 993 vcpu->arch.regs.gpr[10] = exsave[EX_R10/sizeof(u64)]; 994 vcpu->arch.regs.gpr[11] = exsave[EX_R11/sizeof(u64)]; 995 vcpu->arch.regs.gpr[12] = exsave[EX_R12/sizeof(u64)]; 996 vcpu->arch.regs.gpr[13] = exsave[EX_R13/sizeof(u64)]; 997 vcpu->arch.ppr = exsave[EX_PPR/sizeof(u64)]; 998 vcpu->arch.cfar = exsave[EX_CFAR/sizeof(u64)]; 999 vcpu->arch.regs.ctr = exsave[EX_CTR/sizeof(u64)]; 1000 1001 vcpu->arch.last_inst = KVM_INST_FETCH_FAILED; 1002 1003 if (unlikely(trap == BOOK3S_INTERRUPT_MACHINE_CHECK)) { 1004 vcpu->arch.fault_dar = exsave[EX_DAR/sizeof(u64)]; 1005 vcpu->arch.fault_dsisr = exsave[EX_DSISR/sizeof(u64)]; 1006 kvmppc_realmode_machine_check(vcpu); 1007 1008 } else if (unlikely(trap == BOOK3S_INTERRUPT_HMI)) { 1009 kvmppc_p9_realmode_hmi_handler(vcpu); 1010 1011 } else if (trap == BOOK3S_INTERRUPT_H_EMUL_ASSIST) { 1012 vcpu->arch.emul_inst = mfspr(SPRN_HEIR); 1013 1014 } else if (trap == BOOK3S_INTERRUPT_H_DATA_STORAGE) { 1015 vcpu->arch.fault_dar = exsave[EX_DAR/sizeof(u64)]; 1016 vcpu->arch.fault_dsisr = exsave[EX_DSISR/sizeof(u64)]; 1017 vcpu->arch.fault_gpa = mfspr(SPRN_ASDR); 1018 1019 } else if (trap == BOOK3S_INTERRUPT_H_INST_STORAGE) { 1020 vcpu->arch.fault_gpa = mfspr(SPRN_ASDR); 1021 1022 } else if (trap == BOOK3S_INTERRUPT_H_FAC_UNAVAIL) { 1023 vcpu->arch.hfscr = mfspr(SPRN_HFSCR); 1024 1025 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1026 /* 1027 * Softpatch interrupt for transactional memory emulation cases 1028 * on POWER9 DD2.2. This is early in the guest exit path - we 1029 * haven't saved registers or done a treclaim yet. 1030 */ 1031 } else if (trap == BOOK3S_INTERRUPT_HV_SOFTPATCH) { 1032 vcpu->arch.emul_inst = mfspr(SPRN_HEIR); 1033 1034 /* 1035 * The cases we want to handle here are those where the guest 1036 * is in real suspend mode and is trying to transition to 1037 * transactional mode. 1038 */ 1039 if (!local_paca->kvm_hstate.fake_suspend && 1040 (vcpu->arch.shregs.msr & MSR_TS_S)) { 1041 if (kvmhv_p9_tm_emulation_early(vcpu)) { 1042 /* 1043 * Go straight back into the guest with the 1044 * new NIP/MSR as set by TM emulation. 1045 */ 1046 mtspr(SPRN_HSRR0, vcpu->arch.regs.nip); 1047 mtspr(SPRN_HSRR1, vcpu->arch.shregs.msr); 1048 goto tm_return_to_guest; 1049 } 1050 } 1051 #endif 1052 } 1053 1054 accumulate_time(vcpu, &vcpu->arch.rm_exit); 1055 1056 /* Advance host PURR/SPURR by the amount used by guest */ 1057 purr = mfspr(SPRN_PURR); 1058 spurr = mfspr(SPRN_SPURR); 1059 local_paca->kvm_hstate.host_purr += purr - vcpu->arch.purr; 1060 local_paca->kvm_hstate.host_spurr += spurr - vcpu->arch.spurr; 1061 vcpu->arch.purr = purr; 1062 vcpu->arch.spurr = spurr; 1063 1064 vcpu->arch.ic = mfspr(SPRN_IC); 1065 vcpu->arch.pid = mfspr(SPRN_PID); 1066 vcpu->arch.psscr = mfspr(SPRN_PSSCR_PR); 1067 1068 vcpu->arch.shregs.sprg0 = mfspr(SPRN_SPRG0); 1069 vcpu->arch.shregs.sprg1 = mfspr(SPRN_SPRG1); 1070 vcpu->arch.shregs.sprg2 = mfspr(SPRN_SPRG2); 1071 vcpu->arch.shregs.sprg3 = mfspr(SPRN_SPRG3); 1072 1073 dpdes = mfspr(SPRN_DPDES); 1074 if (dpdes) 1075 vcpu->arch.doorbell_request = 1; 1076 1077 vc->vtb = mfspr(SPRN_VTB); 1078 1079 dec = mfspr(SPRN_DEC); 1080 if (!(lpcr & LPCR_LD)) /* Sign extend if not using large decrementer */ 1081 dec = (s32) dec; 1082 *tb = mftb(); 1083 vcpu->arch.dec_expires = dec + *tb; 1084 1085 if (vc->tb_offset_applied) { 1086 u64 new_tb = *tb - vc->tb_offset_applied; 1087 mtspr(SPRN_TBU40, new_tb); 1088 if ((mftb() & 0xffffff) < (new_tb & 0xffffff)) { 1089 new_tb += 0x1000000; 1090 mtspr(SPRN_TBU40, new_tb); 1091 } 1092 *tb = new_tb; 1093 vc->tb_offset_applied = 0; 1094 } 1095 1096 save_clear_guest_mmu(kvm, vcpu); 1097 switch_mmu_to_host(kvm, host_pidr); 1098 1099 /* 1100 * Enable MSR here in order to have facilities enabled to save 1101 * guest registers. This enables MMU (if we were in realmode), so 1102 * only switch MMU on after the MMU is switched to host, to avoid 1103 * the P9_RADIX_PREFETCH_BUG or hash guest context. 1104 */ 1105 if (IS_ENABLED(CONFIG_PPC_TRANSACTIONAL_MEM) && 1106 vcpu->arch.shregs.msr & MSR_TS_MASK) 1107 msr |= MSR_TS_S; 1108 __mtmsrd(msr, 0); 1109 1110 store_vcpu_state(vcpu); 1111 1112 mtspr(SPRN_PURR, local_paca->kvm_hstate.host_purr); 1113 mtspr(SPRN_SPURR, local_paca->kvm_hstate.host_spurr); 1114 1115 if (cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) { 1116 /* Preserve PSSCR[FAKE_SUSPEND] until we've called kvmppc_save_tm_hv */ 1117 mtspr(SPRN_PSSCR, host_hpsscr | 1118 (local_paca->kvm_hstate.fake_suspend << PSSCR_FAKE_SUSPEND_LG)); 1119 } 1120 1121 mtspr(SPRN_HFSCR, host_hfscr); 1122 if (vcpu->arch.ciabr != host_ciabr) 1123 mtspr(SPRN_CIABR, host_ciabr); 1124 1125 if (dawr_enabled()) { 1126 if (vcpu->arch.dawr0 != host_dawr0) 1127 mtspr(SPRN_DAWR0, host_dawr0); 1128 if (vcpu->arch.dawrx0 != host_dawrx0) 1129 mtspr(SPRN_DAWRX0, host_dawrx0); 1130 if (cpu_has_feature(CPU_FTR_DAWR1)) { 1131 if (vcpu->arch.dawr1 != host_dawr1) 1132 mtspr(SPRN_DAWR1, host_dawr1); 1133 if (vcpu->arch.dawrx1 != host_dawrx1) 1134 mtspr(SPRN_DAWRX1, host_dawrx1); 1135 } 1136 } 1137 1138 if (dpdes) 1139 mtspr(SPRN_DPDES, 0); 1140 if (vc->pcr) 1141 mtspr(SPRN_PCR, PCR_MASK); 1142 1143 /* HDEC must be at least as large as DEC, so decrementer_max fits */ 1144 mtspr(SPRN_HDEC, decrementer_max); 1145 1146 timer_rearm_host_dec(*tb); 1147 1148 restore_p9_host_os_sprs(vcpu, &host_os_sprs); 1149 1150 barrier(); /* Close in_guest critical section */ 1151 WRITE_ONCE(local_paca->kvm_hstate.in_guest, KVM_GUEST_MODE_NONE); 1152 /* Interrupts are recoverable at this point */ 1153 1154 /* 1155 * cp_abort is required if the processor supports local copy-paste 1156 * to clear the copy buffer that was under control of the guest. 1157 */ 1158 if (cpu_has_feature(CPU_FTR_ARCH_31)) 1159 asm volatile(PPC_CP_ABORT); 1160 1161 out: 1162 end_timing(vcpu); 1163 1164 return trap; 1165 } 1166 EXPORT_SYMBOL_GPL(kvmhv_vcpu_entry_p9); 1167