1 // SPDX-License-Identifier: GPL-2.0 2 3 /* 4 * Clocksource driver for the synthetic counter and timers 5 * provided by the Hyper-V hypervisor to guest VMs, as described 6 * in the Hyper-V Top Level Functional Spec (TLFS). This driver 7 * is instruction set architecture independent. 8 * 9 * Copyright (C) 2019, Microsoft, Inc. 10 * 11 * Author: Michael Kelley <mikelley@microsoft.com> 12 */ 13 14 #include <linux/percpu.h> 15 #include <linux/cpumask.h> 16 #include <linux/clockchips.h> 17 #include <linux/clocksource.h> 18 #include <linux/sched_clock.h> 19 #include <linux/mm.h> 20 #include <linux/cpuhotplug.h> 21 #include <linux/interrupt.h> 22 #include <linux/irq.h> 23 #include <linux/acpi.h> 24 #include <linux/hyperv.h> 25 #include <clocksource/hyperv_timer.h> 26 #include <asm/hyperv-tlfs.h> 27 #include <asm/mshyperv.h> 28 29 static struct clock_event_device __percpu *hv_clock_event; 30 static u64 hv_sched_clock_offset __ro_after_init; 31 32 /* 33 * If false, we're using the old mechanism for stimer0 interrupts 34 * where it sends a VMbus message when it expires. The old 35 * mechanism is used when running on older versions of Hyper-V 36 * that don't support Direct Mode. While Hyper-V provides 37 * four stimer's per CPU, Linux uses only stimer0. 38 * 39 * Because Direct Mode does not require processing a VMbus 40 * message, stimer interrupts can be enabled earlier in the 41 * process of booting a CPU, and consistent with when timer 42 * interrupts are enabled for other clocksource drivers. 43 * However, for legacy versions of Hyper-V when Direct Mode 44 * is not enabled, setting up stimer interrupts must be 45 * delayed until VMbus is initialized and can process the 46 * interrupt message. 47 */ 48 static bool direct_mode_enabled; 49 50 static int stimer0_irq = -1; 51 static int stimer0_message_sint; 52 static DEFINE_PER_CPU(long, stimer0_evt); 53 54 /* 55 * Common code for stimer0 interrupts coming via Direct Mode or 56 * as a VMbus message. 57 */ 58 void hv_stimer0_isr(void) 59 { 60 struct clock_event_device *ce; 61 62 ce = this_cpu_ptr(hv_clock_event); 63 ce->event_handler(ce); 64 } 65 EXPORT_SYMBOL_GPL(hv_stimer0_isr); 66 67 /* 68 * stimer0 interrupt handler for architectures that support 69 * per-cpu interrupts, which also implies Direct Mode. 70 */ 71 static irqreturn_t hv_stimer0_percpu_isr(int irq, void *dev_id) 72 { 73 hv_stimer0_isr(); 74 return IRQ_HANDLED; 75 } 76 77 static int hv_ce_set_next_event(unsigned long delta, 78 struct clock_event_device *evt) 79 { 80 u64 current_tick; 81 82 current_tick = hv_read_reference_counter(); 83 current_tick += delta; 84 hv_set_register(HV_REGISTER_STIMER0_COUNT, current_tick); 85 return 0; 86 } 87 88 static int hv_ce_shutdown(struct clock_event_device *evt) 89 { 90 hv_set_register(HV_REGISTER_STIMER0_COUNT, 0); 91 hv_set_register(HV_REGISTER_STIMER0_CONFIG, 0); 92 if (direct_mode_enabled && stimer0_irq >= 0) 93 disable_percpu_irq(stimer0_irq); 94 95 return 0; 96 } 97 98 static int hv_ce_set_oneshot(struct clock_event_device *evt) 99 { 100 union hv_stimer_config timer_cfg; 101 102 timer_cfg.as_uint64 = 0; 103 timer_cfg.enable = 1; 104 timer_cfg.auto_enable = 1; 105 if (direct_mode_enabled) { 106 /* 107 * When it expires, the timer will directly interrupt 108 * on the specified hardware vector/IRQ. 109 */ 110 timer_cfg.direct_mode = 1; 111 timer_cfg.apic_vector = HYPERV_STIMER0_VECTOR; 112 if (stimer0_irq >= 0) 113 enable_percpu_irq(stimer0_irq, IRQ_TYPE_NONE); 114 } else { 115 /* 116 * When it expires, the timer will generate a VMbus message, 117 * to be handled by the normal VMbus interrupt handler. 118 */ 119 timer_cfg.direct_mode = 0; 120 timer_cfg.sintx = stimer0_message_sint; 121 } 122 hv_set_register(HV_REGISTER_STIMER0_CONFIG, timer_cfg.as_uint64); 123 return 0; 124 } 125 126 /* 127 * hv_stimer_init - Per-cpu initialization of the clockevent 128 */ 129 static int hv_stimer_init(unsigned int cpu) 130 { 131 struct clock_event_device *ce; 132 133 if (!hv_clock_event) 134 return 0; 135 136 ce = per_cpu_ptr(hv_clock_event, cpu); 137 ce->name = "Hyper-V clockevent"; 138 ce->features = CLOCK_EVT_FEAT_ONESHOT; 139 ce->cpumask = cpumask_of(cpu); 140 ce->rating = 1000; 141 ce->set_state_shutdown = hv_ce_shutdown; 142 ce->set_state_oneshot = hv_ce_set_oneshot; 143 ce->set_next_event = hv_ce_set_next_event; 144 145 clockevents_config_and_register(ce, 146 HV_CLOCK_HZ, 147 HV_MIN_DELTA_TICKS, 148 HV_MAX_MAX_DELTA_TICKS); 149 return 0; 150 } 151 152 /* 153 * hv_stimer_cleanup - Per-cpu cleanup of the clockevent 154 */ 155 int hv_stimer_cleanup(unsigned int cpu) 156 { 157 struct clock_event_device *ce; 158 159 if (!hv_clock_event) 160 return 0; 161 162 /* 163 * In the legacy case where Direct Mode is not enabled 164 * (which can only be on x86/64), stimer cleanup happens 165 * relatively early in the CPU offlining process. We 166 * must unbind the stimer-based clockevent device so 167 * that the LAPIC timer can take over until clockevents 168 * are no longer needed in the offlining process. Note 169 * that clockevents_unbind_device() eventually calls 170 * hv_ce_shutdown(). 171 * 172 * The unbind should not be done when Direct Mode is 173 * enabled because we may be on an architecture where 174 * there are no other clockevent devices to fallback to. 175 */ 176 ce = per_cpu_ptr(hv_clock_event, cpu); 177 if (direct_mode_enabled) 178 hv_ce_shutdown(ce); 179 else 180 clockevents_unbind_device(ce, cpu); 181 182 return 0; 183 } 184 EXPORT_SYMBOL_GPL(hv_stimer_cleanup); 185 186 /* 187 * These placeholders are overridden by arch specific code on 188 * architectures that need special setup of the stimer0 IRQ because 189 * they don't support per-cpu IRQs (such as x86/x64). 190 */ 191 void __weak hv_setup_stimer0_handler(void (*handler)(void)) 192 { 193 }; 194 195 void __weak hv_remove_stimer0_handler(void) 196 { 197 }; 198 199 /* Called only on architectures with per-cpu IRQs (i.e., not x86/x64) */ 200 static int hv_setup_stimer0_irq(void) 201 { 202 int ret; 203 204 ret = acpi_register_gsi(NULL, HYPERV_STIMER0_VECTOR, 205 ACPI_EDGE_SENSITIVE, ACPI_ACTIVE_HIGH); 206 if (ret < 0) { 207 pr_err("Can't register Hyper-V stimer0 GSI. Error %d", ret); 208 return ret; 209 } 210 stimer0_irq = ret; 211 212 ret = request_percpu_irq(stimer0_irq, hv_stimer0_percpu_isr, 213 "Hyper-V stimer0", &stimer0_evt); 214 if (ret) { 215 pr_err("Can't request Hyper-V stimer0 IRQ %d. Error %d", 216 stimer0_irq, ret); 217 acpi_unregister_gsi(stimer0_irq); 218 stimer0_irq = -1; 219 } 220 return ret; 221 } 222 223 static void hv_remove_stimer0_irq(void) 224 { 225 if (stimer0_irq == -1) { 226 hv_remove_stimer0_handler(); 227 } else { 228 free_percpu_irq(stimer0_irq, &stimer0_evt); 229 acpi_unregister_gsi(stimer0_irq); 230 stimer0_irq = -1; 231 } 232 } 233 234 /* hv_stimer_alloc - Global initialization of the clockevent and stimer0 */ 235 int hv_stimer_alloc(bool have_percpu_irqs) 236 { 237 int ret; 238 239 /* 240 * Synthetic timers are always available except on old versions of 241 * Hyper-V on x86. In that case, return as error as Linux will use a 242 * clockevent based on emulated LAPIC timer hardware. 243 */ 244 if (!(ms_hyperv.features & HV_MSR_SYNTIMER_AVAILABLE)) 245 return -EINVAL; 246 247 hv_clock_event = alloc_percpu(struct clock_event_device); 248 if (!hv_clock_event) 249 return -ENOMEM; 250 251 direct_mode_enabled = ms_hyperv.misc_features & 252 HV_STIMER_DIRECT_MODE_AVAILABLE; 253 254 /* 255 * If Direct Mode isn't enabled, the remainder of the initialization 256 * is done later by hv_stimer_legacy_init() 257 */ 258 if (!direct_mode_enabled) 259 return 0; 260 261 if (have_percpu_irqs) { 262 ret = hv_setup_stimer0_irq(); 263 if (ret) 264 goto free_clock_event; 265 } else { 266 hv_setup_stimer0_handler(hv_stimer0_isr); 267 } 268 269 /* 270 * Since we are in Direct Mode, stimer initialization 271 * can be done now with a CPUHP value in the same range 272 * as other clockevent devices. 273 */ 274 ret = cpuhp_setup_state(CPUHP_AP_HYPERV_TIMER_STARTING, 275 "clockevents/hyperv/stimer:starting", 276 hv_stimer_init, hv_stimer_cleanup); 277 if (ret < 0) { 278 hv_remove_stimer0_irq(); 279 goto free_clock_event; 280 } 281 return ret; 282 283 free_clock_event: 284 free_percpu(hv_clock_event); 285 hv_clock_event = NULL; 286 return ret; 287 } 288 EXPORT_SYMBOL_GPL(hv_stimer_alloc); 289 290 /* 291 * hv_stimer_legacy_init -- Called from the VMbus driver to handle 292 * the case when Direct Mode is not enabled, and the stimer 293 * must be initialized late in the CPU onlining process. 294 * 295 */ 296 void hv_stimer_legacy_init(unsigned int cpu, int sint) 297 { 298 if (direct_mode_enabled) 299 return; 300 301 /* 302 * This function gets called by each vCPU, so setting the 303 * global stimer_message_sint value each time is conceptually 304 * not ideal, but the value passed in is always the same and 305 * it avoids introducing yet another interface into this 306 * clocksource driver just to set the sint in the legacy case. 307 */ 308 stimer0_message_sint = sint; 309 (void)hv_stimer_init(cpu); 310 } 311 EXPORT_SYMBOL_GPL(hv_stimer_legacy_init); 312 313 /* 314 * hv_stimer_legacy_cleanup -- Called from the VMbus driver to 315 * handle the case when Direct Mode is not enabled, and the 316 * stimer must be cleaned up early in the CPU offlining 317 * process. 318 */ 319 void hv_stimer_legacy_cleanup(unsigned int cpu) 320 { 321 if (direct_mode_enabled) 322 return; 323 (void)hv_stimer_cleanup(cpu); 324 } 325 EXPORT_SYMBOL_GPL(hv_stimer_legacy_cleanup); 326 327 /* 328 * Do a global cleanup of clockevents for the cases of kexec and 329 * vmbus exit 330 */ 331 void hv_stimer_global_cleanup(void) 332 { 333 int cpu; 334 335 /* 336 * hv_stime_legacy_cleanup() will stop the stimer if Direct 337 * Mode is not enabled, and fallback to the LAPIC timer. 338 */ 339 for_each_present_cpu(cpu) { 340 hv_stimer_legacy_cleanup(cpu); 341 } 342 343 if (!hv_clock_event) 344 return; 345 346 if (direct_mode_enabled) { 347 cpuhp_remove_state(CPUHP_AP_HYPERV_TIMER_STARTING); 348 hv_remove_stimer0_irq(); 349 stimer0_irq = -1; 350 } 351 free_percpu(hv_clock_event); 352 hv_clock_event = NULL; 353 354 } 355 EXPORT_SYMBOL_GPL(hv_stimer_global_cleanup); 356 357 /* 358 * Code and definitions for the Hyper-V clocksources. Two 359 * clocksources are defined: one that reads the Hyper-V defined MSR, and 360 * the other that uses the TSC reference page feature as defined in the 361 * TLFS. The MSR version is for compatibility with old versions of 362 * Hyper-V and 32-bit x86. The TSC reference page version is preferred. 363 */ 364 365 static union { 366 struct ms_hyperv_tsc_page page; 367 u8 reserved[PAGE_SIZE]; 368 } tsc_pg __aligned(PAGE_SIZE); 369 370 static struct ms_hyperv_tsc_page *tsc_page = &tsc_pg.page; 371 static unsigned long tsc_pfn; 372 373 unsigned long hv_get_tsc_pfn(void) 374 { 375 return tsc_pfn; 376 } 377 EXPORT_SYMBOL_GPL(hv_get_tsc_pfn); 378 379 struct ms_hyperv_tsc_page *hv_get_tsc_page(void) 380 { 381 return tsc_page; 382 } 383 EXPORT_SYMBOL_GPL(hv_get_tsc_page); 384 385 static u64 notrace read_hv_clock_tsc(void) 386 { 387 u64 current_tick = hv_read_tsc_page(hv_get_tsc_page()); 388 389 if (current_tick == U64_MAX) 390 current_tick = hv_get_register(HV_REGISTER_TIME_REF_COUNT); 391 392 return current_tick; 393 } 394 395 static u64 notrace read_hv_clock_tsc_cs(struct clocksource *arg) 396 { 397 return read_hv_clock_tsc(); 398 } 399 400 static u64 notrace read_hv_sched_clock_tsc(void) 401 { 402 return (read_hv_clock_tsc() - hv_sched_clock_offset) * 403 (NSEC_PER_SEC / HV_CLOCK_HZ); 404 } 405 406 static void suspend_hv_clock_tsc(struct clocksource *arg) 407 { 408 union hv_reference_tsc_msr tsc_msr; 409 410 /* Disable the TSC page */ 411 tsc_msr.as_uint64 = hv_get_register(HV_REGISTER_REFERENCE_TSC); 412 tsc_msr.enable = 0; 413 hv_set_register(HV_REGISTER_REFERENCE_TSC, tsc_msr.as_uint64); 414 } 415 416 417 static void resume_hv_clock_tsc(struct clocksource *arg) 418 { 419 union hv_reference_tsc_msr tsc_msr; 420 421 /* Re-enable the TSC page */ 422 tsc_msr.as_uint64 = hv_get_register(HV_REGISTER_REFERENCE_TSC); 423 tsc_msr.enable = 1; 424 tsc_msr.pfn = tsc_pfn; 425 hv_set_register(HV_REGISTER_REFERENCE_TSC, tsc_msr.as_uint64); 426 } 427 428 #ifdef HAVE_VDSO_CLOCKMODE_HVCLOCK 429 static int hv_cs_enable(struct clocksource *cs) 430 { 431 vclocks_set_used(VDSO_CLOCKMODE_HVCLOCK); 432 return 0; 433 } 434 #endif 435 436 static struct clocksource hyperv_cs_tsc = { 437 .name = "hyperv_clocksource_tsc_page", 438 .rating = 500, 439 .read = read_hv_clock_tsc_cs, 440 .mask = CLOCKSOURCE_MASK(64), 441 .flags = CLOCK_SOURCE_IS_CONTINUOUS, 442 .suspend= suspend_hv_clock_tsc, 443 .resume = resume_hv_clock_tsc, 444 #ifdef HAVE_VDSO_CLOCKMODE_HVCLOCK 445 .enable = hv_cs_enable, 446 .vdso_clock_mode = VDSO_CLOCKMODE_HVCLOCK, 447 #else 448 .vdso_clock_mode = VDSO_CLOCKMODE_NONE, 449 #endif 450 }; 451 452 static u64 notrace read_hv_clock_msr(void) 453 { 454 /* 455 * Read the partition counter to get the current tick count. This count 456 * is set to 0 when the partition is created and is incremented in 457 * 100 nanosecond units. 458 */ 459 return hv_get_register(HV_REGISTER_TIME_REF_COUNT); 460 } 461 462 static u64 notrace read_hv_clock_msr_cs(struct clocksource *arg) 463 { 464 return read_hv_clock_msr(); 465 } 466 467 static u64 notrace read_hv_sched_clock_msr(void) 468 { 469 return (read_hv_clock_msr() - hv_sched_clock_offset) * 470 (NSEC_PER_SEC / HV_CLOCK_HZ); 471 } 472 473 static struct clocksource hyperv_cs_msr = { 474 .name = "hyperv_clocksource_msr", 475 .rating = 500, 476 .read = read_hv_clock_msr_cs, 477 .mask = CLOCKSOURCE_MASK(64), 478 .flags = CLOCK_SOURCE_IS_CONTINUOUS, 479 }; 480 481 /* 482 * Reference to pv_ops must be inline so objtool 483 * detection of noinstr violations can work correctly. 484 */ 485 #ifdef CONFIG_GENERIC_SCHED_CLOCK 486 static __always_inline void hv_setup_sched_clock(void *sched_clock) 487 { 488 /* 489 * We're on an architecture with generic sched clock (not x86/x64). 490 * The Hyper-V sched clock read function returns nanoseconds, not 491 * the normal 100ns units of the Hyper-V synthetic clock. 492 */ 493 sched_clock_register(sched_clock, 64, NSEC_PER_SEC); 494 } 495 #elif defined CONFIG_PARAVIRT 496 static __always_inline void hv_setup_sched_clock(void *sched_clock) 497 { 498 /* We're on x86/x64 *and* using PV ops */ 499 paravirt_set_sched_clock(sched_clock); 500 } 501 #else /* !CONFIG_GENERIC_SCHED_CLOCK && !CONFIG_PARAVIRT */ 502 static __always_inline void hv_setup_sched_clock(void *sched_clock) {} 503 #endif /* CONFIG_GENERIC_SCHED_CLOCK */ 504 505 static bool __init hv_init_tsc_clocksource(void) 506 { 507 union hv_reference_tsc_msr tsc_msr; 508 509 if (!(ms_hyperv.features & HV_MSR_REFERENCE_TSC_AVAILABLE)) 510 return false; 511 512 /* 513 * If Hyper-V offers TSC_INVARIANT, then the virtualized TSC correctly 514 * handles frequency and offset changes due to live migration, 515 * pause/resume, and other VM management operations. So lower the 516 * Hyper-V Reference TSC rating, causing the generic TSC to be used. 517 * TSC_INVARIANT is not offered on ARM64, so the Hyper-V Reference 518 * TSC will be preferred over the virtualized ARM64 arch counter. 519 * While the Hyper-V MSR clocksource won't be used since the 520 * Reference TSC clocksource is present, change its rating as 521 * well for consistency. 522 */ 523 if (ms_hyperv.features & HV_ACCESS_TSC_INVARIANT) { 524 hyperv_cs_tsc.rating = 250; 525 hyperv_cs_msr.rating = 250; 526 } 527 528 hv_read_reference_counter = read_hv_clock_tsc; 529 530 /* 531 * TSC page mapping works differently in root compared to guest. 532 * - In guest partition the guest PFN has to be passed to the 533 * hypervisor. 534 * - In root partition it's other way around: it has to map the PFN 535 * provided by the hypervisor. 536 * But it can't be mapped right here as it's too early and MMU isn't 537 * ready yet. So, we only set the enable bit here and will remap the 538 * page later in hv_remap_tsc_clocksource(). 539 * 540 * It worth mentioning, that TSC clocksource read function 541 * (read_hv_clock_tsc) has a MSR-based fallback mechanism, used when 542 * TSC page is zeroed (which is the case until the PFN is remapped) and 543 * thus TSC clocksource will work even without the real TSC page 544 * mapped. 545 */ 546 tsc_msr.as_uint64 = hv_get_register(HV_REGISTER_REFERENCE_TSC); 547 if (hv_root_partition) 548 tsc_pfn = tsc_msr.pfn; 549 else 550 tsc_pfn = HVPFN_DOWN(virt_to_phys(tsc_page)); 551 tsc_msr.enable = 1; 552 tsc_msr.pfn = tsc_pfn; 553 hv_set_register(HV_REGISTER_REFERENCE_TSC, tsc_msr.as_uint64); 554 555 clocksource_register_hz(&hyperv_cs_tsc, NSEC_PER_SEC/100); 556 557 hv_sched_clock_offset = hv_read_reference_counter(); 558 hv_setup_sched_clock(read_hv_sched_clock_tsc); 559 560 return true; 561 } 562 563 void __init hv_init_clocksource(void) 564 { 565 /* 566 * Try to set up the TSC page clocksource. If it succeeds, we're 567 * done. Otherwise, set up the MSR clocksource. At least one of 568 * these will always be available except on very old versions of 569 * Hyper-V on x86. In that case we won't have a Hyper-V 570 * clocksource, but Linux will still run with a clocksource based 571 * on the emulated PIT or LAPIC timer. 572 */ 573 if (hv_init_tsc_clocksource()) 574 return; 575 576 if (!(ms_hyperv.features & HV_MSR_TIME_REF_COUNT_AVAILABLE)) 577 return; 578 579 hv_read_reference_counter = read_hv_clock_msr; 580 clocksource_register_hz(&hyperv_cs_msr, NSEC_PER_SEC/100); 581 582 hv_sched_clock_offset = hv_read_reference_counter(); 583 hv_setup_sched_clock(read_hv_sched_clock_msr); 584 } 585 586 void __init hv_remap_tsc_clocksource(void) 587 { 588 if (!(ms_hyperv.features & HV_MSR_REFERENCE_TSC_AVAILABLE)) 589 return; 590 591 if (!hv_root_partition) { 592 WARN(1, "%s: attempt to remap TSC page in guest partition\n", 593 __func__); 594 return; 595 } 596 597 tsc_page = memremap(tsc_pfn << HV_HYP_PAGE_SHIFT, sizeof(tsc_pg), 598 MEMREMAP_WB); 599 if (!tsc_page) 600 pr_err("Failed to remap Hyper-V TSC page.\n"); 601 } 602