1 /*- 2 * Copyright (c) 2009-2012,2016 Microsoft Corp. 3 * Copyright (c) 2012 NetApp Inc. 4 * Copyright (c) 2012 Citrix Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include "opt_acpi.h" 30 31 #include <sys/param.h> 32 #include <sys/bus.h> 33 #include <sys/kernel.h> 34 #include <sys/malloc.h> 35 #include <sys/module.h> 36 #include <sys/rman.h> 37 #include <sys/systimer.h> 38 #include <sys/thread.h> 39 #include <sys/thread2.h> 40 41 #include <machine/intr_machdep.h> 42 #include <machine/smp.h> 43 44 #include <dev/virtual/hyperv/hyperv_busdma.h> 45 #include <dev/virtual/hyperv/hyperv_machdep.h> 46 #include <dev/virtual/hyperv/hyperv_reg.h> 47 #include <dev/virtual/hyperv/hyperv_var.h> 48 #include <dev/virtual/hyperv/vmbus/vmbus_reg.h> 49 #include <dev/virtual/hyperv/vmbus/vmbus_var.h> 50 51 #include "acpi.h" 52 #include "acpi_if.h" 53 #include "pcib_if.h" 54 55 #define MSR_HV_STIMER0_CFG_SINT \ 56 ((((uint64_t)VMBUS_SINT_TIMER) << MSR_HV_STIMER_CFG_SINT_SHIFT) & \ 57 MSR_HV_STIMER_CFG_SINT_MASK) 58 59 /* 60 * Additionally required feature: 61 * - SynIC is needed for interrupt generation. 62 */ 63 #define CPUID_HV_TIMER_MASK (CPUID_HV_MSR_SYNIC | \ 64 CPUID_HV_MSR_SYNTIMER) 65 66 /* 67 * NOTE: DO NOT CHANGE THIS. 68 */ 69 #define VMBUS_SINT_MESSAGE 2 70 /* 71 * NOTE: 72 * - DO NOT set it to the same value as VMBUS_SINT_MESSAGE. 73 * - DO NOT set it to 0. 74 */ 75 #define VMBUS_SINT_TIMER 4 76 77 /* 78 * NOTE: DO NOT CHANGE THESE 79 */ 80 #define VMBUS_CONNID_MESSAGE 1 81 #define VMBUS_CONNID_EVENT 2 82 83 struct vmbus_msghc { 84 struct hypercall_postmsg_in *mh_inprm; 85 struct hypercall_postmsg_in mh_inprm_save; 86 struct hyperv_dma mh_inprm_dma; 87 88 struct vmbus_message *mh_resp; 89 struct vmbus_message mh_resp0; 90 }; 91 92 struct vmbus_msghc_ctx { 93 struct vmbus_msghc *mhc_free; 94 struct lwkt_token mhc_free_token; 95 uint32_t mhc_flags; 96 97 struct vmbus_msghc *mhc_active; 98 struct lwkt_token mhc_active_token; 99 }; 100 101 #define VMBUS_MSGHC_CTXF_DESTROY 0x0001 102 103 static int vmbus_probe(device_t); 104 static int vmbus_attach(device_t); 105 static int vmbus_detach(device_t); 106 static void vmbus_intr(void *); 107 static void vmbus_timer_intr_reload(struct cputimer_intr *, 108 sysclock_t); 109 static void vmbus_timer_intr_pcpuhand( 110 struct cputimer_intr *); 111 static void vmbus_timer_intr_restart( 112 struct cputimer_intr *); 113 114 static int vmbus_dma_alloc(struct vmbus_softc *); 115 static void vmbus_dma_free(struct vmbus_softc *); 116 static int vmbus_intr_setup(struct vmbus_softc *); 117 static void vmbus_intr_teardown(struct vmbus_softc *); 118 static void vmbus_synic_setup(void *); 119 static void vmbus_synic_teardown(void *); 120 static void vmbus_timer_stop(void *); 121 static void vmbus_timer_config(void *); 122 static int vmbus_init(struct vmbus_softc *); 123 static int vmbus_init_contact(struct vmbus_softc *, 124 uint32_t); 125 static void vmbus_timer_restart(void *); 126 static void vmbus_timer_msgintr(struct vmbus_pcpu_data *); 127 128 static void vmbus_chan_msgproc(struct vmbus_softc *, 129 const struct vmbus_message *); 130 131 static struct vmbus_msghc_ctx *vmbus_msghc_ctx_create(bus_dma_tag_t); 132 static void vmbus_msghc_ctx_destroy( 133 struct vmbus_msghc_ctx *); 134 static void vmbus_msghc_ctx_free(struct vmbus_msghc_ctx *); 135 static struct vmbus_msghc *vmbus_msghc_alloc(bus_dma_tag_t); 136 static void vmbus_msghc_free(struct vmbus_msghc *); 137 static struct vmbus_msghc *vmbus_msghc_get1(struct vmbus_msghc_ctx *, 138 uint32_t); 139 140 static device_method_t vmbus_methods[] = { 141 /* Device interface */ 142 DEVMETHOD(device_probe, vmbus_probe), 143 DEVMETHOD(device_attach, vmbus_attach), 144 DEVMETHOD(device_detach, vmbus_detach), 145 DEVMETHOD(device_shutdown, bus_generic_shutdown), 146 DEVMETHOD(device_suspend, bus_generic_suspend), 147 DEVMETHOD(device_resume, bus_generic_resume), 148 149 DEVMETHOD_END 150 }; 151 152 static driver_t vmbus_driver = { 153 "vmbus", 154 vmbus_methods, 155 sizeof(struct vmbus_softc) 156 }; 157 158 static devclass_t vmbus_devclass; 159 160 DRIVER_MODULE(vmbus, acpi, vmbus_driver, vmbus_devclass, NULL, NULL); 161 MODULE_DEPEND(vmbus, acpi, 1, 1, 1); 162 MODULE_VERSION(vmbus, 1); 163 164 static struct cputimer_intr vmbus_cputimer_intr = { 165 .freq = HYPERV_TIMER_FREQ, 166 .reload = vmbus_timer_intr_reload, 167 .enable = cputimer_intr_default_enable, 168 .config = cputimer_intr_default_config, 169 .restart = vmbus_timer_intr_restart, 170 .pmfixup = cputimer_intr_default_pmfixup, 171 .initclock = cputimer_intr_default_initclock, 172 .pcpuhand = vmbus_timer_intr_pcpuhand, 173 .next = SLIST_ENTRY_INITIALIZER, 174 .name = "hyperv", 175 .type = CPUTIMER_INTR_VMM, 176 .prio = CPUTIMER_INTR_PRIO_VMM, 177 .caps = CPUTIMER_INTR_CAP_PS, 178 .priv = NULL 179 }; 180 181 static const uint32_t vmbus_version[] = { 182 VMBUS_VERSION_WIN8_1, 183 VMBUS_VERSION_WIN8, 184 VMBUS_VERSION_WIN7, 185 VMBUS_VERSION_WS2008 186 }; 187 188 static int vmbus_timer_intr_enable = 1; 189 TUNABLE_INT("hw.vmbus.timer_intr.enable", &vmbus_timer_intr_enable); 190 191 static int 192 vmbus_probe(device_t dev) 193 { 194 char *id[] = { "VMBUS", NULL }; 195 196 if (ACPI_ID_PROBE(device_get_parent(dev), dev, id) == NULL || 197 device_get_unit(dev) != 0 || vmm_guest != VMM_GUEST_HYPERV || 198 (hyperv_features & CPUID_HV_MSR_SYNIC) == 0) 199 return (ENXIO); 200 201 device_set_desc(dev, "Hyper-V vmbus"); 202 203 return (0); 204 } 205 206 static int 207 vmbus_attach(device_t dev) 208 { 209 struct vmbus_softc *sc = device_get_softc(dev); 210 int error, cpu, use_timer; 211 212 /* 213 * Basic setup. 214 */ 215 sc->vmbus_dev = dev; 216 for (cpu = 0; cpu < ncpus; ++cpu) { 217 struct vmbus_pcpu_data *psc = VMBUS_PCPU(sc, cpu); 218 219 psc->sc = sc; 220 psc->cpuid = cpu; 221 psc->timer_last = UINT64_MAX; 222 } 223 224 /* 225 * Should we use interrupt timer? 226 */ 227 use_timer = 0; 228 if (device_get_unit(dev) == 0 && 229 (hyperv_features & CPUID_HV_TIMER_MASK) == CPUID_HV_TIMER_MASK && 230 hyperv_tc64 != NULL) 231 use_timer = 1; 232 233 /* 234 * Create context for "post message" Hypercalls 235 */ 236 sc->vmbus_msg_hc = vmbus_msghc_ctx_create( 237 bus_get_dma_tag(sc->vmbus_dev)); 238 if (sc->vmbus_msg_hc == NULL) 239 return ENXIO; 240 241 /* 242 * Allocate DMA stuffs. 243 */ 244 error = vmbus_dma_alloc(sc); 245 if (error) 246 goto failed; 247 248 /* 249 * Setup interrupt. 250 */ 251 error = vmbus_intr_setup(sc); 252 if (error) 253 goto failed; 254 255 if (use_timer) { 256 /* 257 * Make sure that interrupt timer is stopped. 258 */ 259 lwkt_cpusync_simple(smp_active_mask, vmbus_timer_stop, sc); 260 } 261 262 /* 263 * Setup SynIC. 264 */ 265 lwkt_cpusync_simple(smp_active_mask, vmbus_synic_setup, sc); 266 sc->vmbus_flags |= VMBUS_FLAG_SYNIC; 267 268 /* 269 * Initialize vmbus. 270 */ 271 error = vmbus_init(sc); 272 if (error) 273 goto failed; 274 275 if (use_timer) { 276 /* 277 * Configure and register vmbus interrupt timer. 278 */ 279 lwkt_cpusync_simple(smp_active_mask, vmbus_timer_config, sc); 280 vmbus_cputimer_intr.priv = sc; 281 cputimer_intr_register(&vmbus_cputimer_intr); 282 if (vmbus_timer_intr_enable) 283 cputimer_intr_select(&vmbus_cputimer_intr, 0); 284 } 285 286 return 0; 287 failed: 288 vmbus_detach(dev); 289 return error; 290 } 291 292 static int 293 vmbus_detach(device_t dev) 294 { 295 struct vmbus_softc *sc = device_get_softc(dev); 296 297 /* TODO: uninitialize vmbus. */ 298 /* TODO: stop and deregister timer */ 299 300 if (sc->vmbus_flags & VMBUS_FLAG_SYNIC) 301 lwkt_cpusync_simple(smp_active_mask, vmbus_synic_teardown, sc); 302 vmbus_intr_teardown(sc); 303 vmbus_dma_free(sc); 304 305 if (sc->vmbus_msg_hc != NULL) { 306 vmbus_msghc_ctx_destroy(sc->vmbus_msg_hc); 307 sc->vmbus_msg_hc = NULL; 308 } 309 return (0); 310 } 311 312 static __inline void 313 vmbus_msg_reset(volatile struct vmbus_message *msg) 314 { 315 msg->msg_type = HYPERV_MSGTYPE_NONE; 316 /* 317 * Make sure that the write to msg_type (i.e. set to 318 * HYPERV_MSGTYPE_NONE) happens before we read the 319 * msg_flags and send EOM to the hypervisor. 320 */ 321 cpu_mfence(); 322 if (msg->msg_flags & VMBUS_MSGFLAG_PENDING) { 323 /* 324 * Ask the hypervisor to rescan message queue, 325 * and deliver new message if any. 326 */ 327 wrmsr(MSR_HV_EOM, 0); 328 } 329 } 330 331 static void 332 vmbus_intr(void *xpsc) 333 { 334 struct vmbus_pcpu_data *psc = xpsc; 335 volatile struct vmbus_message *msg; 336 337 msg = psc->message + VMBUS_SINT_MESSAGE; 338 while (__predict_false(msg->msg_type != HYPERV_MSGTYPE_NONE)) { 339 if (msg->msg_type == HYPERV_MSGTYPE_CHANNEL) { 340 /* Channel message */ 341 vmbus_chan_msgproc(psc->sc, 342 __DEVOLATILE(const struct vmbus_message *, msg)); 343 } 344 vmbus_msg_reset(msg); 345 } 346 } 347 348 static __inline void 349 vmbus_timer_oneshot(struct vmbus_pcpu_data *psc, uint64_t current) 350 { 351 psc->timer_last = current; 352 wrmsr(MSR_HV_STIMER0_COUNT, current); 353 } 354 355 static void 356 vmbus_timer_intr_reload(struct cputimer_intr *cti, sysclock_t reload) 357 { 358 struct globaldata *gd = mycpu; 359 struct vmbus_softc *sc = cti->priv; 360 struct vmbus_pcpu_data *psc = VMBUS_PCPU(sc, gd->gd_cpuid); 361 uint64_t current; 362 363 if ((ssysclock_t)reload < 0) /* neg value */ 364 reload = 1; 365 reload = muldivu64(reload, cti->freq, sys_cputimer->freq); 366 current = hyperv_tc64() + reload; 367 368 if (gd->gd_timer_running) { 369 if (current < psc->timer_last) 370 vmbus_timer_oneshot(psc, current); 371 } else { 372 gd->gd_timer_running = 1; 373 vmbus_timer_oneshot(psc, current); 374 } 375 } 376 377 static void 378 vmbus_timer_intr_pcpuhand(struct cputimer_intr *cti) 379 { 380 struct vmbus_softc *sc = cti->priv; 381 struct vmbus_pcpu_data *psc = VMBUS_PCPU(sc, mycpuid); 382 383 vmbus_timer_msgintr(psc); 384 } 385 386 static void 387 vmbus_timer_intr_restart(struct cputimer_intr *cti) 388 { 389 lwkt_send_ipiq_mask(smp_active_mask, vmbus_timer_restart, cti->priv); 390 } 391 392 static struct vmbus_msghc * 393 vmbus_msghc_alloc(bus_dma_tag_t parent_dtag) 394 { 395 struct vmbus_msghc *mh; 396 397 mh = kmalloc(sizeof(*mh), M_DEVBUF, M_WAITOK | M_ZERO); 398 399 mh->mh_inprm = hyperv_dmamem_alloc(parent_dtag, 400 HYPERCALL_POSTMSGIN_ALIGN, 0, HYPERCALL_POSTMSGIN_SIZE, 401 &mh->mh_inprm_dma, BUS_DMA_WAITOK); 402 if (mh->mh_inprm == NULL) { 403 kfree(mh, M_DEVBUF); 404 return NULL; 405 } 406 return mh; 407 } 408 409 static void 410 vmbus_msghc_free(struct vmbus_msghc *mh) 411 { 412 hyperv_dmamem_free(&mh->mh_inprm_dma, mh->mh_inprm); 413 kfree(mh, M_DEVBUF); 414 } 415 416 static void 417 vmbus_msghc_ctx_free(struct vmbus_msghc_ctx *mhc) 418 { 419 KASSERT(mhc->mhc_active == NULL, ("still have active msg hypercall")); 420 KASSERT(mhc->mhc_free == NULL, ("still have hypercall msg")); 421 422 lwkt_token_uninit(&mhc->mhc_free_token); 423 lwkt_token_uninit(&mhc->mhc_active_token); 424 kfree(mhc, M_DEVBUF); 425 } 426 427 static struct vmbus_msghc_ctx * 428 vmbus_msghc_ctx_create(bus_dma_tag_t parent_dtag) 429 { 430 struct vmbus_msghc_ctx *mhc; 431 432 mhc = kmalloc(sizeof(*mhc), M_DEVBUF, M_WAITOK | M_ZERO); 433 lwkt_token_init(&mhc->mhc_free_token, "msghcf"); 434 lwkt_token_init(&mhc->mhc_active_token, "msghca"); 435 436 mhc->mhc_free = vmbus_msghc_alloc(parent_dtag); 437 if (mhc->mhc_free == NULL) { 438 vmbus_msghc_ctx_free(mhc); 439 return NULL; 440 } 441 return mhc; 442 } 443 444 static struct vmbus_msghc * 445 vmbus_msghc_get1(struct vmbus_msghc_ctx *mhc, uint32_t dtor_flag) 446 { 447 struct vmbus_msghc *mh; 448 449 lwkt_gettoken(&mhc->mhc_free_token); 450 451 while ((mhc->mhc_flags & dtor_flag) == 0 && mhc->mhc_free == NULL) 452 tsleep(&mhc->mhc_free, 0, "gmsghc", 0); 453 if (mhc->mhc_flags & dtor_flag) { 454 /* Being destroyed */ 455 mh = NULL; 456 } else { 457 mh = mhc->mhc_free; 458 KASSERT(mh != NULL, ("no free hypercall msg")); 459 KASSERT(mh->mh_resp == NULL, 460 ("hypercall msg has pending response")); 461 mhc->mhc_free = NULL; 462 } 463 464 lwkt_reltoken(&mhc->mhc_free_token); 465 466 return mh; 467 } 468 469 struct vmbus_msghc * 470 vmbus_msghc_get(struct vmbus_softc *sc, size_t dsize) 471 { 472 struct hypercall_postmsg_in *inprm; 473 struct vmbus_msghc *mh; 474 475 if (dsize > HYPERCALL_POSTMSGIN_DSIZE_MAX) 476 return NULL; 477 478 mh = vmbus_msghc_get1(sc->vmbus_msg_hc, VMBUS_MSGHC_CTXF_DESTROY); 479 if (mh == NULL) 480 return NULL; 481 482 inprm = mh->mh_inprm; 483 memset(inprm, 0, HYPERCALL_POSTMSGIN_SIZE); 484 inprm->hc_connid = VMBUS_CONNID_MESSAGE; 485 inprm->hc_msgtype = HYPERV_MSGTYPE_CHANNEL; 486 inprm->hc_dsize = dsize; 487 488 return mh; 489 } 490 491 void 492 vmbus_msghc_put(struct vmbus_softc *sc, struct vmbus_msghc *mh) 493 { 494 struct vmbus_msghc_ctx *mhc = sc->vmbus_msg_hc; 495 496 KASSERT(mhc->mhc_active == NULL, ("msg hypercall is active")); 497 mh->mh_resp = NULL; 498 499 lwkt_gettoken(&mhc->mhc_free_token); 500 KASSERT(mhc->mhc_free == NULL, ("has free hypercall msg")); 501 mhc->mhc_free = mh; 502 lwkt_reltoken(&mhc->mhc_free_token); 503 wakeup(&mhc->mhc_free); 504 } 505 506 void * 507 vmbus_msghc_dataptr(struct vmbus_msghc *mh) 508 { 509 return mh->mh_inprm->hc_data; 510 } 511 512 static void 513 vmbus_msghc_ctx_destroy(struct vmbus_msghc_ctx *mhc) 514 { 515 struct vmbus_msghc *mh; 516 517 lwkt_gettoken(&mhc->mhc_free_token); 518 mhc->mhc_flags |= VMBUS_MSGHC_CTXF_DESTROY; 519 lwkt_reltoken(&mhc->mhc_free_token); 520 wakeup(&mhc->mhc_free); 521 522 mh = vmbus_msghc_get1(mhc, 0); 523 if (mh == NULL) 524 panic("can't get msghc"); 525 526 vmbus_msghc_free(mh); 527 vmbus_msghc_ctx_free(mhc); 528 } 529 530 int 531 vmbus_msghc_exec_noresult(struct vmbus_msghc *mh) 532 { 533 int i, wait_ticks = 1; 534 535 /* 536 * Save the input parameter so that we could restore the input 537 * parameter if the Hypercall failed. 538 * 539 * XXX 540 * Is this really necessary?! i.e. Will the Hypercall ever 541 * overwrite the input parameter? 542 */ 543 memcpy(&mh->mh_inprm_save, mh->mh_inprm, HYPERCALL_POSTMSGIN_SIZE); 544 545 /* 546 * In order to cope with transient failures, e.g. insufficient 547 * resources on host side, we retry the post message Hypercall 548 * several times. 20 retries seem sufficient. 549 */ 550 #define HC_RETRY_MAX 20 551 552 for (i = 0; i < HC_RETRY_MAX; ++i) { 553 uint64_t status; 554 555 status = hypercall_post_message(mh->mh_inprm_dma.hv_paddr); 556 if (status == HYPERCALL_STATUS_SUCCESS) 557 return 0; 558 559 tsleep(&status, 0, "hcpmsg", wait_ticks); 560 if (wait_ticks < hz) 561 wait_ticks *= 2; 562 563 /* Restore input parameter and try again */ 564 memcpy(mh->mh_inprm, &mh->mh_inprm_save, 565 HYPERCALL_POSTMSGIN_SIZE); 566 } 567 568 #undef HC_RETRY_MAX 569 570 return EIO; 571 } 572 573 int 574 vmbus_msghc_exec(struct vmbus_softc *sc, struct vmbus_msghc *mh) 575 { 576 struct vmbus_msghc_ctx *mhc = sc->vmbus_msg_hc; 577 int error; 578 579 KASSERT(mh->mh_resp == NULL, ("hypercall msg has pending response")); 580 581 lwkt_gettoken(&mhc->mhc_active_token); 582 KASSERT(mhc->mhc_active == NULL, ("pending active msg hypercall")); 583 mhc->mhc_active = mh; 584 lwkt_reltoken(&mhc->mhc_active_token); 585 586 error = vmbus_msghc_exec_noresult(mh); 587 if (error) { 588 lwkt_gettoken(&mhc->mhc_active_token); 589 KASSERT(mhc->mhc_active == mh, ("msghc mismatch")); 590 mhc->mhc_active = NULL; 591 lwkt_reltoken(&mhc->mhc_active_token); 592 } 593 return error; 594 } 595 596 const struct vmbus_message * 597 vmbus_msghc_wait_result(struct vmbus_softc *sc, struct vmbus_msghc *mh) 598 { 599 struct vmbus_msghc_ctx *mhc = sc->vmbus_msg_hc; 600 601 lwkt_gettoken(&mhc->mhc_active_token); 602 603 KASSERT(mhc->mhc_active == mh, ("msghc mismatch")); 604 while (mh->mh_resp == NULL) 605 tsleep(&mhc->mhc_active, 0, "wmsghc", 0); 606 mhc->mhc_active = NULL; 607 608 lwkt_reltoken(&mhc->mhc_active_token); 609 610 return mh->mh_resp; 611 } 612 613 void 614 vmbus_msghc_wakeup(struct vmbus_softc *sc, const struct vmbus_message *msg) 615 { 616 struct vmbus_msghc_ctx *mhc = sc->vmbus_msg_hc; 617 struct vmbus_msghc *mh; 618 619 lwkt_gettoken(&mhc->mhc_active_token); 620 621 mh = mhc->mhc_active; 622 KASSERT(mh != NULL, ("no pending msg hypercall")); 623 memcpy(&mh->mh_resp0, msg, sizeof(mh->mh_resp0)); 624 mh->mh_resp = &mh->mh_resp0; 625 626 lwkt_reltoken(&mhc->mhc_active_token); 627 wakeup(&mhc->mhc_active); 628 } 629 630 static int 631 vmbus_dma_alloc(struct vmbus_softc *sc) 632 { 633 bus_dma_tag_t parent_dtag; 634 uint8_t *evtflags; 635 int cpu; 636 637 parent_dtag = bus_get_dma_tag(sc->vmbus_dev); 638 for (cpu = 0; cpu < ncpus; ++cpu) { 639 struct vmbus_pcpu_data *psc = VMBUS_PCPU(sc, cpu); 640 641 /* 642 * Per-cpu messages and event flags. 643 */ 644 psc->message = hyperv_dmamem_alloc(parent_dtag, 645 PAGE_SIZE, 0, PAGE_SIZE, &psc->message_dma, 646 BUS_DMA_WAITOK | BUS_DMA_ZERO); 647 if (psc->message == NULL) 648 return ENOMEM; 649 650 psc->event_flags = hyperv_dmamem_alloc(parent_dtag, 651 PAGE_SIZE, 0, PAGE_SIZE, &psc->event_flags_dma, 652 BUS_DMA_WAITOK | BUS_DMA_ZERO); 653 if (psc->event_flags == NULL) 654 return ENOMEM; 655 } 656 657 evtflags = hyperv_dmamem_alloc(parent_dtag, PAGE_SIZE, 0, 658 PAGE_SIZE, &sc->vmbus_evtflags_dma, BUS_DMA_WAITOK | BUS_DMA_ZERO); 659 if (evtflags == NULL) 660 return ENOMEM; 661 sc->vmbus_rx_evtflags = (u_long *)evtflags; 662 sc->vmbus_tx_evtflags = (u_long *)(evtflags + (PAGE_SIZE / 2)); 663 sc->vmbus_evtflags = evtflags; 664 665 sc->vmbus_mnf1 = hyperv_dmamem_alloc(parent_dtag, PAGE_SIZE, 0, 666 PAGE_SIZE, &sc->vmbus_mnf1_dma, BUS_DMA_WAITOK | BUS_DMA_ZERO); 667 if (sc->vmbus_mnf1 == NULL) 668 return ENOMEM; 669 670 sc->vmbus_mnf2 = hyperv_dmamem_alloc(parent_dtag, PAGE_SIZE, 0, 671 PAGE_SIZE, &sc->vmbus_mnf2_dma, BUS_DMA_WAITOK | BUS_DMA_ZERO); 672 if (sc->vmbus_mnf2 == NULL) 673 return ENOMEM; 674 675 return 0; 676 } 677 678 static void 679 vmbus_dma_free(struct vmbus_softc *sc) 680 { 681 int cpu; 682 683 if (sc->vmbus_evtflags != NULL) { 684 hyperv_dmamem_free(&sc->vmbus_evtflags_dma, sc->vmbus_evtflags); 685 sc->vmbus_evtflags = NULL; 686 sc->vmbus_rx_evtflags = NULL; 687 sc->vmbus_tx_evtflags = NULL; 688 } 689 if (sc->vmbus_mnf1 != NULL) { 690 hyperv_dmamem_free(&sc->vmbus_mnf1_dma, sc->vmbus_mnf1); 691 sc->vmbus_mnf1 = NULL; 692 } 693 if (sc->vmbus_mnf2 != NULL) { 694 hyperv_dmamem_free(&sc->vmbus_mnf2_dma, sc->vmbus_mnf2); 695 sc->vmbus_mnf2 = NULL; 696 } 697 698 for (cpu = 0; cpu < ncpus; ++cpu) { 699 struct vmbus_pcpu_data *psc = VMBUS_PCPU(sc, cpu); 700 701 if (psc->message != NULL) { 702 hyperv_dmamem_free(&psc->message_dma, psc->message); 703 psc->message = NULL; 704 } 705 if (psc->event_flags != NULL) { 706 hyperv_dmamem_free(&psc->event_flags_dma, 707 psc->event_flags); 708 psc->event_flags = NULL; 709 } 710 } 711 } 712 713 static int 714 vmbus_intr_setup(struct vmbus_softc *sc) 715 { 716 device_t dev = sc->vmbus_dev; 717 device_t bus = device_get_parent(device_get_parent(dev)); 718 int rid, cpu; 719 720 rid = 0; 721 for (cpu = 0; cpu < ncpus; ++cpu) { 722 struct vmbus_pcpu_data *psc = VMBUS_PCPU(sc, cpu); 723 uint64_t msi_addr; 724 uint32_t msi_data; 725 int error; 726 727 error = PCIB_ALLOC_MSIX(bus, dev, &psc->intr_irq, cpu); 728 if (error) { 729 device_printf(dev, "alloc vector on cpu%d failed: %d\n", 730 cpu, error); 731 return ENXIO; 732 } 733 psc->intr_rid = ++rid; 734 735 psc->intr_res = BUS_ALLOC_RESOURCE(bus, dev, SYS_RES_IRQ, 736 &psc->intr_rid, psc->intr_irq, psc->intr_irq, 1, 737 RF_ACTIVE, cpu); 738 if (psc->intr_res == NULL) { 739 device_printf(dev, "alloc irq on cpu%d failed: %d\n", 740 cpu, error); 741 return ENXIO; 742 } 743 744 error = PCIB_MAP_MSI(bus, dev, rman_get_start(psc->intr_res), 745 &msi_addr, &msi_data, cpu); 746 if (error) { 747 device_printf(dev, "map irq on cpu%d failed: %d\n", 748 cpu, error); 749 return ENXIO; 750 } 751 psc->intr_vec = hyperv_msi2vector(msi_addr, msi_data); 752 753 if (bootverbose) { 754 device_printf(dev, "vector %d irq %d on cpu%d\n", 755 psc->intr_vec, psc->intr_irq, cpu); 756 } 757 758 ksnprintf(psc->intr_desc, sizeof(psc->intr_desc), "%s cpu%d", 759 device_get_nameunit(dev), cpu); 760 error = bus_setup_intr_descr(dev, psc->intr_res, INTR_MPSAFE, 761 vmbus_intr, psc, &psc->intr_hand, NULL, psc->intr_desc); 762 if (error) { 763 device_printf(dev, "setup intr on cpu%d failed: %d\n", 764 cpu, error); 765 return ENXIO; 766 } 767 } 768 return 0; 769 } 770 771 static void 772 vmbus_intr_teardown(struct vmbus_softc *sc) 773 { 774 device_t dev = sc->vmbus_dev; 775 device_t bus = device_get_parent(device_get_parent(dev)); 776 int cpu; 777 778 for (cpu = 0; cpu < ncpus; ++cpu) { 779 struct vmbus_pcpu_data *psc = VMBUS_PCPU(sc, cpu); 780 781 if (psc->intr_hand != NULL) { 782 bus_teardown_intr(dev, psc->intr_res, psc->intr_hand); 783 psc->intr_hand = NULL; 784 } 785 786 if (psc->intr_res != NULL) { 787 BUS_RELEASE_RESOURCE(bus, dev, SYS_RES_IRQ, 788 psc->intr_rid, psc->intr_res); 789 psc->intr_res = NULL; 790 } 791 792 if (psc->intr_rid != 0) { 793 PCIB_RELEASE_MSIX(bus, dev, psc->intr_irq, psc->cpuid); 794 psc->intr_rid = 0; 795 } 796 } 797 } 798 799 static void 800 vmbus_synic_setup(void *xsc) 801 { 802 struct vmbus_softc *sc = xsc; 803 struct vmbus_pcpu_data *psc = VMBUS_PCPU(sc, mycpuid); 804 uint64_t val, orig; 805 uint32_t sint; 806 807 if (hyperv_features & CPUID_HV_MSR_VP_INDEX) { 808 /* 809 * Save virtual processor id. 810 */ 811 psc->vcpuid = rdmsr(MSR_HV_VP_INDEX); 812 } else { 813 /* 814 * XXX 815 * Virtual processoor id is only used by a pretty broken 816 * channel selection code from storvsc. It's nothing 817 * critical even if CPUID_HV_MSR_VP_INDEX is not set; keep 818 * moving on. 819 */ 820 psc->vcpuid = mycpuid; 821 } 822 823 /* 824 * Setup the SynIC message. 825 */ 826 orig = rdmsr(MSR_HV_SIMP); 827 val = MSR_HV_SIMP_ENABLE | (orig & MSR_HV_SIMP_RSVD_MASK) | 828 ((psc->message_dma.hv_paddr >> PAGE_SHIFT) << MSR_HV_SIMP_PGSHIFT); 829 wrmsr(MSR_HV_SIMP, val); 830 831 /* 832 * Setup the SynIC event flags. 833 */ 834 orig = rdmsr(MSR_HV_SIEFP); 835 val = MSR_HV_SIEFP_ENABLE | (orig & MSR_HV_SIEFP_RSVD_MASK) | 836 ((psc->event_flags_dma.hv_paddr >> PAGE_SHIFT) << 837 MSR_HV_SIEFP_PGSHIFT); 838 wrmsr(MSR_HV_SIEFP, val); 839 840 841 /* 842 * Configure and unmask SINT for message and event flags. 843 */ 844 sint = MSR_HV_SINT0 + VMBUS_SINT_MESSAGE; 845 orig = rdmsr(sint); 846 val = psc->intr_vec | /* MSR_HV_SINT_AUTOEOI | notyet */ 847 (orig & MSR_HV_SINT_RSVD_MASK); 848 wrmsr(sint, val); 849 850 /* 851 * Configure and unmask SINT for timer. 852 */ 853 sint = MSR_HV_SINT0 + VMBUS_SINT_TIMER; 854 orig = rdmsr(sint); 855 val = XTIMER_OFFSET | /* MSR_HV_SINT_AUTOEOI | notyet */ 856 (orig & MSR_HV_SINT_RSVD_MASK); 857 wrmsr(sint, val); 858 859 /* 860 * All done; enable SynIC. 861 */ 862 orig = rdmsr(MSR_HV_SCONTROL); 863 val = MSR_HV_SCTRL_ENABLE | (orig & MSR_HV_SCTRL_RSVD_MASK); 864 wrmsr(MSR_HV_SCONTROL, val); 865 } 866 867 static void 868 vmbus_timer_stop(void *arg __unused) 869 { 870 for (;;) { 871 uint64_t val; 872 873 /* Stop counting, and this also implies disabling STIMER0 */ 874 wrmsr(MSR_HV_STIMER0_COUNT, 0); 875 876 val = rdmsr(MSR_HV_STIMER0_CONFIG); 877 if ((val & MSR_HV_STIMER_CFG_ENABLE) == 0) 878 break; 879 cpu_pause(); 880 } 881 } 882 883 static void 884 vmbus_timer_config(void *arg __unused) 885 { 886 /* 887 * Make sure that STIMER0 is really disabled before writing 888 * to STIMER0_CONFIG. 889 * 890 * "Writing to the configuration register of a timer that 891 * is already enabled may result in undefined behaviour." 892 */ 893 vmbus_timer_stop(arg); 894 wrmsr(MSR_HV_STIMER0_CONFIG, 895 MSR_HV_STIMER_CFG_AUTOEN | MSR_HV_STIMER0_CFG_SINT); 896 } 897 898 static void 899 vmbus_timer_msgintr(struct vmbus_pcpu_data *psc) 900 { 901 volatile struct vmbus_message *msg; 902 903 msg = psc->message + VMBUS_SINT_TIMER; 904 if (msg->msg_type == HYPERV_MSGTYPE_TIMER_EXPIRED) 905 vmbus_msg_reset(msg); 906 } 907 908 static void 909 vmbus_timer_restart(void *xsc) 910 { 911 struct vmbus_softc *sc = xsc; 912 struct vmbus_pcpu_data *psc = VMBUS_PCPU(sc, mycpuid); 913 914 crit_enter(); 915 vmbus_timer_msgintr(psc); 916 vmbus_timer_oneshot(psc, hyperv_tc64() + 1); 917 crit_exit(); 918 } 919 920 static void 921 vmbus_synic_teardown(void *arg __unused) 922 { 923 uint64_t orig; 924 uint32_t sint; 925 926 /* 927 * Disable SynIC. 928 */ 929 orig = rdmsr(MSR_HV_SCONTROL); 930 wrmsr(MSR_HV_SCONTROL, (orig & MSR_HV_SCTRL_RSVD_MASK)); 931 932 /* 933 * Mask message and event flags SINT. 934 */ 935 sint = MSR_HV_SINT0 + VMBUS_SINT_MESSAGE; 936 orig = rdmsr(sint); 937 wrmsr(sint, orig | MSR_HV_SINT_MASKED); 938 939 /* 940 * Mask timer SINT. 941 */ 942 sint = MSR_HV_SINT0 + VMBUS_SINT_TIMER; 943 orig = rdmsr(sint); 944 wrmsr(sint, orig | MSR_HV_SINT_MASKED); 945 946 /* 947 * Teardown SynIC message. 948 */ 949 orig = rdmsr(MSR_HV_SIMP); 950 wrmsr(MSR_HV_SIMP, (orig & MSR_HV_SIMP_RSVD_MASK)); 951 952 /* 953 * Teardown SynIC event flags. 954 */ 955 orig = rdmsr(MSR_HV_SIEFP); 956 wrmsr(MSR_HV_SIEFP, (orig & MSR_HV_SIEFP_RSVD_MASK)); 957 } 958 959 static int 960 vmbus_init_contact(struct vmbus_softc *sc, uint32_t version) 961 { 962 struct vmbus_chanmsg_init_contact *req; 963 const struct vmbus_chanmsg_version_resp *resp; 964 const struct vmbus_message *msg; 965 struct vmbus_msghc *mh; 966 int error, supp = 0; 967 968 mh = vmbus_msghc_get(sc, sizeof(*req)); 969 if (mh == NULL) 970 return ENXIO; 971 972 req = vmbus_msghc_dataptr(mh); 973 req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_INIT_CONTACT; 974 req->chm_ver = version; 975 req->chm_evtflags = sc->vmbus_evtflags_dma.hv_paddr; 976 req->chm_mnf1 = sc->vmbus_mnf1_dma.hv_paddr; 977 req->chm_mnf2 = sc->vmbus_mnf2_dma.hv_paddr; 978 979 error = vmbus_msghc_exec(sc, mh); 980 if (error) { 981 vmbus_msghc_put(sc, mh); 982 return error; 983 } 984 985 msg = vmbus_msghc_wait_result(sc, mh); 986 resp = (const struct vmbus_chanmsg_version_resp *)msg->msg_data; 987 supp = resp->chm_supp; 988 989 vmbus_msghc_put(sc, mh); 990 991 return (supp ? 0 : EOPNOTSUPP); 992 } 993 994 static int 995 vmbus_init(struct vmbus_softc *sc) 996 { 997 int i; 998 999 for (i = 0; i < nitems(vmbus_version); ++i) { 1000 int error; 1001 1002 error = vmbus_init_contact(sc, vmbus_version[i]); 1003 if (!error) { 1004 sc->vmbus_version = vmbus_version[i]; 1005 device_printf(sc->vmbus_dev, "version %u.%u\n", 1006 (sc->vmbus_version >> 16), 1007 (sc->vmbus_version & 0xffff)); 1008 return 0; 1009 } 1010 } 1011 return ENXIO; 1012 } 1013 1014 static void 1015 vmbus_chan_msgproc(struct vmbus_softc *sc, const struct vmbus_message *msg) 1016 { 1017 const struct vmbus_chanmsg_hdr *hdr; 1018 1019 hdr = (const struct vmbus_chanmsg_hdr *)msg->msg_data; 1020 1021 /* TODO */ 1022 if (hdr->chm_type == VMBUS_CHANMSG_TYPE_VERSION_RESP) 1023 vmbus_msghc_wakeup(sc, msg); 1024 } 1025