1 /*- 2 * Copyright (c) 2016 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: John Baldwin <jhb@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_inet.h" 32 #include "opt_inet6.h" 33 34 #include <sys/param.h> 35 #include <sys/bus.h> 36 #include <sys/conf.h> 37 #include <sys/kernel.h> 38 #include <sys/module.h> 39 #include <sys/priv.h> 40 #include <dev/pci/pcivar.h> 41 #if defined(__i386__) || defined(__amd64__) 42 #include <vm/vm.h> 43 #include <vm/pmap.h> 44 #endif 45 46 #include "common/common.h" 47 #include "common/t4_regs.h" 48 #include "t4_ioctl.h" 49 #include "t4_mp_ring.h" 50 51 /* 52 * Some notes: 53 * 54 * The Virtual Interfaces are connected to an internal switch on the chip 55 * which allows VIs attached to the same port to talk to each other even when 56 * the port link is down. As a result, we might want to always report a 57 * VF's link as being "up". 58 * 59 * XXX: Add a TUNABLE and possible per-device sysctl for this? 60 */ 61 62 struct intrs_and_queues { 63 uint16_t intr_type; /* MSI, or MSI-X */ 64 uint16_t nirq; /* Total # of vectors */ 65 uint16_t intr_flags; /* Interrupt flags for each port */ 66 uint16_t ntxq; /* # of NIC txq's for each port */ 67 uint16_t nrxq; /* # of NIC rxq's for each port */ 68 }; 69 70 struct { 71 uint16_t device; 72 char *desc; 73 } t4vf_pciids[] = { 74 {0x4800, "Chelsio T440-dbg VF"}, 75 {0x4801, "Chelsio T420-CR VF"}, 76 {0x4802, "Chelsio T422-CR VF"}, 77 {0x4803, "Chelsio T440-CR VF"}, 78 {0x4804, "Chelsio T420-BCH VF"}, 79 {0x4805, "Chelsio T440-BCH VF"}, 80 {0x4806, "Chelsio T440-CH VF"}, 81 {0x4807, "Chelsio T420-SO VF"}, 82 {0x4808, "Chelsio T420-CX VF"}, 83 {0x4809, "Chelsio T420-BT VF"}, 84 {0x480a, "Chelsio T404-BT VF"}, 85 {0x480e, "Chelsio T440-LP-CR VF"}, 86 }, t5vf_pciids[] = { 87 {0x5800, "Chelsio T580-dbg VF"}, 88 {0x5801, "Chelsio T520-CR VF"}, /* 2 x 10G */ 89 {0x5802, "Chelsio T522-CR VF"}, /* 2 x 10G, 2 X 1G */ 90 {0x5803, "Chelsio T540-CR VF"}, /* 4 x 10G */ 91 {0x5807, "Chelsio T520-SO VF"}, /* 2 x 10G, nomem */ 92 {0x5809, "Chelsio T520-BT VF"}, /* 2 x 10GBaseT */ 93 {0x580a, "Chelsio T504-BT VF"}, /* 4 x 1G */ 94 {0x580d, "Chelsio T580-CR VF"}, /* 2 x 40G */ 95 {0x580e, "Chelsio T540-LP-CR VF"}, /* 4 x 10G */ 96 {0x5810, "Chelsio T580-LP-CR VF"}, /* 2 x 40G */ 97 {0x5811, "Chelsio T520-LL-CR VF"}, /* 2 x 10G */ 98 {0x5812, "Chelsio T560-CR VF"}, /* 1 x 40G, 2 x 10G */ 99 {0x5814, "Chelsio T580-LP-SO-CR VF"}, /* 2 x 40G, nomem */ 100 {0x5815, "Chelsio T502-BT VF"}, /* 2 x 1G */ 101 #ifdef notyet 102 {0x5804, "Chelsio T520-BCH VF"}, 103 {0x5805, "Chelsio T540-BCH VF"}, 104 {0x5806, "Chelsio T540-CH VF"}, 105 {0x5808, "Chelsio T520-CX VF"}, 106 {0x580b, "Chelsio B520-SR VF"}, 107 {0x580c, "Chelsio B504-BT VF"}, 108 {0x580f, "Chelsio Amsterdam VF"}, 109 {0x5813, "Chelsio T580-CHR VF"}, 110 #endif 111 }, t6vf_pciids[] = { 112 {0x6800, "Chelsio T6-DBG-25 VF"}, /* 2 x 10/25G, debug */ 113 {0x6801, "Chelsio T6225-CR VF"}, /* 2 x 10/25G */ 114 {0x6802, "Chelsio T6225-SO-CR VF"}, /* 2 x 10/25G, nomem */ 115 {0x6803, "Chelsio T6425-CR VF"}, /* 4 x 10/25G */ 116 {0x6804, "Chelsio T6425-SO-CR VF"}, /* 4 x 10/25G, nomem */ 117 {0x6805, "Chelsio T6225-OCP-SO VF"}, /* 2 x 10/25G, nomem */ 118 {0x6806, "Chelsio T62100-OCP-SO VF"}, /* 2 x 40/50/100G, nomem */ 119 {0x6807, "Chelsio T62100-LP-CR VF"}, /* 2 x 40/50/100G */ 120 {0x6808, "Chelsio T62100-SO-CR VF"}, /* 2 x 40/50/100G, nomem */ 121 {0x6809, "Chelsio T6210-BT VF"}, /* 2 x 10GBASE-T */ 122 {0x680d, "Chelsio T62100-CR VF"}, /* 2 x 40/50/100G */ 123 {0x6810, "Chelsio T6-DBG-100 VF"}, /* 2 x 40/50/100G, debug */ 124 {0x6811, "Chelsio T6225-LL-CR VF"}, /* 2 x 10/25G */ 125 {0x6814, "Chelsio T61100-OCP-SO VF"}, /* 1 x 40/50/100G, nomem */ 126 {0x6815, "Chelsio T6201-BT VF"}, /* 2 x 1000BASE-T */ 127 128 /* Custom */ 129 {0x6880, "Chelsio T6225 80 VF"}, 130 {0x6881, "Chelsio T62100 81 VF"}, 131 }; 132 133 static d_ioctl_t t4vf_ioctl; 134 135 static struct cdevsw t4vf_cdevsw = { 136 .d_version = D_VERSION, 137 .d_ioctl = t4vf_ioctl, 138 .d_name = "t4vf", 139 }; 140 141 static int 142 t4vf_probe(device_t dev) 143 { 144 uint16_t d; 145 size_t i; 146 147 d = pci_get_device(dev); 148 for (i = 0; i < nitems(t4vf_pciids); i++) { 149 if (d == t4vf_pciids[i].device) { 150 device_set_desc(dev, t4vf_pciids[i].desc); 151 return (BUS_PROBE_DEFAULT); 152 } 153 } 154 return (ENXIO); 155 } 156 157 static int 158 t5vf_probe(device_t dev) 159 { 160 uint16_t d; 161 size_t i; 162 163 d = pci_get_device(dev); 164 for (i = 0; i < nitems(t5vf_pciids); i++) { 165 if (d == t5vf_pciids[i].device) { 166 device_set_desc(dev, t5vf_pciids[i].desc); 167 return (BUS_PROBE_DEFAULT); 168 } 169 } 170 return (ENXIO); 171 } 172 173 static int 174 t6vf_probe(device_t dev) 175 { 176 uint16_t d; 177 size_t i; 178 179 d = pci_get_device(dev); 180 for (i = 0; i < nitems(t6vf_pciids); i++) { 181 if (d == t6vf_pciids[i].device) { 182 device_set_desc(dev, t6vf_pciids[i].desc); 183 return (BUS_PROBE_DEFAULT); 184 } 185 } 186 return (ENXIO); 187 } 188 189 #define FW_PARAM_DEV(param) \ 190 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 191 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 192 #define FW_PARAM_PFVF(param) \ 193 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 194 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 195 196 static int 197 get_params__pre_init(struct adapter *sc) 198 { 199 int rc; 200 uint32_t param[3], val[3]; 201 202 param[0] = FW_PARAM_DEV(FWREV); 203 param[1] = FW_PARAM_DEV(TPREV); 204 param[2] = FW_PARAM_DEV(CCLK); 205 rc = -t4vf_query_params(sc, nitems(param), param, val); 206 if (rc != 0) { 207 device_printf(sc->dev, 208 "failed to query parameters (pre_init): %d.\n", rc); 209 return (rc); 210 } 211 212 sc->params.fw_vers = val[0]; 213 sc->params.tp_vers = val[1]; 214 sc->params.vpd.cclk = val[2]; 215 216 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 217 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 218 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 219 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 220 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 221 222 snprintf(sc->tp_version, sizeof(sc->tp_version), "%u.%u.%u.%u", 223 G_FW_HDR_FW_VER_MAJOR(sc->params.tp_vers), 224 G_FW_HDR_FW_VER_MINOR(sc->params.tp_vers), 225 G_FW_HDR_FW_VER_MICRO(sc->params.tp_vers), 226 G_FW_HDR_FW_VER_BUILD(sc->params.tp_vers)); 227 228 return (0); 229 } 230 231 static int 232 get_params__post_init(struct adapter *sc) 233 { 234 int rc; 235 236 rc = -t4vf_get_sge_params(sc); 237 if (rc != 0) { 238 device_printf(sc->dev, 239 "unable to retrieve adapter SGE parameters: %d\n", rc); 240 return (rc); 241 } 242 243 rc = -t4vf_get_rss_glb_config(sc); 244 if (rc != 0) { 245 device_printf(sc->dev, 246 "unable to retrieve adapter RSS parameters: %d\n", rc); 247 return (rc); 248 } 249 if (sc->params.rss.mode != FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL) { 250 device_printf(sc->dev, 251 "unable to operate with global RSS mode %d\n", 252 sc->params.rss.mode); 253 return (EINVAL); 254 } 255 256 rc = t4_read_chip_settings(sc); 257 if (rc != 0) 258 return (rc); 259 260 /* 261 * Grab our Virtual Interface resource allocation, extract the 262 * features that we're interested in and do a bit of sanity testing on 263 * what we discover. 264 */ 265 rc = -t4vf_get_vfres(sc); 266 if (rc != 0) { 267 device_printf(sc->dev, 268 "unable to get virtual interface resources: %d\n", rc); 269 return (rc); 270 } 271 272 /* 273 * Check for various parameter sanity issues. 274 */ 275 if (sc->params.vfres.pmask == 0) { 276 device_printf(sc->dev, "no port access configured/usable!\n"); 277 return (EINVAL); 278 } 279 if (sc->params.vfres.nvi == 0) { 280 device_printf(sc->dev, 281 "no virtual interfaces configured/usable!\n"); 282 return (EINVAL); 283 } 284 sc->params.portvec = sc->params.vfres.pmask; 285 286 return (0); 287 } 288 289 static int 290 set_params__post_init(struct adapter *sc) 291 { 292 uint32_t param, val; 293 294 /* ask for encapsulated CPLs */ 295 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 296 val = 1; 297 (void)t4vf_set_params(sc, 1, ¶m, &val); 298 299 return (0); 300 } 301 302 #undef FW_PARAM_PFVF 303 #undef FW_PARAM_DEV 304 305 static int 306 cfg_itype_and_nqueues(struct adapter *sc, struct intrs_and_queues *iaq) 307 { 308 struct vf_resources *vfres; 309 int nrxq, ntxq, nports; 310 int itype, iq_avail, navail, rc; 311 312 /* 313 * Figure out the layout of queues across our VIs and ensure 314 * we can allocate enough interrupts for our layout. 315 */ 316 vfres = &sc->params.vfres; 317 nports = sc->params.nports; 318 bzero(iaq, sizeof(*iaq)); 319 320 for (itype = INTR_MSIX; itype != 0; itype >>= 1) { 321 if (itype == INTR_INTX) 322 continue; 323 324 if (itype == INTR_MSIX) 325 navail = pci_msix_count(sc->dev); 326 else 327 navail = pci_msi_count(sc->dev); 328 329 if (navail == 0) 330 continue; 331 332 iaq->intr_type = itype; 333 iaq->intr_flags = 0; 334 335 /* 336 * XXX: The Linux driver reserves an Ingress Queue for 337 * forwarded interrupts when using MSI (but not MSI-X). 338 * It seems it just always asks for 2 interrupts and 339 * forwards all rxqs to the forwarded interrupt. 340 * 341 * We must reserve one IRQ for the for the firmware 342 * event queue. 343 * 344 * Every rxq requires an ingress queue with a free 345 * list and interrupts and an egress queue. Every txq 346 * requires an ETH egress queue. 347 */ 348 iaq->nirq = T4VF_EXTRA_INTR; 349 350 /* 351 * First, determine how many queues we can allocate. 352 * Start by finding the upper bound on rxqs from the 353 * limit on ingress queues. 354 */ 355 iq_avail = vfres->niqflint - iaq->nirq; 356 if (iq_avail < nports) { 357 device_printf(sc->dev, 358 "Not enough ingress queues (%d) for %d ports\n", 359 vfres->niqflint, nports); 360 return (ENXIO); 361 } 362 363 /* 364 * Try to honor the cap on interrupts. If there aren't 365 * enough interrupts for at least one interrupt per 366 * port, then don't bother, we will just forward all 367 * interrupts to one interrupt in that case. 368 */ 369 if (iaq->nirq + nports <= navail) { 370 if (iq_avail > navail - iaq->nirq) 371 iq_avail = navail - iaq->nirq; 372 } 373 374 nrxq = nports * t4_nrxq; 375 if (nrxq > iq_avail) { 376 /* 377 * Too many ingress queues. Use what we can. 378 */ 379 nrxq = (iq_avail / nports) * nports; 380 } 381 KASSERT(nrxq <= iq_avail, ("too many ingress queues")); 382 383 /* 384 * Next, determine the upper bound on txqs from the limit 385 * on ETH queues. 386 */ 387 if (vfres->nethctrl < nports) { 388 device_printf(sc->dev, 389 "Not enough ETH queues (%d) for %d ports\n", 390 vfres->nethctrl, nports); 391 return (ENXIO); 392 } 393 394 ntxq = nports * t4_ntxq; 395 if (ntxq > vfres->nethctrl) { 396 /* 397 * Too many ETH queues. Use what we can. 398 */ 399 ntxq = (vfres->nethctrl / nports) * nports; 400 } 401 KASSERT(ntxq <= vfres->nethctrl, ("too many ETH queues")); 402 403 /* 404 * Finally, ensure we have enough egress queues. 405 */ 406 if (vfres->neq < nports * 2) { 407 device_printf(sc->dev, 408 "Not enough egress queues (%d) for %d ports\n", 409 vfres->neq, nports); 410 return (ENXIO); 411 } 412 if (nrxq + ntxq > vfres->neq) { 413 /* Just punt and use 1 for everything. */ 414 nrxq = ntxq = nports; 415 } 416 KASSERT(nrxq <= iq_avail, ("too many ingress queues")); 417 KASSERT(ntxq <= vfres->nethctrl, ("too many ETH queues")); 418 KASSERT(nrxq + ntxq <= vfres->neq, ("too many egress queues")); 419 420 /* 421 * Do we have enough interrupts? For MSI the interrupts 422 * have to be a power of 2 as well. 423 */ 424 iaq->nirq += nrxq; 425 iaq->ntxq = ntxq; 426 iaq->nrxq = nrxq; 427 if (iaq->nirq <= navail && 428 (itype != INTR_MSI || powerof2(iaq->nirq))) { 429 navail = iaq->nirq; 430 if (itype == INTR_MSIX) 431 rc = pci_alloc_msix(sc->dev, &navail); 432 else 433 rc = pci_alloc_msi(sc->dev, &navail); 434 if (rc != 0) { 435 device_printf(sc->dev, 436 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 437 itype, rc, iaq->nirq, navail); 438 return (rc); 439 } 440 if (navail == iaq->nirq) { 441 iaq->intr_flags = INTR_RXQ; 442 return (0); 443 } 444 pci_release_msi(sc->dev); 445 } 446 447 /* Fall back to a single interrupt. */ 448 iaq->nirq = 1; 449 navail = iaq->nirq; 450 if (itype == INTR_MSIX) 451 rc = pci_alloc_msix(sc->dev, &navail); 452 else 453 rc = pci_alloc_msi(sc->dev, &navail); 454 if (rc != 0) 455 device_printf(sc->dev, 456 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 457 itype, rc, iaq->nirq, navail); 458 iaq->intr_flags = 0; 459 return (rc); 460 } 461 462 device_printf(sc->dev, 463 "failed to find a usable interrupt type. " 464 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 465 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 466 467 return (ENXIO); 468 } 469 470 static int 471 t4vf_attach(device_t dev) 472 { 473 struct adapter *sc; 474 int rc = 0, i, j, rqidx, tqidx; 475 struct make_dev_args mda; 476 struct intrs_and_queues iaq; 477 struct sge *s; 478 479 sc = device_get_softc(dev); 480 sc->dev = dev; 481 pci_enable_busmaster(dev); 482 pci_set_max_read_req(dev, 4096); 483 sc->params.pci.mps = pci_get_max_payload(dev); 484 485 sc->flags |= IS_VF; 486 487 sc->sge_gts_reg = VF_SGE_REG(A_SGE_VF_GTS); 488 sc->sge_kdoorbell_reg = VF_SGE_REG(A_SGE_VF_KDOORBELL); 489 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 490 device_get_nameunit(dev)); 491 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 492 t4_add_adapter(sc); 493 494 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 495 TAILQ_INIT(&sc->sfl); 496 callout_init_mtx(&sc->sfl_callout, &sc->sfl_lock, 0); 497 498 mtx_init(&sc->reg_lock, "indirect register access", 0, MTX_DEF); 499 500 rc = t4_map_bars_0_and_4(sc); 501 if (rc != 0) 502 goto done; /* error message displayed already */ 503 504 rc = -t4vf_prep_adapter(sc); 505 if (rc != 0) 506 goto done; 507 508 t4_init_devnames(sc); 509 if (sc->names == NULL) { 510 rc = ENOTSUP; 511 goto done; /* error message displayed already */ 512 } 513 514 /* 515 * Leave the 'pf' and 'mbox' values as zero. This ensures 516 * that various firmware messages do not set the fields which 517 * is the correct thing to do for a VF. 518 */ 519 520 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 521 522 make_dev_args_init(&mda); 523 mda.mda_devsw = &t4vf_cdevsw; 524 mda.mda_uid = UID_ROOT; 525 mda.mda_gid = GID_WHEEL; 526 mda.mda_mode = 0600; 527 mda.mda_si_drv1 = sc; 528 rc = make_dev_s(&mda, &sc->cdev, "%s", device_get_nameunit(dev)); 529 if (rc != 0) 530 device_printf(dev, "failed to create nexus char device: %d.\n", 531 rc); 532 533 #if defined(__i386__) 534 if ((cpu_feature & CPUID_CX8) == 0) { 535 device_printf(dev, "64 bit atomics not available.\n"); 536 rc = ENOTSUP; 537 goto done; 538 } 539 #endif 540 541 /* 542 * Some environments do not properly handle PCIE FLRs -- e.g. in Linux 543 * 2.6.31 and later we can't call pci_reset_function() in order to 544 * issue an FLR because of a self- deadlock on the device semaphore. 545 * Meanwhile, the OS infrastructure doesn't issue FLRs in all the 546 * cases where they're needed -- for instance, some versions of KVM 547 * fail to reset "Assigned Devices" when the VM reboots. Therefore we 548 * use the firmware based reset in order to reset any per function 549 * state. 550 */ 551 rc = -t4vf_fw_reset(sc); 552 if (rc != 0) { 553 device_printf(dev, "FW reset failed: %d\n", rc); 554 goto done; 555 } 556 sc->flags |= FW_OK; 557 558 /* 559 * Grab basic operational parameters. These will predominantly have 560 * been set up by the Physical Function Driver or will be hard coded 561 * into the adapter. We just have to live with them ... Note that 562 * we _must_ get our VPD parameters before our SGE parameters because 563 * we need to know the adapter's core clock from the VPD in order to 564 * properly decode the SGE Timer Values. 565 */ 566 rc = get_params__pre_init(sc); 567 if (rc != 0) 568 goto done; /* error message displayed already */ 569 rc = get_params__post_init(sc); 570 if (rc != 0) 571 goto done; /* error message displayed already */ 572 573 rc = set_params__post_init(sc); 574 if (rc != 0) 575 goto done; /* error message displayed already */ 576 577 rc = t4_map_bar_2(sc); 578 if (rc != 0) 579 goto done; /* error message displayed already */ 580 581 rc = t4_create_dma_tag(sc); 582 if (rc != 0) 583 goto done; /* error message displayed already */ 584 585 /* 586 * The number of "ports" which we support is equal to the number of 587 * Virtual Interfaces with which we've been provisioned. 588 */ 589 sc->params.nports = imin(sc->params.vfres.nvi, MAX_NPORTS); 590 591 /* 592 * We may have been provisioned with more VIs than the number of 593 * ports we're allowed to access (our Port Access Rights Mask). 594 * Just use a single VI for each port. 595 */ 596 sc->params.nports = imin(sc->params.nports, 597 bitcount32(sc->params.vfres.pmask)); 598 599 #ifdef notyet 600 /* 601 * XXX: The Linux VF driver will lower nports if it thinks there 602 * are too few resources in vfres (niqflint, nethctrl, neq). 603 */ 604 #endif 605 606 /* 607 * First pass over all the ports - allocate VIs and initialize some 608 * basic parameters like mac address, port type, etc. 609 */ 610 for_each_port(sc, i) { 611 struct port_info *pi; 612 613 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 614 sc->port[i] = pi; 615 616 /* These must be set before t4_port_init */ 617 pi->adapter = sc; 618 pi->port_id = i; 619 pi->nvi = 1; 620 pi->vi = malloc(sizeof(struct vi_info) * pi->nvi, M_CXGBE, 621 M_ZERO | M_WAITOK); 622 623 /* 624 * Allocate the "main" VI and initialize parameters 625 * like mac addr. 626 */ 627 rc = -t4_port_init(sc, sc->mbox, sc->pf, 0, i); 628 if (rc != 0) { 629 device_printf(dev, "unable to initialize port %d: %d\n", 630 i, rc); 631 free(pi->vi, M_CXGBE); 632 free(pi, M_CXGBE); 633 sc->port[i] = NULL; 634 goto done; 635 } 636 637 /* No t4_link_start. */ 638 639 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 640 device_get_nameunit(dev), i); 641 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 642 sc->chan_map[pi->tx_chan] = i; 643 644 pi->dev = device_add_child(dev, sc->names->vf_ifnet_name, -1); 645 if (pi->dev == NULL) { 646 device_printf(dev, 647 "failed to add device for port %d.\n", i); 648 rc = ENXIO; 649 goto done; 650 } 651 pi->vi[0].dev = pi->dev; 652 device_set_softc(pi->dev, pi); 653 } 654 655 /* 656 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 657 */ 658 rc = cfg_itype_and_nqueues(sc, &iaq); 659 if (rc != 0) 660 goto done; /* error message displayed already */ 661 662 sc->intr_type = iaq.intr_type; 663 sc->intr_count = iaq.nirq; 664 665 s = &sc->sge; 666 s->nrxq = sc->params.nports * iaq.nrxq; 667 s->ntxq = sc->params.nports * iaq.ntxq; 668 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 669 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 670 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 671 672 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 673 M_ZERO | M_WAITOK); 674 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 675 M_ZERO | M_WAITOK); 676 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 677 M_ZERO | M_WAITOK); 678 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 679 M_ZERO | M_WAITOK); 680 681 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 682 M_ZERO | M_WAITOK); 683 684 /* 685 * Second pass over the ports. This time we know the number of rx and 686 * tx queues that each port should get. 687 */ 688 rqidx = tqidx = 0; 689 for_each_port(sc, i) { 690 struct port_info *pi = sc->port[i]; 691 struct vi_info *vi; 692 693 if (pi == NULL) 694 continue; 695 696 for_each_vi(pi, j, vi) { 697 vi->pi = pi; 698 vi->qsize_rxq = t4_qsize_rxq; 699 vi->qsize_txq = t4_qsize_txq; 700 701 vi->first_rxq = rqidx; 702 vi->first_txq = tqidx; 703 vi->tmr_idx = t4_tmr_idx; 704 vi->pktc_idx = t4_pktc_idx; 705 vi->flags |= iaq.intr_flags & INTR_RXQ; 706 vi->nrxq = j == 0 ? iaq.nrxq: 1; 707 vi->ntxq = j == 0 ? iaq.ntxq: 1; 708 709 rqidx += vi->nrxq; 710 tqidx += vi->ntxq; 711 712 vi->rsrv_noflowq = 0; 713 } 714 } 715 716 rc = t4_setup_intr_handlers(sc); 717 if (rc != 0) { 718 device_printf(dev, 719 "failed to setup interrupt handlers: %d\n", rc); 720 goto done; 721 } 722 723 rc = bus_generic_attach(dev); 724 if (rc != 0) { 725 device_printf(dev, 726 "failed to attach all child ports: %d\n", rc); 727 goto done; 728 } 729 730 device_printf(dev, 731 "%d ports, %d %s interrupt%s, %d eq, %d iq\n", 732 sc->params.nports, sc->intr_count, sc->intr_type == INTR_MSIX ? 733 "MSI-X" : "MSI", sc->intr_count > 1 ? "s" : "", sc->sge.neq, 734 sc->sge.niq); 735 736 done: 737 if (rc != 0) 738 t4_detach_common(dev); 739 else 740 t4_sysctls(sc); 741 742 return (rc); 743 } 744 745 static void 746 get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 747 { 748 749 /* 0x3f is used as the revision for VFs. */ 750 regs->version = chip_id(sc) | (0x3f << 10); 751 t4_get_regs(sc, buf, regs->len); 752 } 753 754 static void 755 t4_clr_vi_stats(struct adapter *sc) 756 { 757 int reg; 758 759 for (reg = A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L; 760 reg <= A_MPS_VF_STAT_RX_VF_ERR_FRAMES_H; reg += 4) 761 t4_write_reg(sc, VF_MPS_REG(reg), 0); 762 } 763 764 static int 765 t4vf_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 766 struct thread *td) 767 { 768 int rc; 769 struct adapter *sc = dev->si_drv1; 770 771 rc = priv_check(td, PRIV_DRIVER); 772 if (rc != 0) 773 return (rc); 774 775 switch (cmd) { 776 case CHELSIO_T4_GETREG: { 777 struct t4_reg *edata = (struct t4_reg *)data; 778 779 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 780 return (EFAULT); 781 782 if (edata->size == 4) 783 edata->val = t4_read_reg(sc, edata->addr); 784 else if (edata->size == 8) 785 edata->val = t4_read_reg64(sc, edata->addr); 786 else 787 return (EINVAL); 788 789 break; 790 } 791 case CHELSIO_T4_SETREG: { 792 struct t4_reg *edata = (struct t4_reg *)data; 793 794 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 795 return (EFAULT); 796 797 if (edata->size == 4) { 798 if (edata->val & 0xffffffff00000000) 799 return (EINVAL); 800 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 801 } else if (edata->size == 8) 802 t4_write_reg64(sc, edata->addr, edata->val); 803 else 804 return (EINVAL); 805 break; 806 } 807 case CHELSIO_T4_REGDUMP: { 808 struct t4_regdump *regs = (struct t4_regdump *)data; 809 int reglen = t4_get_regs_len(sc); 810 uint8_t *buf; 811 812 if (regs->len < reglen) { 813 regs->len = reglen; /* hint to the caller */ 814 return (ENOBUFS); 815 } 816 817 regs->len = reglen; 818 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 819 get_regs(sc, regs, buf); 820 rc = copyout(buf, regs->data, reglen); 821 free(buf, M_CXGBE); 822 break; 823 } 824 case CHELSIO_T4_CLEAR_STATS: { 825 int i, v; 826 u_int port_id = *(uint32_t *)data; 827 struct port_info *pi; 828 struct vi_info *vi; 829 830 if (port_id >= sc->params.nports) 831 return (EINVAL); 832 pi = sc->port[port_id]; 833 834 /* MAC stats */ 835 pi->tx_parse_error = 0; 836 t4_clr_vi_stats(sc); 837 838 /* 839 * Since this command accepts a port, clear stats for 840 * all VIs on this port. 841 */ 842 for_each_vi(pi, v, vi) { 843 if (vi->flags & VI_INIT_DONE) { 844 struct sge_rxq *rxq; 845 struct sge_txq *txq; 846 847 for_each_rxq(vi, i, rxq) { 848 #if defined(INET) || defined(INET6) 849 rxq->lro.lro_queued = 0; 850 rxq->lro.lro_flushed = 0; 851 #endif 852 rxq->rxcsum = 0; 853 rxq->vlan_extraction = 0; 854 } 855 856 for_each_txq(vi, i, txq) { 857 txq->txcsum = 0; 858 txq->tso_wrs = 0; 859 txq->vlan_insertion = 0; 860 txq->imm_wrs = 0; 861 txq->sgl_wrs = 0; 862 txq->txpkt_wrs = 0; 863 txq->txpkts0_wrs = 0; 864 txq->txpkts1_wrs = 0; 865 txq->txpkts0_pkts = 0; 866 txq->txpkts1_pkts = 0; 867 mp_ring_reset_stats(txq->r); 868 } 869 } 870 } 871 break; 872 } 873 case CHELSIO_T4_SCHED_CLASS: 874 rc = t4_set_sched_class(sc, (struct t4_sched_params *)data); 875 break; 876 case CHELSIO_T4_SCHED_QUEUE: 877 rc = t4_set_sched_queue(sc, (struct t4_sched_queue *)data); 878 break; 879 default: 880 rc = ENOTTY; 881 } 882 883 return (rc); 884 } 885 886 static device_method_t t4vf_methods[] = { 887 DEVMETHOD(device_probe, t4vf_probe), 888 DEVMETHOD(device_attach, t4vf_attach), 889 DEVMETHOD(device_detach, t4_detach_common), 890 891 DEVMETHOD_END 892 }; 893 894 static driver_t t4vf_driver = { 895 "t4vf", 896 t4vf_methods, 897 sizeof(struct adapter) 898 }; 899 900 static device_method_t t5vf_methods[] = { 901 DEVMETHOD(device_probe, t5vf_probe), 902 DEVMETHOD(device_attach, t4vf_attach), 903 DEVMETHOD(device_detach, t4_detach_common), 904 905 DEVMETHOD_END 906 }; 907 908 static driver_t t5vf_driver = { 909 "t5vf", 910 t5vf_methods, 911 sizeof(struct adapter) 912 }; 913 914 static device_method_t t6vf_methods[] = { 915 DEVMETHOD(device_probe, t6vf_probe), 916 DEVMETHOD(device_attach, t4vf_attach), 917 DEVMETHOD(device_detach, t4_detach_common), 918 919 DEVMETHOD_END 920 }; 921 922 static driver_t t6vf_driver = { 923 "t6vf", 924 t6vf_methods, 925 sizeof(struct adapter) 926 }; 927 928 static driver_t cxgbev_driver = { 929 "cxgbev", 930 cxgbe_methods, 931 sizeof(struct port_info) 932 }; 933 934 static driver_t cxlv_driver = { 935 "cxlv", 936 cxgbe_methods, 937 sizeof(struct port_info) 938 }; 939 940 static driver_t ccv_driver = { 941 "ccv", 942 cxgbe_methods, 943 sizeof(struct port_info) 944 }; 945 946 static devclass_t t4vf_devclass, t5vf_devclass, t6vf_devclass; 947 static devclass_t cxgbev_devclass, cxlv_devclass, ccv_devclass; 948 949 DRIVER_MODULE(t4vf, pci, t4vf_driver, t4vf_devclass, 0, 0); 950 MODULE_VERSION(t4vf, 1); 951 MODULE_DEPEND(t4vf, t4nex, 1, 1, 1); 952 953 DRIVER_MODULE(t5vf, pci, t5vf_driver, t5vf_devclass, 0, 0); 954 MODULE_VERSION(t5vf, 1); 955 MODULE_DEPEND(t5vf, t5nex, 1, 1, 1); 956 957 DRIVER_MODULE(t6vf, pci, t6vf_driver, t6vf_devclass, 0, 0); 958 MODULE_VERSION(t6vf, 1); 959 MODULE_DEPEND(t6vf, t6nex, 1, 1, 1); 960 961 DRIVER_MODULE(cxgbev, t4vf, cxgbev_driver, cxgbev_devclass, 0, 0); 962 MODULE_VERSION(cxgbev, 1); 963 964 DRIVER_MODULE(cxlv, t5vf, cxlv_driver, cxlv_devclass, 0, 0); 965 MODULE_VERSION(cxlv, 1); 966 967 DRIVER_MODULE(ccv, t6vf, ccv_driver, ccv_devclass, 0, 0); 968 MODULE_VERSION(ccv, 1); 969