1 /************************************************************************** 2 SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 4 Copyright (c) 2007-2009, Chelsio Inc. 5 All rights reserved. 6 7 Redistribution and use in source and binary forms, with or without 8 modification, are permitted provided that the following conditions are met: 9 10 1. Redistributions of source code must retain the above copyright notice, 11 this list of conditions and the following disclaimer. 12 13 2. Neither the name of the Chelsio Corporation nor the names of its 14 contributors may be used to endorse or promote products derived from 15 this software without specific prior written permission. 16 17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 POSSIBILITY OF SUCH DAMAGE. 28 29 ***************************************************************************/ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include "opt_inet.h" 35 36 #include <sys/param.h> 37 #include <sys/systm.h> 38 #include <sys/kernel.h> 39 #include <sys/bus.h> 40 #include <sys/module.h> 41 #include <sys/pciio.h> 42 #include <sys/conf.h> 43 #include <machine/bus.h> 44 #include <machine/resource.h> 45 #include <sys/ktr.h> 46 #include <sys/rman.h> 47 #include <sys/ioccom.h> 48 #include <sys/mbuf.h> 49 #include <sys/linker.h> 50 #include <sys/firmware.h> 51 #include <sys/socket.h> 52 #include <sys/sockio.h> 53 #include <sys/smp.h> 54 #include <sys/sysctl.h> 55 #include <sys/syslog.h> 56 #include <sys/queue.h> 57 #include <sys/taskqueue.h> 58 #include <sys/proc.h> 59 60 #include <net/bpf.h> 61 #include <net/debugnet.h> 62 #include <net/ethernet.h> 63 #include <net/if.h> 64 #include <net/if_var.h> 65 #include <net/if_arp.h> 66 #include <net/if_dl.h> 67 #include <net/if_media.h> 68 #include <net/if_types.h> 69 #include <net/if_vlan_var.h> 70 71 #include <netinet/in_systm.h> 72 #include <netinet/in.h> 73 #include <netinet/if_ether.h> 74 #include <netinet/ip.h> 75 #include <netinet/ip.h> 76 #include <netinet/tcp.h> 77 #include <netinet/udp.h> 78 79 #include <dev/pci/pcireg.h> 80 #include <dev/pci/pcivar.h> 81 #include <dev/pci/pci_private.h> 82 83 #include <cxgb_include.h> 84 85 #ifdef PRIV_SUPPORTED 86 #include <sys/priv.h> 87 #endif 88 89 static int cxgb_setup_interrupts(adapter_t *); 90 static void cxgb_teardown_interrupts(adapter_t *); 91 static void cxgb_init(void *); 92 static int cxgb_init_locked(struct port_info *); 93 static int cxgb_uninit_locked(struct port_info *); 94 static int cxgb_uninit_synchronized(struct port_info *); 95 static int cxgb_ioctl(struct ifnet *, unsigned long, caddr_t); 96 static int cxgb_media_change(struct ifnet *); 97 static int cxgb_ifm_type(int); 98 static void cxgb_build_medialist(struct port_info *); 99 static void cxgb_media_status(struct ifnet *, struct ifmediareq *); 100 static uint64_t cxgb_get_counter(struct ifnet *, ift_counter); 101 static int setup_sge_qsets(adapter_t *); 102 static void cxgb_async_intr(void *); 103 static void cxgb_tick_handler(void *, int); 104 static void cxgb_tick(void *); 105 static void link_check_callout(void *); 106 static void check_link_status(void *, int); 107 static void setup_rss(adapter_t *sc); 108 static int alloc_filters(struct adapter *); 109 static int setup_hw_filters(struct adapter *); 110 static int set_filter(struct adapter *, int, const struct filter_info *); 111 static inline void mk_set_tcb_field(struct cpl_set_tcb_field *, unsigned int, 112 unsigned int, u64, u64); 113 static inline void set_tcb_field_ulp(struct cpl_set_tcb_field *, unsigned int, 114 unsigned int, u64, u64); 115 #ifdef TCP_OFFLOAD 116 static int cpl_not_handled(struct sge_qset *, struct rsp_desc *, struct mbuf *); 117 #endif 118 119 /* Attachment glue for the PCI controller end of the device. Each port of 120 * the device is attached separately, as defined later. 121 */ 122 static int cxgb_controller_probe(device_t); 123 static int cxgb_controller_attach(device_t); 124 static int cxgb_controller_detach(device_t); 125 static void cxgb_free(struct adapter *); 126 static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start, 127 unsigned int end); 128 static void cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf); 129 static int cxgb_get_regs_len(void); 130 static void touch_bars(device_t dev); 131 static void cxgb_update_mac_settings(struct port_info *p); 132 #ifdef TCP_OFFLOAD 133 static int toe_capability(struct port_info *, int); 134 #endif 135 136 /* Table for probing the cards. The desc field isn't actually used */ 137 struct cxgb_ident { 138 uint16_t vendor; 139 uint16_t device; 140 int index; 141 char *desc; 142 } cxgb_identifiers[] = { 143 {PCI_VENDOR_ID_CHELSIO, 0x0020, 0, "PE9000"}, 144 {PCI_VENDOR_ID_CHELSIO, 0x0021, 1, "T302E"}, 145 {PCI_VENDOR_ID_CHELSIO, 0x0022, 2, "T310E"}, 146 {PCI_VENDOR_ID_CHELSIO, 0x0023, 3, "T320X"}, 147 {PCI_VENDOR_ID_CHELSIO, 0x0024, 1, "T302X"}, 148 {PCI_VENDOR_ID_CHELSIO, 0x0025, 3, "T320E"}, 149 {PCI_VENDOR_ID_CHELSIO, 0x0026, 2, "T310X"}, 150 {PCI_VENDOR_ID_CHELSIO, 0x0030, 2, "T3B10"}, 151 {PCI_VENDOR_ID_CHELSIO, 0x0031, 3, "T3B20"}, 152 {PCI_VENDOR_ID_CHELSIO, 0x0032, 1, "T3B02"}, 153 {PCI_VENDOR_ID_CHELSIO, 0x0033, 4, "T3B04"}, 154 {PCI_VENDOR_ID_CHELSIO, 0x0035, 6, "T3C10"}, 155 {PCI_VENDOR_ID_CHELSIO, 0x0036, 3, "S320E-CR"}, 156 {PCI_VENDOR_ID_CHELSIO, 0x0037, 7, "N320E-G2"}, 157 {0, 0, 0, NULL} 158 }; 159 160 static device_method_t cxgb_controller_methods[] = { 161 DEVMETHOD(device_probe, cxgb_controller_probe), 162 DEVMETHOD(device_attach, cxgb_controller_attach), 163 DEVMETHOD(device_detach, cxgb_controller_detach), 164 165 DEVMETHOD_END 166 }; 167 168 static driver_t cxgb_controller_driver = { 169 "cxgbc", 170 cxgb_controller_methods, 171 sizeof(struct adapter) 172 }; 173 174 static int cxgbc_mod_event(module_t, int, void *); 175 176 DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgbc_mod_event, NULL); 177 MODULE_PNP_INFO("U16:vendor;U16:device", pci, cxgbc, cxgb_identifiers, 178 nitems(cxgb_identifiers) - 1); 179 MODULE_VERSION(cxgbc, 1); 180 MODULE_DEPEND(cxgbc, firmware, 1, 1, 1); 181 182 /* 183 * Attachment glue for the ports. Attachment is done directly to the 184 * controller device. 185 */ 186 static int cxgb_port_probe(device_t); 187 static int cxgb_port_attach(device_t); 188 static int cxgb_port_detach(device_t); 189 190 static device_method_t cxgb_port_methods[] = { 191 DEVMETHOD(device_probe, cxgb_port_probe), 192 DEVMETHOD(device_attach, cxgb_port_attach), 193 DEVMETHOD(device_detach, cxgb_port_detach), 194 { 0, 0 } 195 }; 196 197 static driver_t cxgb_port_driver = { 198 "cxgb", 199 cxgb_port_methods, 200 0 201 }; 202 203 static d_ioctl_t cxgb_extension_ioctl; 204 static d_open_t cxgb_extension_open; 205 static d_close_t cxgb_extension_close; 206 207 static struct cdevsw cxgb_cdevsw = { 208 .d_version = D_VERSION, 209 .d_flags = 0, 210 .d_open = cxgb_extension_open, 211 .d_close = cxgb_extension_close, 212 .d_ioctl = cxgb_extension_ioctl, 213 .d_name = "cxgb", 214 }; 215 216 DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, 0, 0); 217 MODULE_VERSION(cxgb, 1); 218 219 DEBUGNET_DEFINE(cxgb); 220 221 static struct mtx t3_list_lock; 222 static SLIST_HEAD(, adapter) t3_list; 223 #ifdef TCP_OFFLOAD 224 static struct mtx t3_uld_list_lock; 225 static SLIST_HEAD(, uld_info) t3_uld_list; 226 #endif 227 228 /* 229 * The driver uses the best interrupt scheme available on a platform in the 230 * order MSI-X, MSI, legacy pin interrupts. This parameter determines which 231 * of these schemes the driver may consider as follows: 232 * 233 * msi = 2: choose from among all three options 234 * msi = 1 : only consider MSI and pin interrupts 235 * msi = 0: force pin interrupts 236 */ 237 static int msi_allowed = 2; 238 239 SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 240 "CXGB driver parameters"); 241 SYSCTL_INT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0, 242 "MSI-X, MSI, INTx selector"); 243 244 /* 245 * The driver uses an auto-queue algorithm by default. 246 * To disable it and force a single queue-set per port, use multiq = 0 247 */ 248 static int multiq = 1; 249 SYSCTL_INT(_hw_cxgb, OID_AUTO, multiq, CTLFLAG_RDTUN, &multiq, 0, 250 "use min(ncpus/ports, 8) queue-sets per port"); 251 252 /* 253 * By default the driver will not update the firmware unless 254 * it was compiled against a newer version 255 * 256 */ 257 static int force_fw_update = 0; 258 SYSCTL_INT(_hw_cxgb, OID_AUTO, force_fw_update, CTLFLAG_RDTUN, &force_fw_update, 0, 259 "update firmware even if up to date"); 260 261 int cxgb_use_16k_clusters = -1; 262 SYSCTL_INT(_hw_cxgb, OID_AUTO, use_16k_clusters, CTLFLAG_RDTUN, 263 &cxgb_use_16k_clusters, 0, "use 16kB clusters for the jumbo queue "); 264 265 static int nfilters = -1; 266 SYSCTL_INT(_hw_cxgb, OID_AUTO, nfilters, CTLFLAG_RDTUN, 267 &nfilters, 0, "max number of entries in the filter table"); 268 269 enum { 270 MAX_TXQ_ENTRIES = 16384, 271 MAX_CTRL_TXQ_ENTRIES = 1024, 272 MAX_RSPQ_ENTRIES = 16384, 273 MAX_RX_BUFFERS = 16384, 274 MAX_RX_JUMBO_BUFFERS = 16384, 275 MIN_TXQ_ENTRIES = 4, 276 MIN_CTRL_TXQ_ENTRIES = 4, 277 MIN_RSPQ_ENTRIES = 32, 278 MIN_FL_ENTRIES = 32, 279 MIN_FL_JUMBO_ENTRIES = 32 280 }; 281 282 struct filter_info { 283 u32 sip; 284 u32 sip_mask; 285 u32 dip; 286 u16 sport; 287 u16 dport; 288 u32 vlan:12; 289 u32 vlan_prio:3; 290 u32 mac_hit:1; 291 u32 mac_idx:4; 292 u32 mac_vld:1; 293 u32 pkt_type:2; 294 u32 report_filter_id:1; 295 u32 pass:1; 296 u32 rss:1; 297 u32 qset:3; 298 u32 locked:1; 299 u32 valid:1; 300 }; 301 302 enum { FILTER_NO_VLAN_PRI = 7 }; 303 304 #define EEPROM_MAGIC 0x38E2F10C 305 306 #define PORT_MASK ((1 << MAX_NPORTS) - 1) 307 308 309 static int set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset); 310 311 312 static __inline char 313 t3rev2char(struct adapter *adapter) 314 { 315 char rev = 'z'; 316 317 switch(adapter->params.rev) { 318 case T3_REV_A: 319 rev = 'a'; 320 break; 321 case T3_REV_B: 322 case T3_REV_B2: 323 rev = 'b'; 324 break; 325 case T3_REV_C: 326 rev = 'c'; 327 break; 328 } 329 return rev; 330 } 331 332 static struct cxgb_ident * 333 cxgb_get_ident(device_t dev) 334 { 335 struct cxgb_ident *id; 336 337 for (id = cxgb_identifiers; id->desc != NULL; id++) { 338 if ((id->vendor == pci_get_vendor(dev)) && 339 (id->device == pci_get_device(dev))) { 340 return (id); 341 } 342 } 343 return (NULL); 344 } 345 346 static const struct adapter_info * 347 cxgb_get_adapter_info(device_t dev) 348 { 349 struct cxgb_ident *id; 350 const struct adapter_info *ai; 351 352 id = cxgb_get_ident(dev); 353 if (id == NULL) 354 return (NULL); 355 356 ai = t3_get_adapter_info(id->index); 357 358 return (ai); 359 } 360 361 static int 362 cxgb_controller_probe(device_t dev) 363 { 364 const struct adapter_info *ai; 365 char *ports, buf[80]; 366 int nports; 367 368 ai = cxgb_get_adapter_info(dev); 369 if (ai == NULL) 370 return (ENXIO); 371 372 nports = ai->nports0 + ai->nports1; 373 if (nports == 1) 374 ports = "port"; 375 else 376 ports = "ports"; 377 378 snprintf(buf, sizeof(buf), "%s, %d %s", ai->desc, nports, ports); 379 device_set_desc_copy(dev, buf); 380 return (BUS_PROBE_DEFAULT); 381 } 382 383 #define FW_FNAME "cxgb_t3fw" 384 #define TPEEPROM_NAME "cxgb_t3%c_tp_eeprom" 385 #define TPSRAM_NAME "cxgb_t3%c_protocol_sram" 386 387 static int 388 upgrade_fw(adapter_t *sc) 389 { 390 const struct firmware *fw; 391 int status; 392 u32 vers; 393 394 if ((fw = firmware_get(FW_FNAME)) == NULL) { 395 device_printf(sc->dev, "Could not find firmware image %s\n", FW_FNAME); 396 return (ENOENT); 397 } else 398 device_printf(sc->dev, "installing firmware on card\n"); 399 status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize); 400 401 if (status != 0) { 402 device_printf(sc->dev, "failed to install firmware: %d\n", 403 status); 404 } else { 405 t3_get_fw_version(sc, &vers); 406 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d", 407 G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers), 408 G_FW_VERSION_MICRO(vers)); 409 } 410 411 firmware_put(fw, FIRMWARE_UNLOAD); 412 413 return (status); 414 } 415 416 /* 417 * The cxgb_controller_attach function is responsible for the initial 418 * bringup of the device. Its responsibilities include: 419 * 420 * 1. Determine if the device supports MSI or MSI-X. 421 * 2. Allocate bus resources so that we can access the Base Address Register 422 * 3. Create and initialize mutexes for the controller and its control 423 * logic such as SGE and MDIO. 424 * 4. Call hardware specific setup routine for the adapter as a whole. 425 * 5. Allocate the BAR for doing MSI-X. 426 * 6. Setup the line interrupt iff MSI-X is not supported. 427 * 7. Create the driver's taskq. 428 * 8. Start one task queue service thread. 429 * 9. Check if the firmware and SRAM are up-to-date. They will be 430 * auto-updated later (before FULL_INIT_DONE), if required. 431 * 10. Create a child device for each MAC (port) 432 * 11. Initialize T3 private state. 433 * 12. Trigger the LED 434 * 13. Setup offload iff supported. 435 * 14. Reset/restart the tick callout. 436 * 15. Attach sysctls 437 * 438 * NOTE: Any modification or deviation from this list MUST be reflected in 439 * the above comment. Failure to do so will result in problems on various 440 * error conditions including link flapping. 441 */ 442 static int 443 cxgb_controller_attach(device_t dev) 444 { 445 device_t child; 446 const struct adapter_info *ai; 447 struct adapter *sc; 448 int i, error = 0; 449 uint32_t vers; 450 int port_qsets = 1; 451 int msi_needed, reg; 452 char buf[80]; 453 454 sc = device_get_softc(dev); 455 sc->dev = dev; 456 sc->msi_count = 0; 457 ai = cxgb_get_adapter_info(dev); 458 459 snprintf(sc->lockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb controller lock %d", 460 device_get_unit(dev)); 461 ADAPTER_LOCK_INIT(sc, sc->lockbuf); 462 463 snprintf(sc->reglockbuf, ADAPTER_LOCK_NAME_LEN, "SGE reg lock %d", 464 device_get_unit(dev)); 465 snprintf(sc->mdiolockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb mdio lock %d", 466 device_get_unit(dev)); 467 snprintf(sc->elmerlockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb elmer lock %d", 468 device_get_unit(dev)); 469 470 MTX_INIT(&sc->sge.reg_lock, sc->reglockbuf, NULL, MTX_SPIN); 471 MTX_INIT(&sc->mdio_lock, sc->mdiolockbuf, NULL, MTX_DEF); 472 MTX_INIT(&sc->elmer_lock, sc->elmerlockbuf, NULL, MTX_DEF); 473 474 mtx_lock(&t3_list_lock); 475 SLIST_INSERT_HEAD(&t3_list, sc, link); 476 mtx_unlock(&t3_list_lock); 477 478 /* find the PCIe link width and set max read request to 4KB*/ 479 if (pci_find_cap(dev, PCIY_EXPRESS, ®) == 0) { 480 uint16_t lnk; 481 482 lnk = pci_read_config(dev, reg + PCIER_LINK_STA, 2); 483 sc->link_width = (lnk & PCIEM_LINK_STA_WIDTH) >> 4; 484 if (sc->link_width < 8 && 485 (ai->caps & SUPPORTED_10000baseT_Full)) { 486 device_printf(sc->dev, 487 "PCIe x%d Link, expect reduced performance\n", 488 sc->link_width); 489 } 490 491 pci_set_max_read_req(dev, 4096); 492 } 493 494 touch_bars(dev); 495 pci_enable_busmaster(dev); 496 /* 497 * Allocate the registers and make them available to the driver. 498 * The registers that we care about for NIC mode are in BAR 0 499 */ 500 sc->regs_rid = PCIR_BAR(0); 501 if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 502 &sc->regs_rid, RF_ACTIVE)) == NULL) { 503 device_printf(dev, "Cannot allocate BAR region 0\n"); 504 error = ENXIO; 505 goto out; 506 } 507 508 sc->bt = rman_get_bustag(sc->regs_res); 509 sc->bh = rman_get_bushandle(sc->regs_res); 510 sc->mmio_len = rman_get_size(sc->regs_res); 511 512 for (i = 0; i < MAX_NPORTS; i++) 513 sc->port[i].adapter = sc; 514 515 if (t3_prep_adapter(sc, ai, 1) < 0) { 516 printf("prep adapter failed\n"); 517 error = ENODEV; 518 goto out; 519 } 520 521 sc->udbs_rid = PCIR_BAR(2); 522 sc->udbs_res = NULL; 523 if (is_offload(sc) && 524 ((sc->udbs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 525 &sc->udbs_rid, RF_ACTIVE)) == NULL)) { 526 device_printf(dev, "Cannot allocate BAR region 1\n"); 527 error = ENXIO; 528 goto out; 529 } 530 531 /* Allocate the BAR for doing MSI-X. If it succeeds, try to allocate 532 * enough messages for the queue sets. If that fails, try falling 533 * back to MSI. If that fails, then try falling back to the legacy 534 * interrupt pin model. 535 */ 536 sc->msix_regs_rid = 0x20; 537 if ((msi_allowed >= 2) && 538 (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 539 &sc->msix_regs_rid, RF_ACTIVE)) != NULL) { 540 541 if (multiq) 542 port_qsets = min(SGE_QSETS/sc->params.nports, mp_ncpus); 543 msi_needed = sc->msi_count = sc->params.nports * port_qsets + 1; 544 545 if (pci_msix_count(dev) == 0 || 546 (error = pci_alloc_msix(dev, &sc->msi_count)) != 0 || 547 sc->msi_count != msi_needed) { 548 device_printf(dev, "alloc msix failed - " 549 "msi_count=%d, msi_needed=%d, err=%d; " 550 "will try MSI\n", sc->msi_count, 551 msi_needed, error); 552 sc->msi_count = 0; 553 port_qsets = 1; 554 pci_release_msi(dev); 555 bus_release_resource(dev, SYS_RES_MEMORY, 556 sc->msix_regs_rid, sc->msix_regs_res); 557 sc->msix_regs_res = NULL; 558 } else { 559 sc->flags |= USING_MSIX; 560 sc->cxgb_intr = cxgb_async_intr; 561 device_printf(dev, 562 "using MSI-X interrupts (%u vectors)\n", 563 sc->msi_count); 564 } 565 } 566 567 if ((msi_allowed >= 1) && (sc->msi_count == 0)) { 568 sc->msi_count = 1; 569 if ((error = pci_alloc_msi(dev, &sc->msi_count)) != 0) { 570 device_printf(dev, "alloc msi failed - " 571 "err=%d; will try INTx\n", error); 572 sc->msi_count = 0; 573 port_qsets = 1; 574 pci_release_msi(dev); 575 } else { 576 sc->flags |= USING_MSI; 577 sc->cxgb_intr = t3_intr_msi; 578 device_printf(dev, "using MSI interrupts\n"); 579 } 580 } 581 if (sc->msi_count == 0) { 582 device_printf(dev, "using line interrupts\n"); 583 sc->cxgb_intr = t3b_intr; 584 } 585 586 /* Create a private taskqueue thread for handling driver events */ 587 sc->tq = taskqueue_create("cxgb_taskq", M_NOWAIT, 588 taskqueue_thread_enqueue, &sc->tq); 589 if (sc->tq == NULL) { 590 device_printf(dev, "failed to allocate controller task queue\n"); 591 goto out; 592 } 593 594 taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq", 595 device_get_nameunit(dev)); 596 TASK_INIT(&sc->tick_task, 0, cxgb_tick_handler, sc); 597 598 599 /* Create a periodic callout for checking adapter status */ 600 callout_init(&sc->cxgb_tick_ch, 1); 601 602 if (t3_check_fw_version(sc) < 0 || force_fw_update) { 603 /* 604 * Warn user that a firmware update will be attempted in init. 605 */ 606 device_printf(dev, "firmware needs to be updated to version %d.%d.%d\n", 607 FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO); 608 sc->flags &= ~FW_UPTODATE; 609 } else { 610 sc->flags |= FW_UPTODATE; 611 } 612 613 if (t3_check_tpsram_version(sc) < 0) { 614 /* 615 * Warn user that a firmware update will be attempted in init. 616 */ 617 device_printf(dev, "SRAM needs to be updated to version %c-%d.%d.%d\n", 618 t3rev2char(sc), TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO); 619 sc->flags &= ~TPS_UPTODATE; 620 } else { 621 sc->flags |= TPS_UPTODATE; 622 } 623 624 /* 625 * Create a child device for each MAC. The ethernet attachment 626 * will be done in these children. 627 */ 628 for (i = 0; i < (sc)->params.nports; i++) { 629 struct port_info *pi; 630 631 if ((child = device_add_child(dev, "cxgb", -1)) == NULL) { 632 device_printf(dev, "failed to add child port\n"); 633 error = EINVAL; 634 goto out; 635 } 636 pi = &sc->port[i]; 637 pi->adapter = sc; 638 pi->nqsets = port_qsets; 639 pi->first_qset = i*port_qsets; 640 pi->port_id = i; 641 pi->tx_chan = i >= ai->nports0; 642 pi->txpkt_intf = pi->tx_chan ? 2 * (i - ai->nports0) + 1 : 2 * i; 643 sc->rxpkt_map[pi->txpkt_intf] = i; 644 sc->port[i].tx_chan = i >= ai->nports0; 645 sc->portdev[i] = child; 646 device_set_softc(child, pi); 647 } 648 if ((error = bus_generic_attach(dev)) != 0) 649 goto out; 650 651 /* initialize sge private state */ 652 t3_sge_init_adapter(sc); 653 654 t3_led_ready(sc); 655 656 error = t3_get_fw_version(sc, &vers); 657 if (error) 658 goto out; 659 660 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d", 661 G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers), 662 G_FW_VERSION_MICRO(vers)); 663 664 snprintf(buf, sizeof(buf), "%s %sNIC\t E/C: %s S/N: %s", 665 ai->desc, is_offload(sc) ? "R" : "", 666 sc->params.vpd.ec, sc->params.vpd.sn); 667 device_set_desc_copy(dev, buf); 668 669 snprintf(&sc->port_types[0], sizeof(sc->port_types), "%x%x%x%x", 670 sc->params.vpd.port_type[0], sc->params.vpd.port_type[1], 671 sc->params.vpd.port_type[2], sc->params.vpd.port_type[3]); 672 673 device_printf(sc->dev, "Firmware Version %s\n", &sc->fw_version[0]); 674 callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc); 675 t3_add_attach_sysctls(sc); 676 677 #ifdef TCP_OFFLOAD 678 for (i = 0; i < NUM_CPL_HANDLERS; i++) 679 sc->cpl_handler[i] = cpl_not_handled; 680 #endif 681 682 t3_intr_clear(sc); 683 error = cxgb_setup_interrupts(sc); 684 out: 685 if (error) 686 cxgb_free(sc); 687 688 return (error); 689 } 690 691 /* 692 * The cxgb_controller_detach routine is called with the device is 693 * unloaded from the system. 694 */ 695 696 static int 697 cxgb_controller_detach(device_t dev) 698 { 699 struct adapter *sc; 700 701 sc = device_get_softc(dev); 702 703 cxgb_free(sc); 704 705 return (0); 706 } 707 708 /* 709 * The cxgb_free() is called by the cxgb_controller_detach() routine 710 * to tear down the structures that were built up in 711 * cxgb_controller_attach(), and should be the final piece of work 712 * done when fully unloading the driver. 713 * 714 * 715 * 1. Shutting down the threads started by the cxgb_controller_attach() 716 * routine. 717 * 2. Stopping the lower level device and all callouts (cxgb_down_locked()). 718 * 3. Detaching all of the port devices created during the 719 * cxgb_controller_attach() routine. 720 * 4. Removing the device children created via cxgb_controller_attach(). 721 * 5. Releasing PCI resources associated with the device. 722 * 6. Turning off the offload support, iff it was turned on. 723 * 7. Destroying the mutexes created in cxgb_controller_attach(). 724 * 725 */ 726 static void 727 cxgb_free(struct adapter *sc) 728 { 729 int i, nqsets = 0; 730 731 ADAPTER_LOCK(sc); 732 sc->flags |= CXGB_SHUTDOWN; 733 ADAPTER_UNLOCK(sc); 734 735 /* 736 * Make sure all child devices are gone. 737 */ 738 bus_generic_detach(sc->dev); 739 for (i = 0; i < (sc)->params.nports; i++) { 740 if (sc->portdev[i] && 741 device_delete_child(sc->dev, sc->portdev[i]) != 0) 742 device_printf(sc->dev, "failed to delete child port\n"); 743 nqsets += sc->port[i].nqsets; 744 } 745 746 /* 747 * At this point, it is as if cxgb_port_detach has run on all ports, and 748 * cxgb_down has run on the adapter. All interrupts have been silenced, 749 * all open devices have been closed. 750 */ 751 KASSERT(sc->open_device_map == 0, ("%s: device(s) still open (%x)", 752 __func__, sc->open_device_map)); 753 for (i = 0; i < sc->params.nports; i++) { 754 KASSERT(sc->port[i].ifp == NULL, ("%s: port %i undead!", 755 __func__, i)); 756 } 757 758 /* 759 * Finish off the adapter's callouts. 760 */ 761 callout_drain(&sc->cxgb_tick_ch); 762 callout_drain(&sc->sge_timer_ch); 763 764 /* 765 * Release resources grabbed under FULL_INIT_DONE by cxgb_up. The 766 * sysctls are cleaned up by the kernel linker. 767 */ 768 if (sc->flags & FULL_INIT_DONE) { 769 t3_free_sge_resources(sc, nqsets); 770 sc->flags &= ~FULL_INIT_DONE; 771 } 772 773 /* 774 * Release all interrupt resources. 775 */ 776 cxgb_teardown_interrupts(sc); 777 if (sc->flags & (USING_MSI | USING_MSIX)) { 778 device_printf(sc->dev, "releasing msi message(s)\n"); 779 pci_release_msi(sc->dev); 780 } else { 781 device_printf(sc->dev, "no msi message to release\n"); 782 } 783 784 if (sc->msix_regs_res != NULL) { 785 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid, 786 sc->msix_regs_res); 787 } 788 789 /* 790 * Free the adapter's taskqueue. 791 */ 792 if (sc->tq != NULL) { 793 taskqueue_free(sc->tq); 794 sc->tq = NULL; 795 } 796 797 free(sc->filters, M_DEVBUF); 798 t3_sge_free(sc); 799 800 if (sc->udbs_res != NULL) 801 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->udbs_rid, 802 sc->udbs_res); 803 804 if (sc->regs_res != NULL) 805 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid, 806 sc->regs_res); 807 808 MTX_DESTROY(&sc->mdio_lock); 809 MTX_DESTROY(&sc->sge.reg_lock); 810 MTX_DESTROY(&sc->elmer_lock); 811 mtx_lock(&t3_list_lock); 812 SLIST_REMOVE(&t3_list, sc, adapter, link); 813 mtx_unlock(&t3_list_lock); 814 ADAPTER_LOCK_DEINIT(sc); 815 } 816 817 /** 818 * setup_sge_qsets - configure SGE Tx/Rx/response queues 819 * @sc: the controller softc 820 * 821 * Determines how many sets of SGE queues to use and initializes them. 822 * We support multiple queue sets per port if we have MSI-X, otherwise 823 * just one queue set per port. 824 */ 825 static int 826 setup_sge_qsets(adapter_t *sc) 827 { 828 int i, j, err, irq_idx = 0, qset_idx = 0; 829 u_int ntxq = SGE_TXQ_PER_SET; 830 831 if ((err = t3_sge_alloc(sc)) != 0) { 832 device_printf(sc->dev, "t3_sge_alloc returned %d\n", err); 833 return (err); 834 } 835 836 if (sc->params.rev > 0 && !(sc->flags & USING_MSI)) 837 irq_idx = -1; 838 839 for (i = 0; i < (sc)->params.nports; i++) { 840 struct port_info *pi = &sc->port[i]; 841 842 for (j = 0; j < pi->nqsets; j++, qset_idx++) { 843 err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports, 844 (sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx, 845 &sc->params.sge.qset[qset_idx], ntxq, pi); 846 if (err) { 847 t3_free_sge_resources(sc, qset_idx); 848 device_printf(sc->dev, 849 "t3_sge_alloc_qset failed with %d\n", err); 850 return (err); 851 } 852 } 853 } 854 855 sc->nqsets = qset_idx; 856 857 return (0); 858 } 859 860 static void 861 cxgb_teardown_interrupts(adapter_t *sc) 862 { 863 int i; 864 865 for (i = 0; i < SGE_QSETS; i++) { 866 if (sc->msix_intr_tag[i] == NULL) { 867 868 /* Should have been setup fully or not at all */ 869 KASSERT(sc->msix_irq_res[i] == NULL && 870 sc->msix_irq_rid[i] == 0, 871 ("%s: half-done interrupt (%d).", __func__, i)); 872 873 continue; 874 } 875 876 bus_teardown_intr(sc->dev, sc->msix_irq_res[i], 877 sc->msix_intr_tag[i]); 878 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->msix_irq_rid[i], 879 sc->msix_irq_res[i]); 880 881 sc->msix_irq_res[i] = sc->msix_intr_tag[i] = NULL; 882 sc->msix_irq_rid[i] = 0; 883 } 884 885 if (sc->intr_tag) { 886 KASSERT(sc->irq_res != NULL, 887 ("%s: half-done interrupt.", __func__)); 888 889 bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag); 890 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid, 891 sc->irq_res); 892 893 sc->irq_res = sc->intr_tag = NULL; 894 sc->irq_rid = 0; 895 } 896 } 897 898 static int 899 cxgb_setup_interrupts(adapter_t *sc) 900 { 901 struct resource *res; 902 void *tag; 903 int i, rid, err, intr_flag = sc->flags & (USING_MSI | USING_MSIX); 904 905 sc->irq_rid = intr_flag ? 1 : 0; 906 sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &sc->irq_rid, 907 RF_SHAREABLE | RF_ACTIVE); 908 if (sc->irq_res == NULL) { 909 device_printf(sc->dev, "Cannot allocate interrupt (%x, %u)\n", 910 intr_flag, sc->irq_rid); 911 err = EINVAL; 912 sc->irq_rid = 0; 913 } else { 914 err = bus_setup_intr(sc->dev, sc->irq_res, 915 INTR_MPSAFE | INTR_TYPE_NET, NULL, 916 sc->cxgb_intr, sc, &sc->intr_tag); 917 918 if (err) { 919 device_printf(sc->dev, 920 "Cannot set up interrupt (%x, %u, %d)\n", 921 intr_flag, sc->irq_rid, err); 922 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid, 923 sc->irq_res); 924 sc->irq_res = sc->intr_tag = NULL; 925 sc->irq_rid = 0; 926 } 927 } 928 929 /* That's all for INTx or MSI */ 930 if (!(intr_flag & USING_MSIX) || err) 931 return (err); 932 933 bus_describe_intr(sc->dev, sc->irq_res, sc->intr_tag, "err"); 934 for (i = 0; i < sc->msi_count - 1; i++) { 935 rid = i + 2; 936 res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &rid, 937 RF_SHAREABLE | RF_ACTIVE); 938 if (res == NULL) { 939 device_printf(sc->dev, "Cannot allocate interrupt " 940 "for message %d\n", rid); 941 err = EINVAL; 942 break; 943 } 944 945 err = bus_setup_intr(sc->dev, res, INTR_MPSAFE | INTR_TYPE_NET, 946 NULL, t3_intr_msix, &sc->sge.qs[i], &tag); 947 if (err) { 948 device_printf(sc->dev, "Cannot set up interrupt " 949 "for message %d (%d)\n", rid, err); 950 bus_release_resource(sc->dev, SYS_RES_IRQ, rid, res); 951 break; 952 } 953 954 sc->msix_irq_rid[i] = rid; 955 sc->msix_irq_res[i] = res; 956 sc->msix_intr_tag[i] = tag; 957 bus_describe_intr(sc->dev, res, tag, "qs%d", i); 958 } 959 960 if (err) 961 cxgb_teardown_interrupts(sc); 962 963 return (err); 964 } 965 966 967 static int 968 cxgb_port_probe(device_t dev) 969 { 970 struct port_info *p; 971 char buf[80]; 972 const char *desc; 973 974 p = device_get_softc(dev); 975 desc = p->phy.desc; 976 snprintf(buf, sizeof(buf), "Port %d %s", p->port_id, desc); 977 device_set_desc_copy(dev, buf); 978 return (0); 979 } 980 981 982 static int 983 cxgb_makedev(struct port_info *pi) 984 { 985 986 pi->port_cdev = make_dev(&cxgb_cdevsw, pi->ifp->if_dunit, 987 UID_ROOT, GID_WHEEL, 0600, "%s", if_name(pi->ifp)); 988 989 if (pi->port_cdev == NULL) 990 return (ENOMEM); 991 992 pi->port_cdev->si_drv1 = (void *)pi; 993 994 return (0); 995 } 996 997 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 998 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 999 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6) 1000 #define CXGB_CAP_ENABLE CXGB_CAP 1001 1002 static int 1003 cxgb_port_attach(device_t dev) 1004 { 1005 struct port_info *p; 1006 struct ifnet *ifp; 1007 int err; 1008 struct adapter *sc; 1009 1010 p = device_get_softc(dev); 1011 sc = p->adapter; 1012 snprintf(p->lockbuf, PORT_NAME_LEN, "cxgb port lock %d:%d", 1013 device_get_unit(device_get_parent(dev)), p->port_id); 1014 PORT_LOCK_INIT(p, p->lockbuf); 1015 1016 callout_init(&p->link_check_ch, 1); 1017 TASK_INIT(&p->link_check_task, 0, check_link_status, p); 1018 1019 /* Allocate an ifnet object and set it up */ 1020 ifp = p->ifp = if_alloc(IFT_ETHER); 1021 if (ifp == NULL) { 1022 device_printf(dev, "Cannot allocate ifnet\n"); 1023 return (ENOMEM); 1024 } 1025 1026 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1027 ifp->if_init = cxgb_init; 1028 ifp->if_softc = p; 1029 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1030 ifp->if_ioctl = cxgb_ioctl; 1031 ifp->if_transmit = cxgb_transmit; 1032 ifp->if_qflush = cxgb_qflush; 1033 ifp->if_get_counter = cxgb_get_counter; 1034 1035 ifp->if_capabilities = CXGB_CAP; 1036 #ifdef TCP_OFFLOAD 1037 if (is_offload(sc)) 1038 ifp->if_capabilities |= IFCAP_TOE4; 1039 #endif 1040 ifp->if_capenable = CXGB_CAP_ENABLE; 1041 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 1042 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 1043 ifp->if_hw_tsomax = IP_MAXPACKET; 1044 ifp->if_hw_tsomaxsegcount = 36; 1045 ifp->if_hw_tsomaxsegsize = 65536; 1046 1047 /* 1048 * Disable TSO on 4-port - it isn't supported by the firmware. 1049 */ 1050 if (sc->params.nports > 2) { 1051 ifp->if_capabilities &= ~(IFCAP_TSO | IFCAP_VLAN_HWTSO); 1052 ifp->if_capenable &= ~(IFCAP_TSO | IFCAP_VLAN_HWTSO); 1053 ifp->if_hwassist &= ~CSUM_TSO; 1054 } 1055 1056 ether_ifattach(ifp, p->hw_addr); 1057 1058 /* Attach driver debugnet methods. */ 1059 DEBUGNET_SET(ifp, cxgb); 1060 1061 #ifdef DEFAULT_JUMBO 1062 if (sc->params.nports <= 2) 1063 ifp->if_mtu = ETHERMTU_JUMBO; 1064 #endif 1065 if ((err = cxgb_makedev(p)) != 0) { 1066 printf("makedev failed %d\n", err); 1067 return (err); 1068 } 1069 1070 /* Create a list of media supported by this port */ 1071 ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change, 1072 cxgb_media_status); 1073 cxgb_build_medialist(p); 1074 1075 t3_sge_init_port(p); 1076 1077 return (err); 1078 } 1079 1080 /* 1081 * cxgb_port_detach() is called via the device_detach methods when 1082 * cxgb_free() calls the bus_generic_detach. It is responsible for 1083 * removing the device from the view of the kernel, i.e. from all 1084 * interfaces lists etc. This routine is only called when the driver is 1085 * being unloaded, not when the link goes down. 1086 */ 1087 static int 1088 cxgb_port_detach(device_t dev) 1089 { 1090 struct port_info *p; 1091 struct adapter *sc; 1092 int i; 1093 1094 p = device_get_softc(dev); 1095 sc = p->adapter; 1096 1097 /* Tell cxgb_ioctl and if_init that the port is going away */ 1098 ADAPTER_LOCK(sc); 1099 SET_DOOMED(p); 1100 wakeup(&sc->flags); 1101 while (IS_BUSY(sc)) 1102 mtx_sleep(&sc->flags, &sc->lock, 0, "cxgbdtch", 0); 1103 SET_BUSY(sc); 1104 ADAPTER_UNLOCK(sc); 1105 1106 if (p->port_cdev != NULL) 1107 destroy_dev(p->port_cdev); 1108 1109 cxgb_uninit_synchronized(p); 1110 ether_ifdetach(p->ifp); 1111 1112 for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) { 1113 struct sge_qset *qs = &sc->sge.qs[i]; 1114 struct sge_txq *txq = &qs->txq[TXQ_ETH]; 1115 1116 callout_drain(&txq->txq_watchdog); 1117 callout_drain(&txq->txq_timer); 1118 } 1119 1120 PORT_LOCK_DEINIT(p); 1121 if_free(p->ifp); 1122 p->ifp = NULL; 1123 1124 ADAPTER_LOCK(sc); 1125 CLR_BUSY(sc); 1126 wakeup_one(&sc->flags); 1127 ADAPTER_UNLOCK(sc); 1128 return (0); 1129 } 1130 1131 void 1132 t3_fatal_err(struct adapter *sc) 1133 { 1134 u_int fw_status[4]; 1135 1136 if (sc->flags & FULL_INIT_DONE) { 1137 t3_sge_stop(sc); 1138 t3_write_reg(sc, A_XGM_TX_CTRL, 0); 1139 t3_write_reg(sc, A_XGM_RX_CTRL, 0); 1140 t3_write_reg(sc, XGM_REG(A_XGM_TX_CTRL, 1), 0); 1141 t3_write_reg(sc, XGM_REG(A_XGM_RX_CTRL, 1), 0); 1142 t3_intr_disable(sc); 1143 } 1144 device_printf(sc->dev,"encountered fatal error, operation suspended\n"); 1145 if (!t3_cim_ctl_blk_read(sc, 0xa0, 4, fw_status)) 1146 device_printf(sc->dev, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n", 1147 fw_status[0], fw_status[1], fw_status[2], fw_status[3]); 1148 } 1149 1150 int 1151 t3_os_find_pci_capability(adapter_t *sc, int cap) 1152 { 1153 device_t dev; 1154 struct pci_devinfo *dinfo; 1155 pcicfgregs *cfg; 1156 uint32_t status; 1157 uint8_t ptr; 1158 1159 dev = sc->dev; 1160 dinfo = device_get_ivars(dev); 1161 cfg = &dinfo->cfg; 1162 1163 status = pci_read_config(dev, PCIR_STATUS, 2); 1164 if (!(status & PCIM_STATUS_CAPPRESENT)) 1165 return (0); 1166 1167 switch (cfg->hdrtype & PCIM_HDRTYPE) { 1168 case 0: 1169 case 1: 1170 ptr = PCIR_CAP_PTR; 1171 break; 1172 case 2: 1173 ptr = PCIR_CAP_PTR_2; 1174 break; 1175 default: 1176 return (0); 1177 break; 1178 } 1179 ptr = pci_read_config(dev, ptr, 1); 1180 1181 while (ptr != 0) { 1182 if (pci_read_config(dev, ptr + PCICAP_ID, 1) == cap) 1183 return (ptr); 1184 ptr = pci_read_config(dev, ptr + PCICAP_NEXTPTR, 1); 1185 } 1186 1187 return (0); 1188 } 1189 1190 int 1191 t3_os_pci_save_state(struct adapter *sc) 1192 { 1193 device_t dev; 1194 struct pci_devinfo *dinfo; 1195 1196 dev = sc->dev; 1197 dinfo = device_get_ivars(dev); 1198 1199 pci_cfg_save(dev, dinfo, 0); 1200 return (0); 1201 } 1202 1203 int 1204 t3_os_pci_restore_state(struct adapter *sc) 1205 { 1206 device_t dev; 1207 struct pci_devinfo *dinfo; 1208 1209 dev = sc->dev; 1210 dinfo = device_get_ivars(dev); 1211 1212 pci_cfg_restore(dev, dinfo); 1213 return (0); 1214 } 1215 1216 /** 1217 * t3_os_link_changed - handle link status changes 1218 * @sc: the adapter associated with the link change 1219 * @port_id: the port index whose link status has changed 1220 * @link_status: the new status of the link 1221 * @speed: the new speed setting 1222 * @duplex: the new duplex setting 1223 * @fc: the new flow-control setting 1224 * 1225 * This is the OS-dependent handler for link status changes. The OS 1226 * neutral handler takes care of most of the processing for these events, 1227 * then calls this handler for any OS-specific processing. 1228 */ 1229 void 1230 t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed, 1231 int duplex, int fc, int mac_was_reset) 1232 { 1233 struct port_info *pi = &adapter->port[port_id]; 1234 struct ifnet *ifp = pi->ifp; 1235 1236 /* no race with detach, so ifp should always be good */ 1237 KASSERT(ifp, ("%s: if detached.", __func__)); 1238 1239 /* Reapply mac settings if they were lost due to a reset */ 1240 if (mac_was_reset) { 1241 PORT_LOCK(pi); 1242 cxgb_update_mac_settings(pi); 1243 PORT_UNLOCK(pi); 1244 } 1245 1246 if (link_status) { 1247 ifp->if_baudrate = IF_Mbps(speed); 1248 if_link_state_change(ifp, LINK_STATE_UP); 1249 } else 1250 if_link_state_change(ifp, LINK_STATE_DOWN); 1251 } 1252 1253 /** 1254 * t3_os_phymod_changed - handle PHY module changes 1255 * @phy: the PHY reporting the module change 1256 * @mod_type: new module type 1257 * 1258 * This is the OS-dependent handler for PHY module changes. It is 1259 * invoked when a PHY module is removed or inserted for any OS-specific 1260 * processing. 1261 */ 1262 void t3_os_phymod_changed(struct adapter *adap, int port_id) 1263 { 1264 static const char *mod_str[] = { 1265 NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX-L", "unknown" 1266 }; 1267 struct port_info *pi = &adap->port[port_id]; 1268 int mod = pi->phy.modtype; 1269 1270 if (mod != pi->media.ifm_cur->ifm_data) 1271 cxgb_build_medialist(pi); 1272 1273 if (mod == phy_modtype_none) 1274 if_printf(pi->ifp, "PHY module unplugged\n"); 1275 else { 1276 KASSERT(mod < ARRAY_SIZE(mod_str), 1277 ("invalid PHY module type %d", mod)); 1278 if_printf(pi->ifp, "%s PHY module inserted\n", mod_str[mod]); 1279 } 1280 } 1281 1282 void 1283 t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[]) 1284 { 1285 1286 /* 1287 * The ifnet might not be allocated before this gets called, 1288 * as this is called early on in attach by t3_prep_adapter 1289 * save the address off in the port structure 1290 */ 1291 if (cxgb_debug) 1292 printf("set_hw_addr on idx %d addr %6D\n", port_idx, hw_addr, ":"); 1293 bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN); 1294 } 1295 1296 /* 1297 * Programs the XGMAC based on the settings in the ifnet. These settings 1298 * include MTU, MAC address, mcast addresses, etc. 1299 */ 1300 static void 1301 cxgb_update_mac_settings(struct port_info *p) 1302 { 1303 struct ifnet *ifp = p->ifp; 1304 struct t3_rx_mode rm; 1305 struct cmac *mac = &p->mac; 1306 int mtu, hwtagging; 1307 1308 PORT_LOCK_ASSERT_OWNED(p); 1309 1310 bcopy(IF_LLADDR(ifp), p->hw_addr, ETHER_ADDR_LEN); 1311 1312 mtu = ifp->if_mtu; 1313 if (ifp->if_capenable & IFCAP_VLAN_MTU) 1314 mtu += ETHER_VLAN_ENCAP_LEN; 1315 1316 hwtagging = (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0; 1317 1318 t3_mac_set_mtu(mac, mtu); 1319 t3_set_vlan_accel(p->adapter, 1 << p->tx_chan, hwtagging); 1320 t3_mac_set_address(mac, 0, p->hw_addr); 1321 t3_init_rx_mode(&rm, p); 1322 t3_mac_set_rx_mode(mac, &rm); 1323 } 1324 1325 1326 static int 1327 await_mgmt_replies(struct adapter *adap, unsigned long init_cnt, 1328 unsigned long n) 1329 { 1330 int attempts = 5; 1331 1332 while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) { 1333 if (!--attempts) 1334 return (ETIMEDOUT); 1335 t3_os_sleep(10); 1336 } 1337 return 0; 1338 } 1339 1340 static int 1341 init_tp_parity(struct adapter *adap) 1342 { 1343 int i; 1344 struct mbuf *m; 1345 struct cpl_set_tcb_field *greq; 1346 unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts; 1347 1348 t3_tp_set_offload_mode(adap, 1); 1349 1350 for (i = 0; i < 16; i++) { 1351 struct cpl_smt_write_req *req; 1352 1353 m = m_gethdr(M_WAITOK, MT_DATA); 1354 req = mtod(m, struct cpl_smt_write_req *); 1355 m->m_len = m->m_pkthdr.len = sizeof(*req); 1356 memset(req, 0, sizeof(*req)); 1357 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1358 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i)); 1359 req->iff = i; 1360 t3_mgmt_tx(adap, m); 1361 } 1362 1363 for (i = 0; i < 2048; i++) { 1364 struct cpl_l2t_write_req *req; 1365 1366 m = m_gethdr(M_WAITOK, MT_DATA); 1367 req = mtod(m, struct cpl_l2t_write_req *); 1368 m->m_len = m->m_pkthdr.len = sizeof(*req); 1369 memset(req, 0, sizeof(*req)); 1370 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1371 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i)); 1372 req->params = htonl(V_L2T_W_IDX(i)); 1373 t3_mgmt_tx(adap, m); 1374 } 1375 1376 for (i = 0; i < 2048; i++) { 1377 struct cpl_rte_write_req *req; 1378 1379 m = m_gethdr(M_WAITOK, MT_DATA); 1380 req = mtod(m, struct cpl_rte_write_req *); 1381 m->m_len = m->m_pkthdr.len = sizeof(*req); 1382 memset(req, 0, sizeof(*req)); 1383 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1384 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i)); 1385 req->l2t_idx = htonl(V_L2T_W_IDX(i)); 1386 t3_mgmt_tx(adap, m); 1387 } 1388 1389 m = m_gethdr(M_WAITOK, MT_DATA); 1390 greq = mtod(m, struct cpl_set_tcb_field *); 1391 m->m_len = m->m_pkthdr.len = sizeof(*greq); 1392 memset(greq, 0, sizeof(*greq)); 1393 greq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1394 OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0)); 1395 greq->mask = htobe64(1); 1396 t3_mgmt_tx(adap, m); 1397 1398 i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1); 1399 t3_tp_set_offload_mode(adap, 0); 1400 return (i); 1401 } 1402 1403 /** 1404 * setup_rss - configure Receive Side Steering (per-queue connection demux) 1405 * @adap: the adapter 1406 * 1407 * Sets up RSS to distribute packets to multiple receive queues. We 1408 * configure the RSS CPU lookup table to distribute to the number of HW 1409 * receive queues, and the response queue lookup table to narrow that 1410 * down to the response queues actually configured for each port. 1411 * We always configure the RSS mapping for two ports since the mapping 1412 * table has plenty of entries. 1413 */ 1414 static void 1415 setup_rss(adapter_t *adap) 1416 { 1417 int i; 1418 u_int nq[2]; 1419 uint8_t cpus[SGE_QSETS + 1]; 1420 uint16_t rspq_map[RSS_TABLE_SIZE]; 1421 1422 for (i = 0; i < SGE_QSETS; ++i) 1423 cpus[i] = i; 1424 cpus[SGE_QSETS] = 0xff; 1425 1426 nq[0] = nq[1] = 0; 1427 for_each_port(adap, i) { 1428 const struct port_info *pi = adap2pinfo(adap, i); 1429 1430 nq[pi->tx_chan] += pi->nqsets; 1431 } 1432 for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) { 1433 rspq_map[i] = nq[0] ? i % nq[0] : 0; 1434 rspq_map[i + RSS_TABLE_SIZE / 2] = nq[1] ? i % nq[1] + nq[0] : 0; 1435 } 1436 1437 /* Calculate the reverse RSS map table */ 1438 for (i = 0; i < SGE_QSETS; ++i) 1439 adap->rrss_map[i] = 0xff; 1440 for (i = 0; i < RSS_TABLE_SIZE; ++i) 1441 if (adap->rrss_map[rspq_map[i]] == 0xff) 1442 adap->rrss_map[rspq_map[i]] = i; 1443 1444 t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN | 1445 F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN | F_OFDMAPEN | 1446 F_RRCPLMAPEN | V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ, 1447 cpus, rspq_map); 1448 1449 } 1450 static void 1451 send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo, 1452 int hi, int port) 1453 { 1454 struct mbuf *m; 1455 struct mngt_pktsched_wr *req; 1456 1457 m = m_gethdr(M_NOWAIT, MT_DATA); 1458 if (m) { 1459 req = mtod(m, struct mngt_pktsched_wr *); 1460 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT)); 1461 req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET; 1462 req->sched = sched; 1463 req->idx = qidx; 1464 req->min = lo; 1465 req->max = hi; 1466 req->binding = port; 1467 m->m_len = m->m_pkthdr.len = sizeof(*req); 1468 t3_mgmt_tx(adap, m); 1469 } 1470 } 1471 1472 static void 1473 bind_qsets(adapter_t *sc) 1474 { 1475 int i, j; 1476 1477 for (i = 0; i < (sc)->params.nports; ++i) { 1478 const struct port_info *pi = adap2pinfo(sc, i); 1479 1480 for (j = 0; j < pi->nqsets; ++j) { 1481 send_pktsched_cmd(sc, 1, pi->first_qset + j, -1, 1482 -1, pi->tx_chan); 1483 1484 } 1485 } 1486 } 1487 1488 static void 1489 update_tpeeprom(struct adapter *adap) 1490 { 1491 const struct firmware *tpeeprom; 1492 1493 uint32_t version; 1494 unsigned int major, minor; 1495 int ret, len; 1496 char rev, name[32]; 1497 1498 t3_seeprom_read(adap, TP_SRAM_OFFSET, &version); 1499 1500 major = G_TP_VERSION_MAJOR(version); 1501 minor = G_TP_VERSION_MINOR(version); 1502 if (major == TP_VERSION_MAJOR && minor == TP_VERSION_MINOR) 1503 return; 1504 1505 rev = t3rev2char(adap); 1506 snprintf(name, sizeof(name), TPEEPROM_NAME, rev); 1507 1508 tpeeprom = firmware_get(name); 1509 if (tpeeprom == NULL) { 1510 device_printf(adap->dev, 1511 "could not load TP EEPROM: unable to load %s\n", 1512 name); 1513 return; 1514 } 1515 1516 len = tpeeprom->datasize - 4; 1517 1518 ret = t3_check_tpsram(adap, tpeeprom->data, tpeeprom->datasize); 1519 if (ret) 1520 goto release_tpeeprom; 1521 1522 if (len != TP_SRAM_LEN) { 1523 device_printf(adap->dev, 1524 "%s length is wrong len=%d expected=%d\n", name, 1525 len, TP_SRAM_LEN); 1526 return; 1527 } 1528 1529 ret = set_eeprom(&adap->port[0], tpeeprom->data, tpeeprom->datasize, 1530 TP_SRAM_OFFSET); 1531 1532 if (!ret) { 1533 device_printf(adap->dev, 1534 "Protocol SRAM image updated in EEPROM to %d.%d.%d\n", 1535 TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO); 1536 } else 1537 device_printf(adap->dev, 1538 "Protocol SRAM image update in EEPROM failed\n"); 1539 1540 release_tpeeprom: 1541 firmware_put(tpeeprom, FIRMWARE_UNLOAD); 1542 1543 return; 1544 } 1545 1546 static int 1547 update_tpsram(struct adapter *adap) 1548 { 1549 const struct firmware *tpsram; 1550 int ret; 1551 char rev, name[32]; 1552 1553 rev = t3rev2char(adap); 1554 snprintf(name, sizeof(name), TPSRAM_NAME, rev); 1555 1556 update_tpeeprom(adap); 1557 1558 tpsram = firmware_get(name); 1559 if (tpsram == NULL){ 1560 device_printf(adap->dev, "could not load TP SRAM\n"); 1561 return (EINVAL); 1562 } else 1563 device_printf(adap->dev, "updating TP SRAM\n"); 1564 1565 ret = t3_check_tpsram(adap, tpsram->data, tpsram->datasize); 1566 if (ret) 1567 goto release_tpsram; 1568 1569 ret = t3_set_proto_sram(adap, tpsram->data); 1570 if (ret) 1571 device_printf(adap->dev, "loading protocol SRAM failed\n"); 1572 1573 release_tpsram: 1574 firmware_put(tpsram, FIRMWARE_UNLOAD); 1575 1576 return ret; 1577 } 1578 1579 /** 1580 * cxgb_up - enable the adapter 1581 * @adap: adapter being enabled 1582 * 1583 * Called when the first port is enabled, this function performs the 1584 * actions necessary to make an adapter operational, such as completing 1585 * the initialization of HW modules, and enabling interrupts. 1586 */ 1587 static int 1588 cxgb_up(struct adapter *sc) 1589 { 1590 int err = 0; 1591 unsigned int mxf = t3_mc5_size(&sc->mc5) - MC5_MIN_TIDS; 1592 1593 KASSERT(sc->open_device_map == 0, ("%s: device(s) already open (%x)", 1594 __func__, sc->open_device_map)); 1595 1596 if ((sc->flags & FULL_INIT_DONE) == 0) { 1597 1598 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 1599 1600 if ((sc->flags & FW_UPTODATE) == 0) 1601 if ((err = upgrade_fw(sc))) 1602 goto out; 1603 1604 if ((sc->flags & TPS_UPTODATE) == 0) 1605 if ((err = update_tpsram(sc))) 1606 goto out; 1607 1608 if (is_offload(sc) && nfilters != 0) { 1609 sc->params.mc5.nservers = 0; 1610 1611 if (nfilters < 0) 1612 sc->params.mc5.nfilters = mxf; 1613 else 1614 sc->params.mc5.nfilters = min(nfilters, mxf); 1615 } 1616 1617 err = t3_init_hw(sc, 0); 1618 if (err) 1619 goto out; 1620 1621 t3_set_reg_field(sc, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT); 1622 t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12)); 1623 1624 err = setup_sge_qsets(sc); 1625 if (err) 1626 goto out; 1627 1628 alloc_filters(sc); 1629 setup_rss(sc); 1630 1631 t3_add_configured_sysctls(sc); 1632 sc->flags |= FULL_INIT_DONE; 1633 } 1634 1635 t3_intr_clear(sc); 1636 t3_sge_start(sc); 1637 t3_intr_enable(sc); 1638 1639 if (sc->params.rev >= T3_REV_C && !(sc->flags & TP_PARITY_INIT) && 1640 is_offload(sc) && init_tp_parity(sc) == 0) 1641 sc->flags |= TP_PARITY_INIT; 1642 1643 if (sc->flags & TP_PARITY_INIT) { 1644 t3_write_reg(sc, A_TP_INT_CAUSE, F_CMCACHEPERR | F_ARPLUTPERR); 1645 t3_write_reg(sc, A_TP_INT_ENABLE, 0x7fbfffff); 1646 } 1647 1648 if (!(sc->flags & QUEUES_BOUND)) { 1649 bind_qsets(sc); 1650 setup_hw_filters(sc); 1651 sc->flags |= QUEUES_BOUND; 1652 } 1653 1654 t3_sge_reset_adapter(sc); 1655 out: 1656 return (err); 1657 } 1658 1659 /* 1660 * Called when the last open device is closed. Does NOT undo all of cxgb_up's 1661 * work. Specifically, the resources grabbed under FULL_INIT_DONE are released 1662 * during controller_detach, not here. 1663 */ 1664 static void 1665 cxgb_down(struct adapter *sc) 1666 { 1667 t3_sge_stop(sc); 1668 t3_intr_disable(sc); 1669 } 1670 1671 /* 1672 * if_init for cxgb ports. 1673 */ 1674 static void 1675 cxgb_init(void *arg) 1676 { 1677 struct port_info *p = arg; 1678 struct adapter *sc = p->adapter; 1679 1680 ADAPTER_LOCK(sc); 1681 cxgb_init_locked(p); /* releases adapter lock */ 1682 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 1683 } 1684 1685 static int 1686 cxgb_init_locked(struct port_info *p) 1687 { 1688 struct adapter *sc = p->adapter; 1689 struct ifnet *ifp = p->ifp; 1690 struct cmac *mac = &p->mac; 1691 int i, rc = 0, may_sleep = 0, gave_up_lock = 0; 1692 1693 ADAPTER_LOCK_ASSERT_OWNED(sc); 1694 1695 while (!IS_DOOMED(p) && IS_BUSY(sc)) { 1696 gave_up_lock = 1; 1697 if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, "cxgbinit", 0)) { 1698 rc = EINTR; 1699 goto done; 1700 } 1701 } 1702 if (IS_DOOMED(p)) { 1703 rc = ENXIO; 1704 goto done; 1705 } 1706 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 1707 1708 /* 1709 * The code that runs during one-time adapter initialization can sleep 1710 * so it's important not to hold any locks across it. 1711 */ 1712 may_sleep = sc->flags & FULL_INIT_DONE ? 0 : 1; 1713 1714 if (may_sleep) { 1715 SET_BUSY(sc); 1716 gave_up_lock = 1; 1717 ADAPTER_UNLOCK(sc); 1718 } 1719 1720 if (sc->open_device_map == 0 && ((rc = cxgb_up(sc)) != 0)) 1721 goto done; 1722 1723 PORT_LOCK(p); 1724 if (isset(&sc->open_device_map, p->port_id) && 1725 (ifp->if_drv_flags & IFF_DRV_RUNNING)) { 1726 PORT_UNLOCK(p); 1727 goto done; 1728 } 1729 t3_port_intr_enable(sc, p->port_id); 1730 if (!mac->multiport) 1731 t3_mac_init(mac); 1732 cxgb_update_mac_settings(p); 1733 t3_link_start(&p->phy, mac, &p->link_config); 1734 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); 1735 ifp->if_drv_flags |= IFF_DRV_RUNNING; 1736 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 1737 PORT_UNLOCK(p); 1738 1739 for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) { 1740 struct sge_qset *qs = &sc->sge.qs[i]; 1741 struct sge_txq *txq = &qs->txq[TXQ_ETH]; 1742 1743 callout_reset_on(&txq->txq_watchdog, hz, cxgb_tx_watchdog, qs, 1744 txq->txq_watchdog.c_cpu); 1745 } 1746 1747 /* all ok */ 1748 setbit(&sc->open_device_map, p->port_id); 1749 callout_reset(&p->link_check_ch, 1750 p->phy.caps & SUPPORTED_LINK_IRQ ? hz * 3 : hz / 4, 1751 link_check_callout, p); 1752 1753 done: 1754 if (may_sleep) { 1755 ADAPTER_LOCK(sc); 1756 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 1757 CLR_BUSY(sc); 1758 } 1759 if (gave_up_lock) 1760 wakeup_one(&sc->flags); 1761 ADAPTER_UNLOCK(sc); 1762 return (rc); 1763 } 1764 1765 static int 1766 cxgb_uninit_locked(struct port_info *p) 1767 { 1768 struct adapter *sc = p->adapter; 1769 int rc; 1770 1771 ADAPTER_LOCK_ASSERT_OWNED(sc); 1772 1773 while (!IS_DOOMED(p) && IS_BUSY(sc)) { 1774 if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, "cxgbunin", 0)) { 1775 rc = EINTR; 1776 goto done; 1777 } 1778 } 1779 if (IS_DOOMED(p)) { 1780 rc = ENXIO; 1781 goto done; 1782 } 1783 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 1784 SET_BUSY(sc); 1785 ADAPTER_UNLOCK(sc); 1786 1787 rc = cxgb_uninit_synchronized(p); 1788 1789 ADAPTER_LOCK(sc); 1790 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 1791 CLR_BUSY(sc); 1792 wakeup_one(&sc->flags); 1793 done: 1794 ADAPTER_UNLOCK(sc); 1795 return (rc); 1796 } 1797 1798 /* 1799 * Called on "ifconfig down", and from port_detach 1800 */ 1801 static int 1802 cxgb_uninit_synchronized(struct port_info *pi) 1803 { 1804 struct adapter *sc = pi->adapter; 1805 struct ifnet *ifp = pi->ifp; 1806 1807 /* 1808 * taskqueue_drain may cause a deadlock if the adapter lock is held. 1809 */ 1810 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 1811 1812 /* 1813 * Clear this port's bit from the open device map, and then drain all 1814 * the tasks that can access/manipulate this port's port_info or ifp. 1815 * We disable this port's interrupts here and so the slow/ext 1816 * interrupt tasks won't be enqueued. The tick task will continue to 1817 * be enqueued every second but the runs after this drain will not see 1818 * this port in the open device map. 1819 * 1820 * A well behaved task must take open_device_map into account and ignore 1821 * ports that are not open. 1822 */ 1823 clrbit(&sc->open_device_map, pi->port_id); 1824 t3_port_intr_disable(sc, pi->port_id); 1825 taskqueue_drain(sc->tq, &sc->slow_intr_task); 1826 taskqueue_drain(sc->tq, &sc->tick_task); 1827 1828 callout_drain(&pi->link_check_ch); 1829 taskqueue_drain(sc->tq, &pi->link_check_task); 1830 1831 PORT_LOCK(pi); 1832 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 1833 1834 /* disable pause frames */ 1835 t3_set_reg_field(sc, A_XGM_TX_CFG + pi->mac.offset, F_TXPAUSEEN, 0); 1836 1837 /* Reset RX FIFO HWM */ 1838 t3_set_reg_field(sc, A_XGM_RXFIFO_CFG + pi->mac.offset, 1839 V_RXFIFOPAUSEHWM(M_RXFIFOPAUSEHWM), 0); 1840 1841 DELAY(100 * 1000); 1842 1843 /* Wait for TXFIFO empty */ 1844 t3_wait_op_done(sc, A_XGM_TXFIFO_CFG + pi->mac.offset, 1845 F_TXFIFO_EMPTY, 1, 20, 5); 1846 1847 DELAY(100 * 1000); 1848 t3_mac_disable(&pi->mac, MAC_DIRECTION_RX); 1849 1850 pi->phy.ops->power_down(&pi->phy, 1); 1851 1852 PORT_UNLOCK(pi); 1853 1854 pi->link_config.link_ok = 0; 1855 t3_os_link_changed(sc, pi->port_id, 0, 0, 0, 0, 0); 1856 1857 if (sc->open_device_map == 0) 1858 cxgb_down(pi->adapter); 1859 1860 return (0); 1861 } 1862 1863 /* 1864 * Mark lro enabled or disabled in all qsets for this port 1865 */ 1866 static int 1867 cxgb_set_lro(struct port_info *p, int enabled) 1868 { 1869 int i; 1870 struct adapter *adp = p->adapter; 1871 struct sge_qset *q; 1872 1873 for (i = 0; i < p->nqsets; i++) { 1874 q = &adp->sge.qs[p->first_qset + i]; 1875 q->lro.enabled = (enabled != 0); 1876 } 1877 return (0); 1878 } 1879 1880 static int 1881 cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data) 1882 { 1883 struct port_info *p = ifp->if_softc; 1884 struct adapter *sc = p->adapter; 1885 struct ifreq *ifr = (struct ifreq *)data; 1886 int flags, error = 0, mtu; 1887 uint32_t mask; 1888 1889 switch (command) { 1890 case SIOCSIFMTU: 1891 ADAPTER_LOCK(sc); 1892 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 1893 if (error) { 1894 fail: 1895 ADAPTER_UNLOCK(sc); 1896 return (error); 1897 } 1898 1899 mtu = ifr->ifr_mtu; 1900 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) { 1901 error = EINVAL; 1902 } else { 1903 ifp->if_mtu = mtu; 1904 PORT_LOCK(p); 1905 cxgb_update_mac_settings(p); 1906 PORT_UNLOCK(p); 1907 } 1908 ADAPTER_UNLOCK(sc); 1909 break; 1910 case SIOCSIFFLAGS: 1911 ADAPTER_LOCK(sc); 1912 if (IS_DOOMED(p)) { 1913 error = ENXIO; 1914 goto fail; 1915 } 1916 if (ifp->if_flags & IFF_UP) { 1917 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1918 flags = p->if_flags; 1919 if (((ifp->if_flags ^ flags) & IFF_PROMISC) || 1920 ((ifp->if_flags ^ flags) & IFF_ALLMULTI)) { 1921 if (IS_BUSY(sc)) { 1922 error = EBUSY; 1923 goto fail; 1924 } 1925 PORT_LOCK(p); 1926 cxgb_update_mac_settings(p); 1927 PORT_UNLOCK(p); 1928 } 1929 ADAPTER_UNLOCK(sc); 1930 } else 1931 error = cxgb_init_locked(p); 1932 p->if_flags = ifp->if_flags; 1933 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1934 error = cxgb_uninit_locked(p); 1935 else 1936 ADAPTER_UNLOCK(sc); 1937 1938 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 1939 break; 1940 case SIOCADDMULTI: 1941 case SIOCDELMULTI: 1942 ADAPTER_LOCK(sc); 1943 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 1944 if (error) 1945 goto fail; 1946 1947 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1948 PORT_LOCK(p); 1949 cxgb_update_mac_settings(p); 1950 PORT_UNLOCK(p); 1951 } 1952 ADAPTER_UNLOCK(sc); 1953 1954 break; 1955 case SIOCSIFCAP: 1956 ADAPTER_LOCK(sc); 1957 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 1958 if (error) 1959 goto fail; 1960 1961 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1962 if (mask & IFCAP_TXCSUM) { 1963 ifp->if_capenable ^= IFCAP_TXCSUM; 1964 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1965 1966 if (IFCAP_TSO4 & ifp->if_capenable && 1967 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1968 mask &= ~IFCAP_TSO4; 1969 ifp->if_capenable &= ~IFCAP_TSO4; 1970 if_printf(ifp, 1971 "tso4 disabled due to -txcsum.\n"); 1972 } 1973 } 1974 if (mask & IFCAP_TXCSUM_IPV6) { 1975 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1976 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1977 1978 if (IFCAP_TSO6 & ifp->if_capenable && 1979 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1980 mask &= ~IFCAP_TSO6; 1981 ifp->if_capenable &= ~IFCAP_TSO6; 1982 if_printf(ifp, 1983 "tso6 disabled due to -txcsum6.\n"); 1984 } 1985 } 1986 if (mask & IFCAP_RXCSUM) 1987 ifp->if_capenable ^= IFCAP_RXCSUM; 1988 if (mask & IFCAP_RXCSUM_IPV6) 1989 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1990 1991 /* 1992 * Note that we leave CSUM_TSO alone (it is always set). The 1993 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1994 * sending a TSO request our way, so it's sufficient to toggle 1995 * IFCAP_TSOx only. 1996 */ 1997 if (mask & IFCAP_TSO4) { 1998 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1999 !(IFCAP_TXCSUM & ifp->if_capenable)) { 2000 if_printf(ifp, "enable txcsum first.\n"); 2001 error = EAGAIN; 2002 goto fail; 2003 } 2004 ifp->if_capenable ^= IFCAP_TSO4; 2005 } 2006 if (mask & IFCAP_TSO6) { 2007 if (!(IFCAP_TSO6 & ifp->if_capenable) && 2008 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 2009 if_printf(ifp, "enable txcsum6 first.\n"); 2010 error = EAGAIN; 2011 goto fail; 2012 } 2013 ifp->if_capenable ^= IFCAP_TSO6; 2014 } 2015 if (mask & IFCAP_LRO) { 2016 ifp->if_capenable ^= IFCAP_LRO; 2017 2018 /* Safe to do this even if cxgb_up not called yet */ 2019 cxgb_set_lro(p, ifp->if_capenable & IFCAP_LRO); 2020 } 2021 #ifdef TCP_OFFLOAD 2022 if (mask & IFCAP_TOE4) { 2023 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE4; 2024 2025 error = toe_capability(p, enable); 2026 if (error == 0) 2027 ifp->if_capenable ^= mask; 2028 } 2029 #endif 2030 if (mask & IFCAP_VLAN_HWTAGGING) { 2031 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 2032 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 2033 PORT_LOCK(p); 2034 cxgb_update_mac_settings(p); 2035 PORT_UNLOCK(p); 2036 } 2037 } 2038 if (mask & IFCAP_VLAN_MTU) { 2039 ifp->if_capenable ^= IFCAP_VLAN_MTU; 2040 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 2041 PORT_LOCK(p); 2042 cxgb_update_mac_settings(p); 2043 PORT_UNLOCK(p); 2044 } 2045 } 2046 if (mask & IFCAP_VLAN_HWTSO) 2047 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 2048 if (mask & IFCAP_VLAN_HWCSUM) 2049 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 2050 2051 #ifdef VLAN_CAPABILITIES 2052 VLAN_CAPABILITIES(ifp); 2053 #endif 2054 ADAPTER_UNLOCK(sc); 2055 break; 2056 case SIOCSIFMEDIA: 2057 case SIOCGIFMEDIA: 2058 error = ifmedia_ioctl(ifp, ifr, &p->media, command); 2059 break; 2060 default: 2061 error = ether_ioctl(ifp, command, data); 2062 } 2063 2064 return (error); 2065 } 2066 2067 static int 2068 cxgb_media_change(struct ifnet *ifp) 2069 { 2070 return (EOPNOTSUPP); 2071 } 2072 2073 /* 2074 * Translates phy->modtype to the correct Ethernet media subtype. 2075 */ 2076 static int 2077 cxgb_ifm_type(int mod) 2078 { 2079 switch (mod) { 2080 case phy_modtype_sr: 2081 return (IFM_10G_SR); 2082 case phy_modtype_lr: 2083 return (IFM_10G_LR); 2084 case phy_modtype_lrm: 2085 return (IFM_10G_LRM); 2086 case phy_modtype_twinax: 2087 return (IFM_10G_TWINAX); 2088 case phy_modtype_twinax_long: 2089 return (IFM_10G_TWINAX_LONG); 2090 case phy_modtype_none: 2091 return (IFM_NONE); 2092 case phy_modtype_unknown: 2093 return (IFM_UNKNOWN); 2094 } 2095 2096 KASSERT(0, ("%s: modtype %d unknown", __func__, mod)); 2097 return (IFM_UNKNOWN); 2098 } 2099 2100 /* 2101 * Rebuilds the ifmedia list for this port, and sets the current media. 2102 */ 2103 static void 2104 cxgb_build_medialist(struct port_info *p) 2105 { 2106 struct cphy *phy = &p->phy; 2107 struct ifmedia *media = &p->media; 2108 int mod = phy->modtype; 2109 int m = IFM_ETHER | IFM_FDX; 2110 2111 PORT_LOCK(p); 2112 2113 ifmedia_removeall(media); 2114 if (phy->caps & SUPPORTED_TP && phy->caps & SUPPORTED_Autoneg) { 2115 /* Copper (RJ45) */ 2116 2117 if (phy->caps & SUPPORTED_10000baseT_Full) 2118 ifmedia_add(media, m | IFM_10G_T, mod, NULL); 2119 2120 if (phy->caps & SUPPORTED_1000baseT_Full) 2121 ifmedia_add(media, m | IFM_1000_T, mod, NULL); 2122 2123 if (phy->caps & SUPPORTED_100baseT_Full) 2124 ifmedia_add(media, m | IFM_100_TX, mod, NULL); 2125 2126 if (phy->caps & SUPPORTED_10baseT_Full) 2127 ifmedia_add(media, m | IFM_10_T, mod, NULL); 2128 2129 ifmedia_add(media, IFM_ETHER | IFM_AUTO, mod, NULL); 2130 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 2131 2132 } else if (phy->caps & SUPPORTED_TP) { 2133 /* Copper (CX4) */ 2134 2135 KASSERT(phy->caps & SUPPORTED_10000baseT_Full, 2136 ("%s: unexpected cap 0x%x", __func__, phy->caps)); 2137 2138 ifmedia_add(media, m | IFM_10G_CX4, mod, NULL); 2139 ifmedia_set(media, m | IFM_10G_CX4); 2140 2141 } else if (phy->caps & SUPPORTED_FIBRE && 2142 phy->caps & SUPPORTED_10000baseT_Full) { 2143 /* 10G optical (but includes SFP+ twinax) */ 2144 2145 m |= cxgb_ifm_type(mod); 2146 if (IFM_SUBTYPE(m) == IFM_NONE) 2147 m &= ~IFM_FDX; 2148 2149 ifmedia_add(media, m, mod, NULL); 2150 ifmedia_set(media, m); 2151 2152 } else if (phy->caps & SUPPORTED_FIBRE && 2153 phy->caps & SUPPORTED_1000baseT_Full) { 2154 /* 1G optical */ 2155 2156 /* XXX: Lie and claim to be SX, could actually be any 1G-X */ 2157 ifmedia_add(media, m | IFM_1000_SX, mod, NULL); 2158 ifmedia_set(media, m | IFM_1000_SX); 2159 2160 } else { 2161 KASSERT(0, ("%s: don't know how to handle 0x%x.", __func__, 2162 phy->caps)); 2163 } 2164 2165 PORT_UNLOCK(p); 2166 } 2167 2168 static void 2169 cxgb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 2170 { 2171 struct port_info *p = ifp->if_softc; 2172 struct ifmedia_entry *cur = p->media.ifm_cur; 2173 int speed = p->link_config.speed; 2174 2175 if (cur->ifm_data != p->phy.modtype) { 2176 cxgb_build_medialist(p); 2177 cur = p->media.ifm_cur; 2178 } 2179 2180 ifmr->ifm_status = IFM_AVALID; 2181 if (!p->link_config.link_ok) 2182 return; 2183 2184 ifmr->ifm_status |= IFM_ACTIVE; 2185 2186 /* 2187 * active and current will differ iff current media is autoselect. That 2188 * can happen only for copper RJ45. 2189 */ 2190 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 2191 return; 2192 KASSERT(p->phy.caps & SUPPORTED_TP && p->phy.caps & SUPPORTED_Autoneg, 2193 ("%s: unexpected PHY caps 0x%x", __func__, p->phy.caps)); 2194 2195 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 2196 if (speed == SPEED_10000) 2197 ifmr->ifm_active |= IFM_10G_T; 2198 else if (speed == SPEED_1000) 2199 ifmr->ifm_active |= IFM_1000_T; 2200 else if (speed == SPEED_100) 2201 ifmr->ifm_active |= IFM_100_TX; 2202 else if (speed == SPEED_10) 2203 ifmr->ifm_active |= IFM_10_T; 2204 else 2205 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 2206 speed)); 2207 } 2208 2209 static uint64_t 2210 cxgb_get_counter(struct ifnet *ifp, ift_counter c) 2211 { 2212 struct port_info *pi = ifp->if_softc; 2213 struct adapter *sc = pi->adapter; 2214 struct cmac *mac = &pi->mac; 2215 struct mac_stats *mstats = &mac->stats; 2216 2217 cxgb_refresh_stats(pi); 2218 2219 switch (c) { 2220 case IFCOUNTER_IPACKETS: 2221 return (mstats->rx_frames); 2222 2223 case IFCOUNTER_IERRORS: 2224 return (mstats->rx_jabber + mstats->rx_data_errs + 2225 mstats->rx_sequence_errs + mstats->rx_runt + 2226 mstats->rx_too_long + mstats->rx_mac_internal_errs + 2227 mstats->rx_short + mstats->rx_fcs_errs); 2228 2229 case IFCOUNTER_OPACKETS: 2230 return (mstats->tx_frames); 2231 2232 case IFCOUNTER_OERRORS: 2233 return (mstats->tx_excess_collisions + mstats->tx_underrun + 2234 mstats->tx_len_errs + mstats->tx_mac_internal_errs + 2235 mstats->tx_excess_deferral + mstats->tx_fcs_errs); 2236 2237 case IFCOUNTER_COLLISIONS: 2238 return (mstats->tx_total_collisions); 2239 2240 case IFCOUNTER_IBYTES: 2241 return (mstats->rx_octets); 2242 2243 case IFCOUNTER_OBYTES: 2244 return (mstats->tx_octets); 2245 2246 case IFCOUNTER_IMCASTS: 2247 return (mstats->rx_mcast_frames); 2248 2249 case IFCOUNTER_OMCASTS: 2250 return (mstats->tx_mcast_frames); 2251 2252 case IFCOUNTER_IQDROPS: 2253 return (mstats->rx_cong_drops); 2254 2255 case IFCOUNTER_OQDROPS: { 2256 int i; 2257 uint64_t drops; 2258 2259 drops = 0; 2260 if (sc->flags & FULL_INIT_DONE) { 2261 for (i = pi->first_qset; i < pi->first_qset + pi->nqsets; i++) 2262 drops += sc->sge.qs[i].txq[TXQ_ETH].txq_mr->br_drops; 2263 } 2264 2265 return (drops); 2266 2267 } 2268 2269 default: 2270 return (if_get_counter_default(ifp, c)); 2271 } 2272 } 2273 2274 static void 2275 cxgb_async_intr(void *data) 2276 { 2277 adapter_t *sc = data; 2278 2279 t3_write_reg(sc, A_PL_INT_ENABLE0, 0); 2280 (void) t3_read_reg(sc, A_PL_INT_ENABLE0); 2281 taskqueue_enqueue(sc->tq, &sc->slow_intr_task); 2282 } 2283 2284 static void 2285 link_check_callout(void *arg) 2286 { 2287 struct port_info *pi = arg; 2288 struct adapter *sc = pi->adapter; 2289 2290 if (!isset(&sc->open_device_map, pi->port_id)) 2291 return; 2292 2293 taskqueue_enqueue(sc->tq, &pi->link_check_task); 2294 } 2295 2296 static void 2297 check_link_status(void *arg, int pending) 2298 { 2299 struct port_info *pi = arg; 2300 struct adapter *sc = pi->adapter; 2301 2302 if (!isset(&sc->open_device_map, pi->port_id)) 2303 return; 2304 2305 t3_link_changed(sc, pi->port_id); 2306 2307 if (pi->link_fault || !(pi->phy.caps & SUPPORTED_LINK_IRQ) || 2308 pi->link_config.link_ok == 0) 2309 callout_reset(&pi->link_check_ch, hz, link_check_callout, pi); 2310 } 2311 2312 void 2313 t3_os_link_intr(struct port_info *pi) 2314 { 2315 /* 2316 * Schedule a link check in the near future. If the link is flapping 2317 * rapidly we'll keep resetting the callout and delaying the check until 2318 * things stabilize a bit. 2319 */ 2320 callout_reset(&pi->link_check_ch, hz / 4, link_check_callout, pi); 2321 } 2322 2323 static void 2324 check_t3b2_mac(struct adapter *sc) 2325 { 2326 int i; 2327 2328 if (sc->flags & CXGB_SHUTDOWN) 2329 return; 2330 2331 for_each_port(sc, i) { 2332 struct port_info *p = &sc->port[i]; 2333 int status; 2334 #ifdef INVARIANTS 2335 struct ifnet *ifp = p->ifp; 2336 #endif 2337 2338 if (!isset(&sc->open_device_map, p->port_id) || p->link_fault || 2339 !p->link_config.link_ok) 2340 continue; 2341 2342 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 2343 ("%s: state mismatch (drv_flags %x, device_map %x)", 2344 __func__, ifp->if_drv_flags, sc->open_device_map)); 2345 2346 PORT_LOCK(p); 2347 status = t3b2_mac_watchdog_task(&p->mac); 2348 if (status == 1) 2349 p->mac.stats.num_toggled++; 2350 else if (status == 2) { 2351 struct cmac *mac = &p->mac; 2352 2353 cxgb_update_mac_settings(p); 2354 t3_link_start(&p->phy, mac, &p->link_config); 2355 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); 2356 t3_port_intr_enable(sc, p->port_id); 2357 p->mac.stats.num_resets++; 2358 } 2359 PORT_UNLOCK(p); 2360 } 2361 } 2362 2363 static void 2364 cxgb_tick(void *arg) 2365 { 2366 adapter_t *sc = (adapter_t *)arg; 2367 2368 if (sc->flags & CXGB_SHUTDOWN) 2369 return; 2370 2371 taskqueue_enqueue(sc->tq, &sc->tick_task); 2372 callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc); 2373 } 2374 2375 void 2376 cxgb_refresh_stats(struct port_info *pi) 2377 { 2378 struct timeval tv; 2379 const struct timeval interval = {0, 250000}; /* 250ms */ 2380 2381 getmicrotime(&tv); 2382 timevalsub(&tv, &interval); 2383 if (timevalcmp(&tv, &pi->last_refreshed, <)) 2384 return; 2385 2386 PORT_LOCK(pi); 2387 t3_mac_update_stats(&pi->mac); 2388 PORT_UNLOCK(pi); 2389 getmicrotime(&pi->last_refreshed); 2390 } 2391 2392 static void 2393 cxgb_tick_handler(void *arg, int count) 2394 { 2395 adapter_t *sc = (adapter_t *)arg; 2396 const struct adapter_params *p = &sc->params; 2397 int i; 2398 uint32_t cause, reset; 2399 2400 if (sc->flags & CXGB_SHUTDOWN || !(sc->flags & FULL_INIT_DONE)) 2401 return; 2402 2403 if (p->rev == T3_REV_B2 && p->nports < 4 && sc->open_device_map) 2404 check_t3b2_mac(sc); 2405 2406 cause = t3_read_reg(sc, A_SG_INT_CAUSE) & (F_RSPQSTARVE | F_FLEMPTY); 2407 if (cause) { 2408 struct sge_qset *qs = &sc->sge.qs[0]; 2409 uint32_t mask, v; 2410 2411 v = t3_read_reg(sc, A_SG_RSPQ_FL_STATUS) & ~0xff00; 2412 2413 mask = 1; 2414 for (i = 0; i < SGE_QSETS; i++) { 2415 if (v & mask) 2416 qs[i].rspq.starved++; 2417 mask <<= 1; 2418 } 2419 2420 mask <<= SGE_QSETS; /* skip RSPQXDISABLED */ 2421 2422 for (i = 0; i < SGE_QSETS * 2; i++) { 2423 if (v & mask) { 2424 qs[i / 2].fl[i % 2].empty++; 2425 } 2426 mask <<= 1; 2427 } 2428 2429 /* clear */ 2430 t3_write_reg(sc, A_SG_RSPQ_FL_STATUS, v); 2431 t3_write_reg(sc, A_SG_INT_CAUSE, cause); 2432 } 2433 2434 for (i = 0; i < sc->params.nports; i++) { 2435 struct port_info *pi = &sc->port[i]; 2436 struct cmac *mac = &pi->mac; 2437 2438 if (!isset(&sc->open_device_map, pi->port_id)) 2439 continue; 2440 2441 cxgb_refresh_stats(pi); 2442 2443 if (mac->multiport) 2444 continue; 2445 2446 /* Count rx fifo overflows, once per second */ 2447 cause = t3_read_reg(sc, A_XGM_INT_CAUSE + mac->offset); 2448 reset = 0; 2449 if (cause & F_RXFIFO_OVERFLOW) { 2450 mac->stats.rx_fifo_ovfl++; 2451 reset |= F_RXFIFO_OVERFLOW; 2452 } 2453 t3_write_reg(sc, A_XGM_INT_CAUSE + mac->offset, reset); 2454 } 2455 } 2456 2457 static void 2458 touch_bars(device_t dev) 2459 { 2460 /* 2461 * Don't enable yet 2462 */ 2463 #if !defined(__LP64__) && 0 2464 u32 v; 2465 2466 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_1, &v); 2467 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_1, v); 2468 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_3, &v); 2469 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_3, v); 2470 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_5, &v); 2471 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_5, v); 2472 #endif 2473 } 2474 2475 static int 2476 set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset) 2477 { 2478 uint8_t *buf; 2479 int err = 0; 2480 u32 aligned_offset, aligned_len, *p; 2481 struct adapter *adapter = pi->adapter; 2482 2483 2484 aligned_offset = offset & ~3; 2485 aligned_len = (len + (offset & 3) + 3) & ~3; 2486 2487 if (aligned_offset != offset || aligned_len != len) { 2488 buf = malloc(aligned_len, M_DEVBUF, M_WAITOK|M_ZERO); 2489 if (!buf) 2490 return (ENOMEM); 2491 err = t3_seeprom_read(adapter, aligned_offset, (u32 *)buf); 2492 if (!err && aligned_len > 4) 2493 err = t3_seeprom_read(adapter, 2494 aligned_offset + aligned_len - 4, 2495 (u32 *)&buf[aligned_len - 4]); 2496 if (err) 2497 goto out; 2498 memcpy(buf + (offset & 3), data, len); 2499 } else 2500 buf = (uint8_t *)(uintptr_t)data; 2501 2502 err = t3_seeprom_wp(adapter, 0); 2503 if (err) 2504 goto out; 2505 2506 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) { 2507 err = t3_seeprom_write(adapter, aligned_offset, *p); 2508 aligned_offset += 4; 2509 } 2510 2511 if (!err) 2512 err = t3_seeprom_wp(adapter, 1); 2513 out: 2514 if (buf != data) 2515 free(buf, M_DEVBUF); 2516 return err; 2517 } 2518 2519 2520 static int 2521 in_range(int val, int lo, int hi) 2522 { 2523 return val < 0 || (val <= hi && val >= lo); 2524 } 2525 2526 static int 2527 cxgb_extension_open(struct cdev *dev, int flags, int fmp, struct thread *td) 2528 { 2529 return (0); 2530 } 2531 2532 static int 2533 cxgb_extension_close(struct cdev *dev, int flags, int fmt, struct thread *td) 2534 { 2535 return (0); 2536 } 2537 2538 static int 2539 cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, 2540 int fflag, struct thread *td) 2541 { 2542 int mmd, error = 0; 2543 struct port_info *pi = dev->si_drv1; 2544 adapter_t *sc = pi->adapter; 2545 2546 #ifdef PRIV_SUPPORTED 2547 if (priv_check(td, PRIV_DRIVER)) { 2548 if (cxgb_debug) 2549 printf("user does not have access to privileged ioctls\n"); 2550 return (EPERM); 2551 } 2552 #else 2553 if (suser(td)) { 2554 if (cxgb_debug) 2555 printf("user does not have access to privileged ioctls\n"); 2556 return (EPERM); 2557 } 2558 #endif 2559 2560 switch (cmd) { 2561 case CHELSIO_GET_MIIREG: { 2562 uint32_t val; 2563 struct cphy *phy = &pi->phy; 2564 struct ch_mii_data *mid = (struct ch_mii_data *)data; 2565 2566 if (!phy->mdio_read) 2567 return (EOPNOTSUPP); 2568 if (is_10G(sc)) { 2569 mmd = mid->phy_id >> 8; 2570 if (!mmd) 2571 mmd = MDIO_DEV_PCS; 2572 else if (mmd > MDIO_DEV_VEND2) 2573 return (EINVAL); 2574 2575 error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd, 2576 mid->reg_num, &val); 2577 } else 2578 error = phy->mdio_read(sc, mid->phy_id & 0x1f, 0, 2579 mid->reg_num & 0x1f, &val); 2580 if (error == 0) 2581 mid->val_out = val; 2582 break; 2583 } 2584 case CHELSIO_SET_MIIREG: { 2585 struct cphy *phy = &pi->phy; 2586 struct ch_mii_data *mid = (struct ch_mii_data *)data; 2587 2588 if (!phy->mdio_write) 2589 return (EOPNOTSUPP); 2590 if (is_10G(sc)) { 2591 mmd = mid->phy_id >> 8; 2592 if (!mmd) 2593 mmd = MDIO_DEV_PCS; 2594 else if (mmd > MDIO_DEV_VEND2) 2595 return (EINVAL); 2596 2597 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 2598 mmd, mid->reg_num, mid->val_in); 2599 } else 2600 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 0, 2601 mid->reg_num & 0x1f, 2602 mid->val_in); 2603 break; 2604 } 2605 case CHELSIO_SETREG: { 2606 struct ch_reg *edata = (struct ch_reg *)data; 2607 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 2608 return (EFAULT); 2609 t3_write_reg(sc, edata->addr, edata->val); 2610 break; 2611 } 2612 case CHELSIO_GETREG: { 2613 struct ch_reg *edata = (struct ch_reg *)data; 2614 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 2615 return (EFAULT); 2616 edata->val = t3_read_reg(sc, edata->addr); 2617 break; 2618 } 2619 case CHELSIO_GET_SGE_CONTEXT: { 2620 struct ch_cntxt *ecntxt = (struct ch_cntxt *)data; 2621 mtx_lock_spin(&sc->sge.reg_lock); 2622 switch (ecntxt->cntxt_type) { 2623 case CNTXT_TYPE_EGRESS: 2624 error = -t3_sge_read_ecntxt(sc, ecntxt->cntxt_id, 2625 ecntxt->data); 2626 break; 2627 case CNTXT_TYPE_FL: 2628 error = -t3_sge_read_fl(sc, ecntxt->cntxt_id, 2629 ecntxt->data); 2630 break; 2631 case CNTXT_TYPE_RSP: 2632 error = -t3_sge_read_rspq(sc, ecntxt->cntxt_id, 2633 ecntxt->data); 2634 break; 2635 case CNTXT_TYPE_CQ: 2636 error = -t3_sge_read_cq(sc, ecntxt->cntxt_id, 2637 ecntxt->data); 2638 break; 2639 default: 2640 error = EINVAL; 2641 break; 2642 } 2643 mtx_unlock_spin(&sc->sge.reg_lock); 2644 break; 2645 } 2646 case CHELSIO_GET_SGE_DESC: { 2647 struct ch_desc *edesc = (struct ch_desc *)data; 2648 int ret; 2649 if (edesc->queue_num >= SGE_QSETS * 6) 2650 return (EINVAL); 2651 ret = t3_get_desc(&sc->sge.qs[edesc->queue_num / 6], 2652 edesc->queue_num % 6, edesc->idx, edesc->data); 2653 if (ret < 0) 2654 return (EINVAL); 2655 edesc->size = ret; 2656 break; 2657 } 2658 case CHELSIO_GET_QSET_PARAMS: { 2659 struct qset_params *q; 2660 struct ch_qset_params *t = (struct ch_qset_params *)data; 2661 int q1 = pi->first_qset; 2662 int nqsets = pi->nqsets; 2663 int i; 2664 2665 if (t->qset_idx >= nqsets) 2666 return EINVAL; 2667 2668 i = q1 + t->qset_idx; 2669 q = &sc->params.sge.qset[i]; 2670 t->rspq_size = q->rspq_size; 2671 t->txq_size[0] = q->txq_size[0]; 2672 t->txq_size[1] = q->txq_size[1]; 2673 t->txq_size[2] = q->txq_size[2]; 2674 t->fl_size[0] = q->fl_size; 2675 t->fl_size[1] = q->jumbo_size; 2676 t->polling = q->polling; 2677 t->lro = q->lro; 2678 t->intr_lat = q->coalesce_usecs; 2679 t->cong_thres = q->cong_thres; 2680 t->qnum = i; 2681 2682 if ((sc->flags & FULL_INIT_DONE) == 0) 2683 t->vector = 0; 2684 else if (sc->flags & USING_MSIX) 2685 t->vector = rman_get_start(sc->msix_irq_res[i]); 2686 else 2687 t->vector = rman_get_start(sc->irq_res); 2688 2689 break; 2690 } 2691 case CHELSIO_GET_QSET_NUM: { 2692 struct ch_reg *edata = (struct ch_reg *)data; 2693 edata->val = pi->nqsets; 2694 break; 2695 } 2696 case CHELSIO_LOAD_FW: { 2697 uint8_t *fw_data; 2698 uint32_t vers; 2699 struct ch_mem_range *t = (struct ch_mem_range *)data; 2700 2701 /* 2702 * You're allowed to load a firmware only before FULL_INIT_DONE 2703 * 2704 * FW_UPTODATE is also set so the rest of the initialization 2705 * will not overwrite what was loaded here. This gives you the 2706 * flexibility to load any firmware (and maybe shoot yourself in 2707 * the foot). 2708 */ 2709 2710 ADAPTER_LOCK(sc); 2711 if (sc->open_device_map || sc->flags & FULL_INIT_DONE) { 2712 ADAPTER_UNLOCK(sc); 2713 return (EBUSY); 2714 } 2715 2716 fw_data = malloc(t->len, M_DEVBUF, M_NOWAIT); 2717 if (!fw_data) 2718 error = ENOMEM; 2719 else 2720 error = copyin(t->buf, fw_data, t->len); 2721 2722 if (!error) 2723 error = -t3_load_fw(sc, fw_data, t->len); 2724 2725 if (t3_get_fw_version(sc, &vers) == 0) { 2726 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), 2727 "%d.%d.%d", G_FW_VERSION_MAJOR(vers), 2728 G_FW_VERSION_MINOR(vers), G_FW_VERSION_MICRO(vers)); 2729 } 2730 2731 if (!error) 2732 sc->flags |= FW_UPTODATE; 2733 2734 free(fw_data, M_DEVBUF); 2735 ADAPTER_UNLOCK(sc); 2736 break; 2737 } 2738 case CHELSIO_LOAD_BOOT: { 2739 uint8_t *boot_data; 2740 struct ch_mem_range *t = (struct ch_mem_range *)data; 2741 2742 boot_data = malloc(t->len, M_DEVBUF, M_NOWAIT); 2743 if (!boot_data) 2744 return ENOMEM; 2745 2746 error = copyin(t->buf, boot_data, t->len); 2747 if (!error) 2748 error = -t3_load_boot(sc, boot_data, t->len); 2749 2750 free(boot_data, M_DEVBUF); 2751 break; 2752 } 2753 case CHELSIO_GET_PM: { 2754 struct ch_pm *m = (struct ch_pm *)data; 2755 struct tp_params *p = &sc->params.tp; 2756 2757 if (!is_offload(sc)) 2758 return (EOPNOTSUPP); 2759 2760 m->tx_pg_sz = p->tx_pg_size; 2761 m->tx_num_pg = p->tx_num_pgs; 2762 m->rx_pg_sz = p->rx_pg_size; 2763 m->rx_num_pg = p->rx_num_pgs; 2764 m->pm_total = p->pmtx_size + p->chan_rx_size * p->nchan; 2765 2766 break; 2767 } 2768 case CHELSIO_SET_PM: { 2769 struct ch_pm *m = (struct ch_pm *)data; 2770 struct tp_params *p = &sc->params.tp; 2771 2772 if (!is_offload(sc)) 2773 return (EOPNOTSUPP); 2774 if (sc->flags & FULL_INIT_DONE) 2775 return (EBUSY); 2776 2777 if (!m->rx_pg_sz || (m->rx_pg_sz & (m->rx_pg_sz - 1)) || 2778 !m->tx_pg_sz || (m->tx_pg_sz & (m->tx_pg_sz - 1))) 2779 return (EINVAL); /* not power of 2 */ 2780 if (!(m->rx_pg_sz & 0x14000)) 2781 return (EINVAL); /* not 16KB or 64KB */ 2782 if (!(m->tx_pg_sz & 0x1554000)) 2783 return (EINVAL); 2784 if (m->tx_num_pg == -1) 2785 m->tx_num_pg = p->tx_num_pgs; 2786 if (m->rx_num_pg == -1) 2787 m->rx_num_pg = p->rx_num_pgs; 2788 if (m->tx_num_pg % 24 || m->rx_num_pg % 24) 2789 return (EINVAL); 2790 if (m->rx_num_pg * m->rx_pg_sz > p->chan_rx_size || 2791 m->tx_num_pg * m->tx_pg_sz > p->chan_tx_size) 2792 return (EINVAL); 2793 2794 p->rx_pg_size = m->rx_pg_sz; 2795 p->tx_pg_size = m->tx_pg_sz; 2796 p->rx_num_pgs = m->rx_num_pg; 2797 p->tx_num_pgs = m->tx_num_pg; 2798 break; 2799 } 2800 case CHELSIO_SETMTUTAB: { 2801 struct ch_mtus *m = (struct ch_mtus *)data; 2802 int i; 2803 2804 if (!is_offload(sc)) 2805 return (EOPNOTSUPP); 2806 if (offload_running(sc)) 2807 return (EBUSY); 2808 if (m->nmtus != NMTUS) 2809 return (EINVAL); 2810 if (m->mtus[0] < 81) /* accommodate SACK */ 2811 return (EINVAL); 2812 2813 /* 2814 * MTUs must be in ascending order 2815 */ 2816 for (i = 1; i < NMTUS; ++i) 2817 if (m->mtus[i] < m->mtus[i - 1]) 2818 return (EINVAL); 2819 2820 memcpy(sc->params.mtus, m->mtus, sizeof(sc->params.mtus)); 2821 break; 2822 } 2823 case CHELSIO_GETMTUTAB: { 2824 struct ch_mtus *m = (struct ch_mtus *)data; 2825 2826 if (!is_offload(sc)) 2827 return (EOPNOTSUPP); 2828 2829 memcpy(m->mtus, sc->params.mtus, sizeof(m->mtus)); 2830 m->nmtus = NMTUS; 2831 break; 2832 } 2833 case CHELSIO_GET_MEM: { 2834 struct ch_mem_range *t = (struct ch_mem_range *)data; 2835 struct mc7 *mem; 2836 uint8_t *useraddr; 2837 u64 buf[32]; 2838 2839 /* 2840 * Use these to avoid modifying len/addr in the return 2841 * struct 2842 */ 2843 uint32_t len = t->len, addr = t->addr; 2844 2845 if (!is_offload(sc)) 2846 return (EOPNOTSUPP); 2847 if (!(sc->flags & FULL_INIT_DONE)) 2848 return (EIO); /* need the memory controllers */ 2849 if ((addr & 0x7) || (len & 0x7)) 2850 return (EINVAL); 2851 if (t->mem_id == MEM_CM) 2852 mem = &sc->cm; 2853 else if (t->mem_id == MEM_PMRX) 2854 mem = &sc->pmrx; 2855 else if (t->mem_id == MEM_PMTX) 2856 mem = &sc->pmtx; 2857 else 2858 return (EINVAL); 2859 2860 /* 2861 * Version scheme: 2862 * bits 0..9: chip version 2863 * bits 10..15: chip revision 2864 */ 2865 t->version = 3 | (sc->params.rev << 10); 2866 2867 /* 2868 * Read 256 bytes at a time as len can be large and we don't 2869 * want to use huge intermediate buffers. 2870 */ 2871 useraddr = (uint8_t *)t->buf; 2872 while (len) { 2873 unsigned int chunk = min(len, sizeof(buf)); 2874 2875 error = t3_mc7_bd_read(mem, addr / 8, chunk / 8, buf); 2876 if (error) 2877 return (-error); 2878 if (copyout(buf, useraddr, chunk)) 2879 return (EFAULT); 2880 useraddr += chunk; 2881 addr += chunk; 2882 len -= chunk; 2883 } 2884 break; 2885 } 2886 case CHELSIO_READ_TCAM_WORD: { 2887 struct ch_tcam_word *t = (struct ch_tcam_word *)data; 2888 2889 if (!is_offload(sc)) 2890 return (EOPNOTSUPP); 2891 if (!(sc->flags & FULL_INIT_DONE)) 2892 return (EIO); /* need MC5 */ 2893 return -t3_read_mc5_range(&sc->mc5, t->addr, 1, t->buf); 2894 break; 2895 } 2896 case CHELSIO_SET_TRACE_FILTER: { 2897 struct ch_trace *t = (struct ch_trace *)data; 2898 const struct trace_params *tp; 2899 2900 tp = (const struct trace_params *)&t->sip; 2901 if (t->config_tx) 2902 t3_config_trace_filter(sc, tp, 0, t->invert_match, 2903 t->trace_tx); 2904 if (t->config_rx) 2905 t3_config_trace_filter(sc, tp, 1, t->invert_match, 2906 t->trace_rx); 2907 break; 2908 } 2909 case CHELSIO_SET_PKTSCHED: { 2910 struct ch_pktsched_params *p = (struct ch_pktsched_params *)data; 2911 if (sc->open_device_map == 0) 2912 return (EAGAIN); 2913 send_pktsched_cmd(sc, p->sched, p->idx, p->min, p->max, 2914 p->binding); 2915 break; 2916 } 2917 case CHELSIO_IFCONF_GETREGS: { 2918 struct ch_ifconf_regs *regs = (struct ch_ifconf_regs *)data; 2919 int reglen = cxgb_get_regs_len(); 2920 uint8_t *buf = malloc(reglen, M_DEVBUF, M_NOWAIT); 2921 if (buf == NULL) { 2922 return (ENOMEM); 2923 } 2924 if (regs->len > reglen) 2925 regs->len = reglen; 2926 else if (regs->len < reglen) 2927 error = ENOBUFS; 2928 2929 if (!error) { 2930 cxgb_get_regs(sc, regs, buf); 2931 error = copyout(buf, regs->data, reglen); 2932 } 2933 free(buf, M_DEVBUF); 2934 2935 break; 2936 } 2937 case CHELSIO_SET_HW_SCHED: { 2938 struct ch_hw_sched *t = (struct ch_hw_sched *)data; 2939 unsigned int ticks_per_usec = core_ticks_per_usec(sc); 2940 2941 if ((sc->flags & FULL_INIT_DONE) == 0) 2942 return (EAGAIN); /* need TP to be initialized */ 2943 if (t->sched >= NTX_SCHED || !in_range(t->mode, 0, 1) || 2944 !in_range(t->channel, 0, 1) || 2945 !in_range(t->kbps, 0, 10000000) || 2946 !in_range(t->class_ipg, 0, 10000 * 65535 / ticks_per_usec) || 2947 !in_range(t->flow_ipg, 0, 2948 dack_ticks_to_usec(sc, 0x7ff))) 2949 return (EINVAL); 2950 2951 if (t->kbps >= 0) { 2952 error = t3_config_sched(sc, t->kbps, t->sched); 2953 if (error < 0) 2954 return (-error); 2955 } 2956 if (t->class_ipg >= 0) 2957 t3_set_sched_ipg(sc, t->sched, t->class_ipg); 2958 if (t->flow_ipg >= 0) { 2959 t->flow_ipg *= 1000; /* us -> ns */ 2960 t3_set_pace_tbl(sc, &t->flow_ipg, t->sched, 1); 2961 } 2962 if (t->mode >= 0) { 2963 int bit = 1 << (S_TX_MOD_TIMER_MODE + t->sched); 2964 2965 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP, 2966 bit, t->mode ? bit : 0); 2967 } 2968 if (t->channel >= 0) 2969 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP, 2970 1 << t->sched, t->channel << t->sched); 2971 break; 2972 } 2973 case CHELSIO_GET_EEPROM: { 2974 int i; 2975 struct ch_eeprom *e = (struct ch_eeprom *)data; 2976 uint8_t *buf; 2977 2978 if (e->offset & 3 || e->offset >= EEPROMSIZE || 2979 e->len > EEPROMSIZE || e->offset + e->len > EEPROMSIZE) { 2980 return (EINVAL); 2981 } 2982 2983 buf = malloc(EEPROMSIZE, M_DEVBUF, M_NOWAIT); 2984 if (buf == NULL) { 2985 return (ENOMEM); 2986 } 2987 e->magic = EEPROM_MAGIC; 2988 for (i = e->offset & ~3; !error && i < e->offset + e->len; i += 4) 2989 error = -t3_seeprom_read(sc, i, (uint32_t *)&buf[i]); 2990 2991 if (!error) 2992 error = copyout(buf + e->offset, e->data, e->len); 2993 2994 free(buf, M_DEVBUF); 2995 break; 2996 } 2997 case CHELSIO_CLEAR_STATS: { 2998 if (!(sc->flags & FULL_INIT_DONE)) 2999 return EAGAIN; 3000 3001 PORT_LOCK(pi); 3002 t3_mac_update_stats(&pi->mac); 3003 memset(&pi->mac.stats, 0, sizeof(pi->mac.stats)); 3004 PORT_UNLOCK(pi); 3005 break; 3006 } 3007 case CHELSIO_GET_UP_LA: { 3008 struct ch_up_la *la = (struct ch_up_la *)data; 3009 uint8_t *buf = malloc(LA_BUFSIZE, M_DEVBUF, M_NOWAIT); 3010 if (buf == NULL) { 3011 return (ENOMEM); 3012 } 3013 if (la->bufsize < LA_BUFSIZE) 3014 error = ENOBUFS; 3015 3016 if (!error) 3017 error = -t3_get_up_la(sc, &la->stopped, &la->idx, 3018 &la->bufsize, buf); 3019 if (!error) 3020 error = copyout(buf, la->data, la->bufsize); 3021 3022 free(buf, M_DEVBUF); 3023 break; 3024 } 3025 case CHELSIO_GET_UP_IOQS: { 3026 struct ch_up_ioqs *ioqs = (struct ch_up_ioqs *)data; 3027 uint8_t *buf = malloc(IOQS_BUFSIZE, M_DEVBUF, M_NOWAIT); 3028 uint32_t *v; 3029 3030 if (buf == NULL) { 3031 return (ENOMEM); 3032 } 3033 if (ioqs->bufsize < IOQS_BUFSIZE) 3034 error = ENOBUFS; 3035 3036 if (!error) 3037 error = -t3_get_up_ioqs(sc, &ioqs->bufsize, buf); 3038 3039 if (!error) { 3040 v = (uint32_t *)buf; 3041 3042 ioqs->ioq_rx_enable = *v++; 3043 ioqs->ioq_tx_enable = *v++; 3044 ioqs->ioq_rx_status = *v++; 3045 ioqs->ioq_tx_status = *v++; 3046 3047 error = copyout(v, ioqs->data, ioqs->bufsize); 3048 } 3049 3050 free(buf, M_DEVBUF); 3051 break; 3052 } 3053 case CHELSIO_SET_FILTER: { 3054 struct ch_filter *f = (struct ch_filter *)data; 3055 struct filter_info *p; 3056 unsigned int nfilters = sc->params.mc5.nfilters; 3057 3058 if (!is_offload(sc)) 3059 return (EOPNOTSUPP); /* No TCAM */ 3060 if (!(sc->flags & FULL_INIT_DONE)) 3061 return (EAGAIN); /* mc5 not setup yet */ 3062 if (nfilters == 0) 3063 return (EBUSY); /* TOE will use TCAM */ 3064 3065 /* sanity checks */ 3066 if (f->filter_id >= nfilters || 3067 (f->val.dip && f->mask.dip != 0xffffffff) || 3068 (f->val.sport && f->mask.sport != 0xffff) || 3069 (f->val.dport && f->mask.dport != 0xffff) || 3070 (f->val.vlan && f->mask.vlan != 0xfff) || 3071 (f->val.vlan_prio && 3072 f->mask.vlan_prio != FILTER_NO_VLAN_PRI) || 3073 (f->mac_addr_idx != 0xffff && f->mac_addr_idx > 15) || 3074 f->qset >= SGE_QSETS || 3075 sc->rrss_map[f->qset] >= RSS_TABLE_SIZE) 3076 return (EINVAL); 3077 3078 /* Was allocated with M_WAITOK */ 3079 KASSERT(sc->filters, ("filter table NULL\n")); 3080 3081 p = &sc->filters[f->filter_id]; 3082 if (p->locked) 3083 return (EPERM); 3084 3085 bzero(p, sizeof(*p)); 3086 p->sip = f->val.sip; 3087 p->sip_mask = f->mask.sip; 3088 p->dip = f->val.dip; 3089 p->sport = f->val.sport; 3090 p->dport = f->val.dport; 3091 p->vlan = f->mask.vlan ? f->val.vlan : 0xfff; 3092 p->vlan_prio = f->mask.vlan_prio ? (f->val.vlan_prio & 6) : 3093 FILTER_NO_VLAN_PRI; 3094 p->mac_hit = f->mac_hit; 3095 p->mac_vld = f->mac_addr_idx != 0xffff; 3096 p->mac_idx = f->mac_addr_idx; 3097 p->pkt_type = f->proto; 3098 p->report_filter_id = f->want_filter_id; 3099 p->pass = f->pass; 3100 p->rss = f->rss; 3101 p->qset = f->qset; 3102 3103 error = set_filter(sc, f->filter_id, p); 3104 if (error == 0) 3105 p->valid = 1; 3106 break; 3107 } 3108 case CHELSIO_DEL_FILTER: { 3109 struct ch_filter *f = (struct ch_filter *)data; 3110 struct filter_info *p; 3111 unsigned int nfilters = sc->params.mc5.nfilters; 3112 3113 if (!is_offload(sc)) 3114 return (EOPNOTSUPP); 3115 if (!(sc->flags & FULL_INIT_DONE)) 3116 return (EAGAIN); 3117 if (nfilters == 0 || sc->filters == NULL) 3118 return (EINVAL); 3119 if (f->filter_id >= nfilters) 3120 return (EINVAL); 3121 3122 p = &sc->filters[f->filter_id]; 3123 if (p->locked) 3124 return (EPERM); 3125 if (!p->valid) 3126 return (EFAULT); /* Read "Bad address" as "Bad index" */ 3127 3128 bzero(p, sizeof(*p)); 3129 p->sip = p->sip_mask = 0xffffffff; 3130 p->vlan = 0xfff; 3131 p->vlan_prio = FILTER_NO_VLAN_PRI; 3132 p->pkt_type = 1; 3133 error = set_filter(sc, f->filter_id, p); 3134 break; 3135 } 3136 case CHELSIO_GET_FILTER: { 3137 struct ch_filter *f = (struct ch_filter *)data; 3138 struct filter_info *p; 3139 unsigned int i, nfilters = sc->params.mc5.nfilters; 3140 3141 if (!is_offload(sc)) 3142 return (EOPNOTSUPP); 3143 if (!(sc->flags & FULL_INIT_DONE)) 3144 return (EAGAIN); 3145 if (nfilters == 0 || sc->filters == NULL) 3146 return (EINVAL); 3147 3148 i = f->filter_id == 0xffffffff ? 0 : f->filter_id + 1; 3149 for (; i < nfilters; i++) { 3150 p = &sc->filters[i]; 3151 if (!p->valid) 3152 continue; 3153 3154 bzero(f, sizeof(*f)); 3155 3156 f->filter_id = i; 3157 f->val.sip = p->sip; 3158 f->mask.sip = p->sip_mask; 3159 f->val.dip = p->dip; 3160 f->mask.dip = p->dip ? 0xffffffff : 0; 3161 f->val.sport = p->sport; 3162 f->mask.sport = p->sport ? 0xffff : 0; 3163 f->val.dport = p->dport; 3164 f->mask.dport = p->dport ? 0xffff : 0; 3165 f->val.vlan = p->vlan == 0xfff ? 0 : p->vlan; 3166 f->mask.vlan = p->vlan == 0xfff ? 0 : 0xfff; 3167 f->val.vlan_prio = p->vlan_prio == FILTER_NO_VLAN_PRI ? 3168 0 : p->vlan_prio; 3169 f->mask.vlan_prio = p->vlan_prio == FILTER_NO_VLAN_PRI ? 3170 0 : FILTER_NO_VLAN_PRI; 3171 f->mac_hit = p->mac_hit; 3172 f->mac_addr_idx = p->mac_vld ? p->mac_idx : 0xffff; 3173 f->proto = p->pkt_type; 3174 f->want_filter_id = p->report_filter_id; 3175 f->pass = p->pass; 3176 f->rss = p->rss; 3177 f->qset = p->qset; 3178 3179 break; 3180 } 3181 3182 if (i == nfilters) 3183 f->filter_id = 0xffffffff; 3184 break; 3185 } 3186 default: 3187 return (EOPNOTSUPP); 3188 break; 3189 } 3190 3191 return (error); 3192 } 3193 3194 static __inline void 3195 reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start, 3196 unsigned int end) 3197 { 3198 uint32_t *p = (uint32_t *)(buf + start); 3199 3200 for ( ; start <= end; start += sizeof(uint32_t)) 3201 *p++ = t3_read_reg(ap, start); 3202 } 3203 3204 #define T3_REGMAP_SIZE (3 * 1024) 3205 static int 3206 cxgb_get_regs_len(void) 3207 { 3208 return T3_REGMAP_SIZE; 3209 } 3210 3211 static void 3212 cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf) 3213 { 3214 3215 /* 3216 * Version scheme: 3217 * bits 0..9: chip version 3218 * bits 10..15: chip revision 3219 * bit 31: set for PCIe cards 3220 */ 3221 regs->version = 3 | (sc->params.rev << 10) | (is_pcie(sc) << 31); 3222 3223 /* 3224 * We skip the MAC statistics registers because they are clear-on-read. 3225 * Also reading multi-register stats would need to synchronize with the 3226 * periodic mac stats accumulation. Hard to justify the complexity. 3227 */ 3228 memset(buf, 0, cxgb_get_regs_len()); 3229 reg_block_dump(sc, buf, 0, A_SG_RSPQ_CREDIT_RETURN); 3230 reg_block_dump(sc, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT); 3231 reg_block_dump(sc, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE); 3232 reg_block_dump(sc, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA); 3233 reg_block_dump(sc, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3); 3234 reg_block_dump(sc, buf, A_XGM_SERDES_STATUS0, 3235 XGM_REG(A_XGM_SERDES_STAT3, 1)); 3236 reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1), 3237 XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1)); 3238 } 3239 3240 static int 3241 alloc_filters(struct adapter *sc) 3242 { 3243 struct filter_info *p; 3244 unsigned int nfilters = sc->params.mc5.nfilters; 3245 3246 if (nfilters == 0) 3247 return (0); 3248 3249 p = malloc(sizeof(*p) * nfilters, M_DEVBUF, M_WAITOK | M_ZERO); 3250 sc->filters = p; 3251 3252 p = &sc->filters[nfilters - 1]; 3253 p->vlan = 0xfff; 3254 p->vlan_prio = FILTER_NO_VLAN_PRI; 3255 p->pass = p->rss = p->valid = p->locked = 1; 3256 3257 return (0); 3258 } 3259 3260 static int 3261 setup_hw_filters(struct adapter *sc) 3262 { 3263 int i, rc; 3264 unsigned int nfilters = sc->params.mc5.nfilters; 3265 3266 if (!sc->filters) 3267 return (0); 3268 3269 t3_enable_filters(sc); 3270 3271 for (i = rc = 0; i < nfilters && !rc; i++) { 3272 if (sc->filters[i].locked) 3273 rc = set_filter(sc, i, &sc->filters[i]); 3274 } 3275 3276 return (rc); 3277 } 3278 3279 static int 3280 set_filter(struct adapter *sc, int id, const struct filter_info *f) 3281 { 3282 int len; 3283 struct mbuf *m; 3284 struct ulp_txpkt *txpkt; 3285 struct work_request_hdr *wr; 3286 struct cpl_pass_open_req *oreq; 3287 struct cpl_set_tcb_field *sreq; 3288 3289 len = sizeof(*wr) + sizeof(*oreq) + 2 * sizeof(*sreq); 3290 KASSERT(len <= MHLEN, ("filter request too big for an mbuf")); 3291 3292 id += t3_mc5_size(&sc->mc5) - sc->params.mc5.nroutes - 3293 sc->params.mc5.nfilters; 3294 3295 m = m_gethdr(M_WAITOK, MT_DATA); 3296 m->m_len = m->m_pkthdr.len = len; 3297 bzero(mtod(m, char *), len); 3298 3299 wr = mtod(m, struct work_request_hdr *); 3300 wr->wrh_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS) | F_WR_ATOMIC); 3301 3302 oreq = (struct cpl_pass_open_req *)(wr + 1); 3303 txpkt = (struct ulp_txpkt *)oreq; 3304 txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT)); 3305 txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*oreq) / 8)); 3306 OPCODE_TID(oreq) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, id)); 3307 oreq->local_port = htons(f->dport); 3308 oreq->peer_port = htons(f->sport); 3309 oreq->local_ip = htonl(f->dip); 3310 oreq->peer_ip = htonl(f->sip); 3311 oreq->peer_netmask = htonl(f->sip_mask); 3312 oreq->opt0h = 0; 3313 oreq->opt0l = htonl(F_NO_OFFLOAD); 3314 oreq->opt1 = htonl(V_MAC_MATCH_VALID(f->mac_vld) | 3315 V_CONN_POLICY(CPL_CONN_POLICY_FILTER) | 3316 V_VLAN_PRI(f->vlan_prio >> 1) | 3317 V_VLAN_PRI_VALID(f->vlan_prio != FILTER_NO_VLAN_PRI) | 3318 V_PKT_TYPE(f->pkt_type) | V_OPT1_VLAN(f->vlan) | 3319 V_MAC_MATCH(f->mac_idx | (f->mac_hit << 4))); 3320 3321 sreq = (struct cpl_set_tcb_field *)(oreq + 1); 3322 set_tcb_field_ulp(sreq, id, 1, 0x1800808000ULL, 3323 (f->report_filter_id << 15) | (1 << 23) | 3324 ((u64)f->pass << 35) | ((u64)!f->rss << 36)); 3325 set_tcb_field_ulp(sreq + 1, id, 0, 0xffffffff, (2 << 19) | 1); 3326 t3_mgmt_tx(sc, m); 3327 3328 if (f->pass && !f->rss) { 3329 len = sizeof(*sreq); 3330 m = m_gethdr(M_WAITOK, MT_DATA); 3331 m->m_len = m->m_pkthdr.len = len; 3332 bzero(mtod(m, char *), len); 3333 sreq = mtod(m, struct cpl_set_tcb_field *); 3334 sreq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 3335 mk_set_tcb_field(sreq, id, 25, 0x3f80000, 3336 (u64)sc->rrss_map[f->qset] << 19); 3337 t3_mgmt_tx(sc, m); 3338 } 3339 return 0; 3340 } 3341 3342 static inline void 3343 mk_set_tcb_field(struct cpl_set_tcb_field *req, unsigned int tid, 3344 unsigned int word, u64 mask, u64 val) 3345 { 3346 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid)); 3347 req->reply = V_NO_REPLY(1); 3348 req->cpu_idx = 0; 3349 req->word = htons(word); 3350 req->mask = htobe64(mask); 3351 req->val = htobe64(val); 3352 } 3353 3354 static inline void 3355 set_tcb_field_ulp(struct cpl_set_tcb_field *req, unsigned int tid, 3356 unsigned int word, u64 mask, u64 val) 3357 { 3358 struct ulp_txpkt *txpkt = (struct ulp_txpkt *)req; 3359 3360 txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT)); 3361 txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*req) / 8)); 3362 mk_set_tcb_field(req, tid, word, mask, val); 3363 } 3364 3365 void 3366 t3_iterate(void (*func)(struct adapter *, void *), void *arg) 3367 { 3368 struct adapter *sc; 3369 3370 mtx_lock(&t3_list_lock); 3371 SLIST_FOREACH(sc, &t3_list, link) { 3372 /* 3373 * func should not make any assumptions about what state sc is 3374 * in - the only guarantee is that sc->sc_lock is a valid lock. 3375 */ 3376 func(sc, arg); 3377 } 3378 mtx_unlock(&t3_list_lock); 3379 } 3380 3381 #ifdef TCP_OFFLOAD 3382 static int 3383 toe_capability(struct port_info *pi, int enable) 3384 { 3385 int rc; 3386 struct adapter *sc = pi->adapter; 3387 3388 ADAPTER_LOCK_ASSERT_OWNED(sc); 3389 3390 if (!is_offload(sc)) 3391 return (ENODEV); 3392 3393 if (enable) { 3394 if (!(sc->flags & FULL_INIT_DONE)) { 3395 log(LOG_WARNING, 3396 "You must enable a cxgb interface first\n"); 3397 return (EAGAIN); 3398 } 3399 3400 if (isset(&sc->offload_map, pi->port_id)) 3401 return (0); 3402 3403 if (!(sc->flags & TOM_INIT_DONE)) { 3404 rc = t3_activate_uld(sc, ULD_TOM); 3405 if (rc == EAGAIN) { 3406 log(LOG_WARNING, 3407 "You must kldload t3_tom.ko before trying " 3408 "to enable TOE on a cxgb interface.\n"); 3409 } 3410 if (rc != 0) 3411 return (rc); 3412 KASSERT(sc->tom_softc != NULL, 3413 ("%s: TOM activated but softc NULL", __func__)); 3414 KASSERT(sc->flags & TOM_INIT_DONE, 3415 ("%s: TOM activated but flag not set", __func__)); 3416 } 3417 3418 setbit(&sc->offload_map, pi->port_id); 3419 3420 /* 3421 * XXX: Temporary code to allow iWARP to be enabled when TOE is 3422 * enabled on any port. Need to figure out how to enable, 3423 * disable, load, and unload iWARP cleanly. 3424 */ 3425 if (!isset(&sc->offload_map, MAX_NPORTS) && 3426 t3_activate_uld(sc, ULD_IWARP) == 0) 3427 setbit(&sc->offload_map, MAX_NPORTS); 3428 } else { 3429 if (!isset(&sc->offload_map, pi->port_id)) 3430 return (0); 3431 3432 KASSERT(sc->flags & TOM_INIT_DONE, 3433 ("%s: TOM never initialized?", __func__)); 3434 clrbit(&sc->offload_map, pi->port_id); 3435 } 3436 3437 return (0); 3438 } 3439 3440 /* 3441 * Add an upper layer driver to the global list. 3442 */ 3443 int 3444 t3_register_uld(struct uld_info *ui) 3445 { 3446 int rc = 0; 3447 struct uld_info *u; 3448 3449 mtx_lock(&t3_uld_list_lock); 3450 SLIST_FOREACH(u, &t3_uld_list, link) { 3451 if (u->uld_id == ui->uld_id) { 3452 rc = EEXIST; 3453 goto done; 3454 } 3455 } 3456 3457 SLIST_INSERT_HEAD(&t3_uld_list, ui, link); 3458 ui->refcount = 0; 3459 done: 3460 mtx_unlock(&t3_uld_list_lock); 3461 return (rc); 3462 } 3463 3464 int 3465 t3_unregister_uld(struct uld_info *ui) 3466 { 3467 int rc = EINVAL; 3468 struct uld_info *u; 3469 3470 mtx_lock(&t3_uld_list_lock); 3471 3472 SLIST_FOREACH(u, &t3_uld_list, link) { 3473 if (u == ui) { 3474 if (ui->refcount > 0) { 3475 rc = EBUSY; 3476 goto done; 3477 } 3478 3479 SLIST_REMOVE(&t3_uld_list, ui, uld_info, link); 3480 rc = 0; 3481 goto done; 3482 } 3483 } 3484 done: 3485 mtx_unlock(&t3_uld_list_lock); 3486 return (rc); 3487 } 3488 3489 int 3490 t3_activate_uld(struct adapter *sc, int id) 3491 { 3492 int rc = EAGAIN; 3493 struct uld_info *ui; 3494 3495 mtx_lock(&t3_uld_list_lock); 3496 3497 SLIST_FOREACH(ui, &t3_uld_list, link) { 3498 if (ui->uld_id == id) { 3499 rc = ui->activate(sc); 3500 if (rc == 0) 3501 ui->refcount++; 3502 goto done; 3503 } 3504 } 3505 done: 3506 mtx_unlock(&t3_uld_list_lock); 3507 3508 return (rc); 3509 } 3510 3511 int 3512 t3_deactivate_uld(struct adapter *sc, int id) 3513 { 3514 int rc = EINVAL; 3515 struct uld_info *ui; 3516 3517 mtx_lock(&t3_uld_list_lock); 3518 3519 SLIST_FOREACH(ui, &t3_uld_list, link) { 3520 if (ui->uld_id == id) { 3521 rc = ui->deactivate(sc); 3522 if (rc == 0) 3523 ui->refcount--; 3524 goto done; 3525 } 3526 } 3527 done: 3528 mtx_unlock(&t3_uld_list_lock); 3529 3530 return (rc); 3531 } 3532 3533 static int 3534 cpl_not_handled(struct sge_qset *qs __unused, struct rsp_desc *r __unused, 3535 struct mbuf *m) 3536 { 3537 m_freem(m); 3538 return (EDOOFUS); 3539 } 3540 3541 int 3542 t3_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 3543 { 3544 uintptr_t *loc, new; 3545 3546 if (opcode >= NUM_CPL_HANDLERS) 3547 return (EINVAL); 3548 3549 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 3550 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 3551 atomic_store_rel_ptr(loc, new); 3552 3553 return (0); 3554 } 3555 #endif 3556 3557 static int 3558 cxgbc_mod_event(module_t mod, int cmd, void *arg) 3559 { 3560 int rc = 0; 3561 3562 switch (cmd) { 3563 case MOD_LOAD: 3564 mtx_init(&t3_list_lock, "T3 adapters", 0, MTX_DEF); 3565 SLIST_INIT(&t3_list); 3566 #ifdef TCP_OFFLOAD 3567 mtx_init(&t3_uld_list_lock, "T3 ULDs", 0, MTX_DEF); 3568 SLIST_INIT(&t3_uld_list); 3569 #endif 3570 break; 3571 3572 case MOD_UNLOAD: 3573 #ifdef TCP_OFFLOAD 3574 mtx_lock(&t3_uld_list_lock); 3575 if (!SLIST_EMPTY(&t3_uld_list)) { 3576 rc = EBUSY; 3577 mtx_unlock(&t3_uld_list_lock); 3578 break; 3579 } 3580 mtx_unlock(&t3_uld_list_lock); 3581 mtx_destroy(&t3_uld_list_lock); 3582 #endif 3583 mtx_lock(&t3_list_lock); 3584 if (!SLIST_EMPTY(&t3_list)) { 3585 rc = EBUSY; 3586 mtx_unlock(&t3_list_lock); 3587 break; 3588 } 3589 mtx_unlock(&t3_list_lock); 3590 mtx_destroy(&t3_list_lock); 3591 break; 3592 } 3593 3594 return (rc); 3595 } 3596 3597 #ifdef DEBUGNET 3598 static void 3599 cxgb_debugnet_init(struct ifnet *ifp, int *nrxr, int *ncl, int *clsize) 3600 { 3601 struct port_info *pi; 3602 adapter_t *adap; 3603 3604 pi = if_getsoftc(ifp); 3605 adap = pi->adapter; 3606 ADAPTER_LOCK(adap); 3607 *nrxr = adap->nqsets; 3608 *ncl = adap->sge.qs[0].fl[1].size; 3609 *clsize = adap->sge.qs[0].fl[1].buf_size; 3610 ADAPTER_UNLOCK(adap); 3611 } 3612 3613 static void 3614 cxgb_debugnet_event(struct ifnet *ifp, enum debugnet_ev event) 3615 { 3616 struct port_info *pi; 3617 struct sge_qset *qs; 3618 int i; 3619 3620 pi = if_getsoftc(ifp); 3621 if (event == DEBUGNET_START) 3622 for (i = 0; i < pi->adapter->nqsets; i++) { 3623 qs = &pi->adapter->sge.qs[i]; 3624 3625 /* Need to reinit after debugnet_mbuf_start(). */ 3626 qs->fl[0].zone = zone_pack; 3627 qs->fl[1].zone = zone_clust; 3628 qs->lro.enabled = 0; 3629 } 3630 } 3631 3632 static int 3633 cxgb_debugnet_transmit(struct ifnet *ifp, struct mbuf *m) 3634 { 3635 struct port_info *pi; 3636 struct sge_qset *qs; 3637 3638 pi = if_getsoftc(ifp); 3639 if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 3640 IFF_DRV_RUNNING) 3641 return (ENOENT); 3642 3643 qs = &pi->adapter->sge.qs[pi->first_qset]; 3644 return (cxgb_debugnet_encap(qs, &m)); 3645 } 3646 3647 static int 3648 cxgb_debugnet_poll(struct ifnet *ifp, int count) 3649 { 3650 struct port_info *pi; 3651 adapter_t *adap; 3652 int i; 3653 3654 pi = if_getsoftc(ifp); 3655 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) 3656 return (ENOENT); 3657 3658 adap = pi->adapter; 3659 for (i = 0; i < adap->nqsets; i++) 3660 (void)cxgb_debugnet_poll_rx(adap, &adap->sge.qs[i]); 3661 (void)cxgb_debugnet_poll_tx(&adap->sge.qs[pi->first_qset]); 3662 return (0); 3663 } 3664 #endif /* DEBUGNET */ 3665