1 /************************************************************************** 2 3 Copyright (c) 2007-2008, Chelsio Inc. 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Neither the name of the Chelsio Corporation nor the names of its 13 contributors may be used to endorse or promote products derived from 14 this software without specific prior written permission. 15 16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 POSSIBILITY OF SUCH DAMAGE. 27 28 $FreeBSD$ 29 30 ***************************************************************************/ 31 32 33 #ifndef _CXGB_ADAPTER_H_ 34 #define _CXGB_ADAPTER_H_ 35 36 #include <sys/lock.h> 37 #include <sys/mutex.h> 38 #include <sys/sx.h> 39 #include <sys/rman.h> 40 #include <sys/mbuf.h> 41 #include <sys/socket.h> 42 #include <sys/sockio.h> 43 #include <sys/condvar.h> 44 #include <sys/buf_ring.h> 45 46 #include <net/ethernet.h> 47 #include <net/if.h> 48 #include <net/if_media.h> 49 #include <net/if_dl.h> 50 51 #include <machine/bus.h> 52 #include <machine/resource.h> 53 54 #include <sys/bus_dma.h> 55 #include <dev/pci/pcireg.h> 56 #include <dev/pci/pcivar.h> 57 58 #include <cxgb_osdep.h> 59 #include <t3cdev.h> 60 #include <sys/mbufq.h> 61 62 #ifdef LRO_SUPPORTED 63 #include <netinet/tcp_lro.h> 64 #endif 65 66 #define USE_SX 67 68 struct adapter; 69 struct sge_qset; 70 extern int cxgb_debug; 71 72 #ifdef DEBUG_LOCKING 73 #define MTX_INIT(lock, lockname, class, flags) \ 74 do { \ 75 printf("initializing %s at %s:%d\n", lockname, __FILE__, __LINE__); \ 76 mtx_init((lock), lockname, class, flags); \ 77 } while (0) 78 79 #define MTX_DESTROY(lock) \ 80 do { \ 81 printf("destroying %s at %s:%d\n", (lock)->lock_object.lo_name, __FILE__, __LINE__); \ 82 mtx_destroy((lock)); \ 83 } while (0) 84 85 #define SX_INIT(lock, lockname) \ 86 do { \ 87 printf("initializing %s at %s:%d\n", lockname, __FILE__, __LINE__); \ 88 sx_init((lock), lockname); \ 89 } while (0) 90 91 #define SX_DESTROY(lock) \ 92 do { \ 93 printf("destroying %s at %s:%d\n", (lock)->lock_object.lo_name, __FILE__, __LINE__); \ 94 sx_destroy((lock)); \ 95 } while (0) 96 #else 97 #define MTX_INIT mtx_init 98 #define MTX_DESTROY mtx_destroy 99 #define SX_INIT sx_init 100 #define SX_DESTROY sx_destroy 101 #endif 102 103 struct port_info { 104 struct adapter *adapter; 105 struct ifnet *ifp; 106 int if_flags; 107 const struct port_type_info *port_type; 108 struct cphy phy; 109 struct cmac mac; 110 struct link_config link_config; 111 struct ifmedia media; 112 #ifdef USE_SX 113 struct sx lock; 114 #else 115 struct mtx lock; 116 #endif 117 uint8_t port_id; 118 uint8_t tx_chan; 119 uint8_t txpkt_intf; 120 uint8_t first_qset; 121 uint32_t nqsets; 122 123 uint8_t hw_addr[ETHER_ADDR_LEN]; 124 struct task timer_reclaim_task; 125 struct cdev *port_cdev; 126 127 #define PORT_LOCK_NAME_LEN 32 128 #define PORT_NAME_LEN 32 129 char lockbuf[PORT_LOCK_NAME_LEN]; 130 char namebuf[PORT_NAME_LEN]; 131 }; 132 133 enum { /* adapter flags */ 134 FULL_INIT_DONE = (1 << 0), 135 USING_MSI = (1 << 1), 136 USING_MSIX = (1 << 2), 137 QUEUES_BOUND = (1 << 3), 138 FW_UPTODATE = (1 << 4), 139 TPS_UPTODATE = (1 << 5), 140 CXGB_SHUTDOWN = (1 << 6), 141 CXGB_OFLD_INIT = (1 << 7), 142 TP_PARITY_INIT = (1 << 8), 143 }; 144 145 #define FL_Q_SIZE 4096 146 #define JUMBO_Q_SIZE 1024 147 #define RSPQ_Q_SIZE 1024 148 #define TX_ETH_Q_SIZE 1024 149 150 enum { TXQ_ETH = 0, 151 TXQ_OFLD = 1, 152 TXQ_CTRL = 2, }; 153 154 155 /* 156 * work request size in bytes 157 */ 158 #define WR_LEN (WR_FLITS * 8) 159 #define PIO_LEN (WR_LEN - sizeof(struct cpl_tx_pkt_lso)) 160 161 #ifdef LRO_SUPPORTED 162 struct lro_state { 163 unsigned short enabled; 164 struct lro_ctrl ctrl; 165 }; 166 #endif 167 168 #define RX_BUNDLE_SIZE 8 169 170 struct rsp_desc; 171 172 struct sge_rspq { 173 uint32_t credits; 174 uint32_t size; 175 uint32_t cidx; 176 uint32_t gen; 177 uint32_t polling; 178 uint32_t holdoff_tmr; 179 uint32_t next_holdoff; 180 uint32_t imm_data; 181 uint32_t async_notif; 182 uint32_t cntxt_id; 183 uint32_t offload_pkts; 184 uint32_t offload_bundles; 185 uint32_t pure_rsps; 186 uint32_t unhandled_irqs; 187 188 bus_addr_t phys_addr; 189 bus_dma_tag_t desc_tag; 190 bus_dmamap_t desc_map; 191 192 struct t3_mbuf_hdr rspq_mh; 193 struct rsp_desc *desc; 194 struct mtx lock; 195 #define RSPQ_NAME_LEN 32 196 char lockbuf[RSPQ_NAME_LEN]; 197 uint32_t rspq_dump_start; 198 uint32_t rspq_dump_count; 199 }; 200 201 #ifndef DISABLE_MBUF_IOVEC 202 #define rspq_mbuf rspq_mh.mh_head 203 #endif 204 205 struct rx_desc; 206 struct rx_sw_desc; 207 208 struct sge_fl { 209 uint32_t buf_size; 210 uint32_t credits; 211 uint32_t size; 212 uint32_t cidx; 213 uint32_t pidx; 214 uint32_t gen; 215 bus_addr_t phys_addr; 216 uint32_t cntxt_id; 217 uint64_t empty; 218 bus_dma_tag_t desc_tag; 219 bus_dmamap_t desc_map; 220 bus_dma_tag_t entry_tag; 221 uma_zone_t zone; 222 struct rx_desc *desc; 223 struct rx_sw_desc *sdesc; 224 int type; 225 }; 226 227 struct tx_desc; 228 struct tx_sw_desc; 229 230 #define TXQ_TRANSMITTING 0x1 231 232 struct sge_txq { 233 uint64_t flags; 234 uint32_t in_use; 235 uint32_t size; 236 uint32_t processed; 237 uint32_t cleaned; 238 uint32_t stop_thres; 239 uint32_t cidx; 240 uint32_t pidx; 241 uint32_t gen; 242 uint32_t unacked; 243 struct tx_desc *desc; 244 struct tx_sw_desc *sdesc; 245 uint32_t token; 246 bus_addr_t phys_addr; 247 struct task qresume_task; 248 struct task qreclaim_task; 249 struct port_info *port; 250 uint32_t cntxt_id; 251 uint64_t stops; 252 uint64_t restarts; 253 bus_dma_tag_t desc_tag; 254 bus_dmamap_t desc_map; 255 bus_dma_tag_t entry_tag; 256 struct mbuf_head sendq; 257 /* 258 * cleanq should really be an buf_ring to avoid extra 259 * mbuf touches 260 */ 261 struct mbuf_head cleanq; 262 struct buf_ring *txq_mr; 263 struct ifaltq *txq_ifq; 264 struct mbuf *immpkt; 265 266 uint32_t txq_drops; 267 uint32_t txq_skipped; 268 uint32_t txq_coalesced; 269 uint32_t txq_enqueued; 270 uint32_t txq_dump_start; 271 uint32_t txq_dump_count; 272 unsigned long txq_frees; 273 struct mtx lock; 274 struct sg_ent txq_sgl[TX_MAX_SEGS / 2 + 1]; 275 #define TXQ_NAME_LEN 32 276 char lockbuf[TXQ_NAME_LEN]; 277 }; 278 279 280 enum { 281 SGE_PSTAT_TSO, /* # of TSO requests */ 282 SGE_PSTAT_RX_CSUM_GOOD, /* # of successful RX csum offloads */ 283 SGE_PSTAT_TX_CSUM, /* # of TX checksum offloads */ 284 SGE_PSTAT_VLANEX, /* # of VLAN tag extractions */ 285 SGE_PSTAT_VLANINS, /* # of VLAN tag insertions */ 286 }; 287 288 #define SGE_PSTAT_MAX (SGE_PSTAT_VLANINS+1) 289 290 #define QS_EXITING 0x1 291 #define QS_RUNNING 0x2 292 #define QS_BOUND 0x4 293 294 struct sge_qset { 295 struct sge_rspq rspq; 296 struct sge_fl fl[SGE_RXQ_PER_SET]; 297 #ifdef LRO_SUPPORTED 298 struct lro_state lro; 299 #endif 300 struct sge_txq txq[SGE_TXQ_PER_SET]; 301 uint32_t txq_stopped; /* which Tx queues are stopped */ 302 uint64_t port_stats[SGE_PSTAT_MAX]; 303 struct port_info *port; 304 int idx; /* qset # */ 305 int qs_cpuid; 306 int qs_flags; 307 struct cv qs_cv; 308 struct mtx qs_mtx; 309 #define QS_NAME_LEN 32 310 char namebuf[QS_NAME_LEN]; 311 }; 312 313 struct sge { 314 struct sge_qset qs[SGE_QSETS]; 315 struct mtx reg_lock; 316 }; 317 318 struct filter_info; 319 320 struct adapter { 321 device_t dev; 322 int flags; 323 TAILQ_ENTRY(adapter) adapter_entry; 324 325 /* PCI register resources */ 326 int regs_rid; 327 struct resource *regs_res; 328 int udbs_rid; 329 struct resource *udbs_res; 330 bus_space_handle_t bh; 331 bus_space_tag_t bt; 332 bus_size_t mmio_len; 333 uint32_t link_width; 334 335 /* DMA resources */ 336 bus_dma_tag_t parent_dmat; 337 bus_dma_tag_t rx_dmat; 338 bus_dma_tag_t rx_jumbo_dmat; 339 bus_dma_tag_t tx_dmat; 340 341 /* Interrupt resources */ 342 struct resource *irq_res; 343 int irq_rid; 344 void *intr_tag; 345 346 uint32_t msix_regs_rid; 347 struct resource *msix_regs_res; 348 349 struct resource *msix_irq_res[SGE_QSETS]; 350 int msix_irq_rid[SGE_QSETS]; 351 void *msix_intr_tag[SGE_QSETS]; 352 uint8_t rxpkt_map[8]; /* maps RX_PKT interface values to port ids */ 353 uint8_t rrss_map[SGE_QSETS]; /* revers RSS map table */ 354 uint16_t rspq_map[RSS_TABLE_SIZE]; /* maps 7-bit cookie to qidx */ 355 union { 356 uint8_t fill[SGE_QSETS]; 357 uint64_t coalesce; 358 } u; 359 360 #define tunq_fill u.fill 361 #define tunq_coalesce u.coalesce 362 363 struct filter_info *filters; 364 365 /* Tasks */ 366 struct task ext_intr_task; 367 struct task slow_intr_task; 368 struct task tick_task; 369 struct taskqueue *tq; 370 struct callout cxgb_tick_ch; 371 struct callout sge_timer_ch; 372 373 unsigned int check_task_cnt; 374 375 /* Register lock for use by the hardware layer */ 376 struct mtx mdio_lock; 377 struct mtx elmer_lock; 378 379 /* Bookkeeping for the hardware layer */ 380 struct adapter_params params; 381 unsigned int slow_intr_mask; 382 unsigned long irq_stats[IRQ_NUM_STATS]; 383 384 struct sge sge; 385 struct mc7 pmrx; 386 struct mc7 pmtx; 387 struct mc7 cm; 388 struct mc5 mc5; 389 390 struct port_info port[MAX_NPORTS]; 391 device_t portdev[MAX_NPORTS]; 392 struct t3cdev tdev; 393 char fw_version[64]; 394 uint32_t open_device_map; 395 uint32_t registered_device_map; 396 #ifdef USE_SX 397 struct sx lock; 398 #else 399 struct mtx lock; 400 #endif 401 driver_intr_t *cxgb_intr; 402 int msi_count; 403 404 #define ADAPTER_LOCK_NAME_LEN 32 405 char lockbuf[ADAPTER_LOCK_NAME_LEN]; 406 char reglockbuf[ADAPTER_LOCK_NAME_LEN]; 407 char mdiolockbuf[ADAPTER_LOCK_NAME_LEN]; 408 char elmerlockbuf[ADAPTER_LOCK_NAME_LEN]; 409 }; 410 411 struct t3_rx_mode { 412 413 uint32_t idx; 414 struct port_info *port; 415 }; 416 417 418 #define MDIO_LOCK(adapter) mtx_lock(&(adapter)->mdio_lock) 419 #define MDIO_UNLOCK(adapter) mtx_unlock(&(adapter)->mdio_lock) 420 #define ELMR_LOCK(adapter) mtx_lock(&(adapter)->elmer_lock) 421 #define ELMR_UNLOCK(adapter) mtx_unlock(&(adapter)->elmer_lock) 422 423 424 #ifdef USE_SX 425 #define PORT_LOCK(port) sx_xlock(&(port)->lock); 426 #define PORT_UNLOCK(port) sx_xunlock(&(port)->lock); 427 #define PORT_LOCK_INIT(port, name) SX_INIT(&(port)->lock, name) 428 #define PORT_LOCK_DEINIT(port) SX_DESTROY(&(port)->lock) 429 #define PORT_LOCK_ASSERT_OWNED(port) sx_assert(&(port)->lock, SA_LOCKED) 430 431 #define ADAPTER_LOCK(adap) sx_xlock(&(adap)->lock); 432 #define ADAPTER_UNLOCK(adap) sx_xunlock(&(adap)->lock); 433 #define ADAPTER_LOCK_INIT(adap, name) SX_INIT(&(adap)->lock, name) 434 #define ADAPTER_LOCK_DEINIT(adap) SX_DESTROY(&(adap)->lock) 435 #define ADAPTER_LOCK_ASSERT_NOTOWNED(adap) sx_assert(&(adap)->lock, SA_UNLOCKED) 436 #else 437 #define PORT_LOCK(port) mtx_lock(&(port)->lock); 438 #define PORT_UNLOCK(port) mtx_unlock(&(port)->lock); 439 #define PORT_LOCK_INIT(port, name) mtx_init(&(port)->lock, name, 0, MTX_DEF) 440 #define PORT_LOCK_DEINIT(port) mtx_destroy(&(port)->lock) 441 #define PORT_LOCK_ASSERT_OWNED(port) mtx_assert(&(port)->lock, MA_OWNED) 442 443 #define ADAPTER_LOCK(adap) mtx_lock(&(adap)->lock); 444 #define ADAPTER_UNLOCK(adap) mtx_unlock(&(adap)->lock); 445 #define ADAPTER_LOCK_INIT(adap, name) mtx_init(&(adap)->lock, name, 0, MTX_DEF) 446 #define ADAPTER_LOCK_DEINIT(adap) mtx_destroy(&(adap)->lock) 447 #define ADAPTER_LOCK_ASSERT_NOTOWNED(adap) mtx_assert(&(adap)->lock, MO_NOTOWNED) 448 #endif 449 450 451 static __inline uint32_t 452 t3_read_reg(adapter_t *adapter, uint32_t reg_addr) 453 { 454 return (bus_space_read_4(adapter->bt, adapter->bh, reg_addr)); 455 } 456 457 static __inline void 458 t3_write_reg(adapter_t *adapter, uint32_t reg_addr, uint32_t val) 459 { 460 bus_space_write_4(adapter->bt, adapter->bh, reg_addr, val); 461 } 462 463 static __inline void 464 t3_os_pci_read_config_4(adapter_t *adapter, int reg, uint32_t *val) 465 { 466 *val = pci_read_config(adapter->dev, reg, 4); 467 } 468 469 static __inline void 470 t3_os_pci_write_config_4(adapter_t *adapter, int reg, uint32_t val) 471 { 472 pci_write_config(adapter->dev, reg, val, 4); 473 } 474 475 static __inline void 476 t3_os_pci_read_config_2(adapter_t *adapter, int reg, uint16_t *val) 477 { 478 *val = pci_read_config(adapter->dev, reg, 2); 479 } 480 481 static __inline void 482 t3_os_pci_write_config_2(adapter_t *adapter, int reg, uint16_t val) 483 { 484 pci_write_config(adapter->dev, reg, val, 2); 485 } 486 487 static __inline uint8_t * 488 t3_get_next_mcaddr(struct t3_rx_mode *rm) 489 { 490 uint8_t *macaddr = NULL; 491 struct ifnet *ifp = rm->port->ifp; 492 struct ifmultiaddr *ifma; 493 int i = 0; 494 495 IF_ADDR_LOCK(ifp); 496 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 497 if (ifma->ifma_addr->sa_family != AF_LINK) 498 continue; 499 if (i == rm->idx) { 500 macaddr = LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 501 break; 502 } 503 i++; 504 } 505 IF_ADDR_UNLOCK(ifp); 506 507 508 rm->idx++; 509 return (macaddr); 510 } 511 512 static __inline void 513 t3_init_rx_mode(struct t3_rx_mode *rm, struct port_info *port) 514 { 515 rm->idx = 0; 516 rm->port = port; 517 } 518 519 static __inline struct port_info * 520 adap2pinfo(struct adapter *adap, int idx) 521 { 522 return &adap->port[idx]; 523 } 524 525 int t3_os_find_pci_capability(adapter_t *adapter, int cap); 526 int t3_os_pci_save_state(struct adapter *adapter); 527 int t3_os_pci_restore_state(struct adapter *adapter); 528 void t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, 529 int speed, int duplex, int fc); 530 void t3_os_phymod_changed(struct adapter *adap, int port_id); 531 void t3_sge_err_intr_handler(adapter_t *adapter); 532 int t3_offload_tx(struct t3cdev *, struct mbuf *); 533 void t3_os_ext_intr_handler(adapter_t *adapter); 534 void t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[]); 535 int t3_mgmt_tx(adapter_t *adap, struct mbuf *m); 536 537 538 int t3_sge_alloc(struct adapter *); 539 int t3_sge_free(struct adapter *); 540 int t3_sge_alloc_qset(adapter_t *, uint32_t, int, int, const struct qset_params *, 541 int, struct port_info *); 542 void t3_free_sge_resources(adapter_t *); 543 void t3_sge_start(adapter_t *); 544 void t3_sge_stop(adapter_t *); 545 void t3b_intr(void *data); 546 void t3_intr_msi(void *data); 547 void t3_intr_msix(void *data); 548 int t3_encap(struct sge_qset *, struct mbuf **, int); 549 550 int t3_sge_init_adapter(adapter_t *); 551 int t3_sge_reset_adapter(adapter_t *); 552 int t3_sge_init_port(struct port_info *); 553 void t3_sge_deinit_sw(adapter_t *); 554 void t3_free_tx_desc(struct sge_txq *q, int n); 555 void t3_free_tx_desc_all(struct sge_txq *q); 556 557 void t3_rx_eth(struct adapter *adap, struct sge_rspq *rq, struct mbuf *m, int ethpad); 558 559 void t3_add_attach_sysctls(adapter_t *sc); 560 void t3_add_configured_sysctls(adapter_t *sc); 561 int t3_get_desc(const struct sge_qset *qs, unsigned int qnum, unsigned int idx, 562 unsigned char *data); 563 void t3_update_qset_coalesce(struct sge_qset *qs, const struct qset_params *p); 564 565 #define CXGB_TICKS(a) ((a)->params.linkpoll_period ? \ 566 (hz * (a)->params.linkpoll_period) / 10 : \ 567 (a)->params.stats_update_period * hz) 568 569 /* 570 * XXX figure out how we can return this to being private to sge 571 */ 572 #define desc_reclaimable(q) ((int)((q)->processed - (q)->cleaned - TX_MAX_DESC)) 573 574 #define container_of(p, stype, field) ((stype *)(((uint8_t *)(p)) - offsetof(stype, field))) 575 576 static __inline struct sge_qset * 577 fl_to_qset(struct sge_fl *q, int qidx) 578 { 579 return container_of(q, struct sge_qset, fl[qidx]); 580 } 581 582 static __inline struct sge_qset * 583 rspq_to_qset(struct sge_rspq *q) 584 { 585 return container_of(q, struct sge_qset, rspq); 586 } 587 588 static __inline struct sge_qset * 589 txq_to_qset(struct sge_txq *q, int qidx) 590 { 591 return container_of(q, struct sge_qset, txq[qidx]); 592 } 593 594 static __inline struct adapter * 595 tdev2adap(struct t3cdev *d) 596 { 597 return container_of(d, struct adapter, tdev); 598 } 599 600 #undef container_of 601 602 #define OFFLOAD_DEVMAP_BIT 15 603 static inline int offload_running(adapter_t *adapter) 604 { 605 return isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT); 606 } 607 608 int cxgb_pcpu_enqueue_packet(struct ifnet *ifp, struct mbuf *m); 609 int cxgb_pcpu_transmit(struct ifnet *ifp, struct mbuf *m); 610 void cxgb_pcpu_shutdown_threads(struct adapter *sc); 611 void cxgb_pcpu_startup_threads(struct adapter *sc); 612 613 int process_responses(adapter_t *adap, struct sge_qset *qs, int budget); 614 void t3_free_qset(adapter_t *sc, struct sge_qset *q); 615 void cxgb_start(struct ifnet *ifp); 616 void refill_fl_service(adapter_t *adap, struct sge_fl *fl); 617 #endif 618