1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2011 Chelsio Communications, Inc. 5 * All rights reserved. 6 * Written by: Navdeep Parhar <np@FreeBSD.org> 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include "opt_inet.h" 34 #include "opt_inet6.h" 35 #include "opt_kern_tls.h" 36 #include "opt_ratelimit.h" 37 38 #include <sys/types.h> 39 #include <sys/eventhandler.h> 40 #include <sys/mbuf.h> 41 #include <sys/socket.h> 42 #include <sys/kernel.h> 43 #include <sys/ktls.h> 44 #include <sys/malloc.h> 45 #include <sys/queue.h> 46 #include <sys/sbuf.h> 47 #include <sys/taskqueue.h> 48 #include <sys/time.h> 49 #include <sys/sglist.h> 50 #include <sys/sysctl.h> 51 #include <sys/smp.h> 52 #include <sys/socketvar.h> 53 #include <sys/counter.h> 54 #include <net/bpf.h> 55 #include <net/ethernet.h> 56 #include <net/if.h> 57 #include <net/if_vlan_var.h> 58 #include <netinet/in.h> 59 #include <netinet/ip.h> 60 #include <netinet/ip6.h> 61 #include <netinet/tcp.h> 62 #include <netinet/udp.h> 63 #include <machine/in_cksum.h> 64 #include <machine/md_var.h> 65 #include <vm/vm.h> 66 #include <vm/pmap.h> 67 #ifdef DEV_NETMAP 68 #include <machine/bus.h> 69 #include <sys/selinfo.h> 70 #include <net/if_var.h> 71 #include <net/netmap.h> 72 #include <dev/netmap/netmap_kern.h> 73 #endif 74 75 #include "common/common.h" 76 #include "common/t4_regs.h" 77 #include "common/t4_regs_values.h" 78 #include "common/t4_msg.h" 79 #include "t4_l2t.h" 80 #include "t4_mp_ring.h" 81 82 #ifdef T4_PKT_TIMESTAMP 83 #define RX_COPY_THRESHOLD (MINCLSIZE - 8) 84 #else 85 #define RX_COPY_THRESHOLD MINCLSIZE 86 #endif 87 88 /* Internal mbuf flags stored in PH_loc.eight[1]. */ 89 #define MC_NOMAP 0x01 90 #define MC_RAW_WR 0x02 91 #define MC_TLS 0x04 92 93 /* 94 * Ethernet frames are DMA'd at this byte offset into the freelist buffer. 95 * 0-7 are valid values. 96 */ 97 static int fl_pktshift = 0; 98 SYSCTL_INT(_hw_cxgbe, OID_AUTO, fl_pktshift, CTLFLAG_RDTUN, &fl_pktshift, 0, 99 "payload DMA offset in rx buffer (bytes)"); 100 101 /* 102 * Pad ethernet payload up to this boundary. 103 * -1: driver should figure out a good value. 104 * 0: disable padding. 105 * Any power of 2 from 32 to 4096 (both inclusive) is also a valid value. 106 */ 107 int fl_pad = -1; 108 SYSCTL_INT(_hw_cxgbe, OID_AUTO, fl_pad, CTLFLAG_RDTUN, &fl_pad, 0, 109 "payload pad boundary (bytes)"); 110 111 /* 112 * Status page length. 113 * -1: driver should figure out a good value. 114 * 64 or 128 are the only other valid values. 115 */ 116 static int spg_len = -1; 117 SYSCTL_INT(_hw_cxgbe, OID_AUTO, spg_len, CTLFLAG_RDTUN, &spg_len, 0, 118 "status page size (bytes)"); 119 120 /* 121 * Congestion drops. 122 * -1: no congestion feedback (not recommended). 123 * 0: backpressure the channel instead of dropping packets right away. 124 * 1: no backpressure, drop packets for the congested queue immediately. 125 */ 126 static int cong_drop = 0; 127 SYSCTL_INT(_hw_cxgbe, OID_AUTO, cong_drop, CTLFLAG_RDTUN, &cong_drop, 0, 128 "Congestion control for RX queues (0 = backpressure, 1 = drop"); 129 130 /* 131 * Deliver multiple frames in the same free list buffer if they fit. 132 * -1: let the driver decide whether to enable buffer packing or not. 133 * 0: disable buffer packing. 134 * 1: enable buffer packing. 135 */ 136 static int buffer_packing = -1; 137 SYSCTL_INT(_hw_cxgbe, OID_AUTO, buffer_packing, CTLFLAG_RDTUN, &buffer_packing, 138 0, "Enable buffer packing"); 139 140 /* 141 * Start next frame in a packed buffer at this boundary. 142 * -1: driver should figure out a good value. 143 * T4: driver will ignore this and use the same value as fl_pad above. 144 * T5: 16, or a power of 2 from 64 to 4096 (both inclusive) is a valid value. 145 */ 146 static int fl_pack = -1; 147 SYSCTL_INT(_hw_cxgbe, OID_AUTO, fl_pack, CTLFLAG_RDTUN, &fl_pack, 0, 148 "payload pack boundary (bytes)"); 149 150 /* 151 * Largest rx cluster size that the driver is allowed to allocate. 152 */ 153 static int largest_rx_cluster = MJUM16BYTES; 154 SYSCTL_INT(_hw_cxgbe, OID_AUTO, largest_rx_cluster, CTLFLAG_RDTUN, 155 &largest_rx_cluster, 0, "Largest rx cluster (bytes)"); 156 157 /* 158 * Size of cluster allocation that's most likely to succeed. The driver will 159 * fall back to this size if it fails to allocate clusters larger than this. 160 */ 161 static int safest_rx_cluster = PAGE_SIZE; 162 SYSCTL_INT(_hw_cxgbe, OID_AUTO, safest_rx_cluster, CTLFLAG_RDTUN, 163 &safest_rx_cluster, 0, "Safe rx cluster (bytes)"); 164 165 #ifdef RATELIMIT 166 /* 167 * Knob to control TCP timestamp rewriting, and the granularity of the tick used 168 * for rewriting. -1 and 0-3 are all valid values. 169 * -1: hardware should leave the TCP timestamps alone. 170 * 0: 1ms 171 * 1: 100us 172 * 2: 10us 173 * 3: 1us 174 */ 175 static int tsclk = -1; 176 SYSCTL_INT(_hw_cxgbe, OID_AUTO, tsclk, CTLFLAG_RDTUN, &tsclk, 0, 177 "Control TCP timestamp rewriting when using pacing"); 178 179 static int eo_max_backlog = 1024 * 1024; 180 SYSCTL_INT(_hw_cxgbe, OID_AUTO, eo_max_backlog, CTLFLAG_RDTUN, &eo_max_backlog, 181 0, "Maximum backlog of ratelimited data per flow"); 182 #endif 183 184 /* 185 * The interrupt holdoff timers are multiplied by this value on T6+. 186 * 1 and 3-17 (both inclusive) are legal values. 187 */ 188 static int tscale = 1; 189 SYSCTL_INT(_hw_cxgbe, OID_AUTO, tscale, CTLFLAG_RDTUN, &tscale, 0, 190 "Interrupt holdoff timer scale on T6+"); 191 192 /* 193 * Number of LRO entries in the lro_ctrl structure per rx queue. 194 */ 195 static int lro_entries = TCP_LRO_ENTRIES; 196 SYSCTL_INT(_hw_cxgbe, OID_AUTO, lro_entries, CTLFLAG_RDTUN, &lro_entries, 0, 197 "Number of LRO entries per RX queue"); 198 199 /* 200 * This enables presorting of frames before they're fed into tcp_lro_rx. 201 */ 202 static int lro_mbufs = 0; 203 SYSCTL_INT(_hw_cxgbe, OID_AUTO, lro_mbufs, CTLFLAG_RDTUN, &lro_mbufs, 0, 204 "Enable presorting of LRO frames"); 205 206 static int service_iq(struct sge_iq *, int); 207 static int service_iq_fl(struct sge_iq *, int); 208 static struct mbuf *get_fl_payload(struct adapter *, struct sge_fl *, uint32_t); 209 static int eth_rx(struct adapter *, struct sge_rxq *, const struct iq_desc *, 210 u_int); 211 static inline void init_iq(struct sge_iq *, struct adapter *, int, int, int); 212 static inline void init_fl(struct adapter *, struct sge_fl *, int, int, char *); 213 static inline void init_eq(struct adapter *, struct sge_eq *, int, int, uint8_t, 214 uint16_t, char *); 215 static int alloc_ring(struct adapter *, size_t, bus_dma_tag_t *, bus_dmamap_t *, 216 bus_addr_t *, void **); 217 static int free_ring(struct adapter *, bus_dma_tag_t, bus_dmamap_t, bus_addr_t, 218 void *); 219 static int alloc_iq_fl(struct vi_info *, struct sge_iq *, struct sge_fl *, 220 int, int); 221 static int free_iq_fl(struct vi_info *, struct sge_iq *, struct sge_fl *); 222 static void add_iq_sysctls(struct sysctl_ctx_list *, struct sysctl_oid *, 223 struct sge_iq *); 224 static void add_fl_sysctls(struct adapter *, struct sysctl_ctx_list *, 225 struct sysctl_oid *, struct sge_fl *); 226 static int alloc_fwq(struct adapter *); 227 static int free_fwq(struct adapter *); 228 static int alloc_ctrlq(struct adapter *, struct sge_wrq *, int, 229 struct sysctl_oid *); 230 static int alloc_rxq(struct vi_info *, struct sge_rxq *, int, int, 231 struct sysctl_oid *); 232 static int free_rxq(struct vi_info *, struct sge_rxq *); 233 #ifdef TCP_OFFLOAD 234 static int alloc_ofld_rxq(struct vi_info *, struct sge_ofld_rxq *, int, int, 235 struct sysctl_oid *); 236 static int free_ofld_rxq(struct vi_info *, struct sge_ofld_rxq *); 237 #endif 238 #ifdef DEV_NETMAP 239 static int alloc_nm_rxq(struct vi_info *, struct sge_nm_rxq *, int, int, 240 struct sysctl_oid *); 241 static int free_nm_rxq(struct vi_info *, struct sge_nm_rxq *); 242 static int alloc_nm_txq(struct vi_info *, struct sge_nm_txq *, int, int, 243 struct sysctl_oid *); 244 static int free_nm_txq(struct vi_info *, struct sge_nm_txq *); 245 #endif 246 static int ctrl_eq_alloc(struct adapter *, struct sge_eq *); 247 static int eth_eq_alloc(struct adapter *, struct vi_info *, struct sge_eq *); 248 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 249 static int ofld_eq_alloc(struct adapter *, struct vi_info *, struct sge_eq *); 250 #endif 251 static int alloc_eq(struct adapter *, struct vi_info *, struct sge_eq *); 252 static int free_eq(struct adapter *, struct sge_eq *); 253 static int alloc_wrq(struct adapter *, struct vi_info *, struct sge_wrq *, 254 struct sysctl_oid *); 255 static int free_wrq(struct adapter *, struct sge_wrq *); 256 static int alloc_txq(struct vi_info *, struct sge_txq *, int, 257 struct sysctl_oid *); 258 static int free_txq(struct vi_info *, struct sge_txq *); 259 static void oneseg_dma_callback(void *, bus_dma_segment_t *, int, int); 260 static inline void ring_fl_db(struct adapter *, struct sge_fl *); 261 static int refill_fl(struct adapter *, struct sge_fl *, int); 262 static void refill_sfl(void *); 263 static int alloc_fl_sdesc(struct sge_fl *); 264 static void free_fl_sdesc(struct adapter *, struct sge_fl *); 265 static int find_refill_source(struct adapter *, int, bool); 266 static void add_fl_to_sfl(struct adapter *, struct sge_fl *); 267 268 static inline void get_pkt_gl(struct mbuf *, struct sglist *); 269 static inline u_int txpkt_len16(u_int, u_int); 270 static inline u_int txpkt_vm_len16(u_int, u_int); 271 static inline u_int txpkts0_len16(u_int); 272 static inline u_int txpkts1_len16(void); 273 static u_int write_raw_wr(struct sge_txq *, void *, struct mbuf *, u_int); 274 static u_int write_txpkt_wr(struct adapter *, struct sge_txq *, struct mbuf *, 275 u_int); 276 static u_int write_txpkt_vm_wr(struct adapter *, struct sge_txq *, 277 struct mbuf *); 278 static int add_to_txpkts_vf(struct adapter *, struct sge_txq *, struct mbuf *, 279 int, bool *); 280 static int add_to_txpkts_pf(struct adapter *, struct sge_txq *, struct mbuf *, 281 int, bool *); 282 static u_int write_txpkts_wr(struct adapter *, struct sge_txq *); 283 static u_int write_txpkts_vm_wr(struct adapter *, struct sge_txq *); 284 static void write_gl_to_txd(struct sge_txq *, struct mbuf *, caddr_t *, int); 285 static inline void copy_to_txd(struct sge_eq *, caddr_t, caddr_t *, int); 286 static inline void ring_eq_db(struct adapter *, struct sge_eq *, u_int); 287 static inline uint16_t read_hw_cidx(struct sge_eq *); 288 static inline u_int reclaimable_tx_desc(struct sge_eq *); 289 static inline u_int total_available_tx_desc(struct sge_eq *); 290 static u_int reclaim_tx_descs(struct sge_txq *, u_int); 291 static void tx_reclaim(void *, int); 292 static __be64 get_flit(struct sglist_seg *, int, int); 293 static int handle_sge_egr_update(struct sge_iq *, const struct rss_header *, 294 struct mbuf *); 295 static int handle_fw_msg(struct sge_iq *, const struct rss_header *, 296 struct mbuf *); 297 static int t4_handle_wrerr_rpl(struct adapter *, const __be64 *); 298 static void wrq_tx_drain(void *, int); 299 static void drain_wrq_wr_list(struct adapter *, struct sge_wrq *); 300 301 static int sysctl_uint16(SYSCTL_HANDLER_ARGS); 302 static int sysctl_bufsizes(SYSCTL_HANDLER_ARGS); 303 #ifdef RATELIMIT 304 static inline u_int txpkt_eo_len16(u_int, u_int, u_int); 305 static int ethofld_fw4_ack(struct sge_iq *, const struct rss_header *, 306 struct mbuf *); 307 #endif 308 309 static counter_u64_t extfree_refs; 310 static counter_u64_t extfree_rels; 311 312 an_handler_t t4_an_handler; 313 fw_msg_handler_t t4_fw_msg_handler[NUM_FW6_TYPES]; 314 cpl_handler_t t4_cpl_handler[NUM_CPL_CMDS]; 315 cpl_handler_t set_tcb_rpl_handlers[NUM_CPL_COOKIES]; 316 cpl_handler_t l2t_write_rpl_handlers[NUM_CPL_COOKIES]; 317 cpl_handler_t act_open_rpl_handlers[NUM_CPL_COOKIES]; 318 cpl_handler_t abort_rpl_rss_handlers[NUM_CPL_COOKIES]; 319 cpl_handler_t fw4_ack_handlers[NUM_CPL_COOKIES]; 320 321 void 322 t4_register_an_handler(an_handler_t h) 323 { 324 uintptr_t *loc; 325 326 MPASS(h == NULL || t4_an_handler == NULL); 327 328 loc = (uintptr_t *)&t4_an_handler; 329 atomic_store_rel_ptr(loc, (uintptr_t)h); 330 } 331 332 void 333 t4_register_fw_msg_handler(int type, fw_msg_handler_t h) 334 { 335 uintptr_t *loc; 336 337 MPASS(type < nitems(t4_fw_msg_handler)); 338 MPASS(h == NULL || t4_fw_msg_handler[type] == NULL); 339 /* 340 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 341 * handler dispatch table. Reject any attempt to install a handler for 342 * this subtype. 343 */ 344 MPASS(type != FW_TYPE_RSSCPL); 345 MPASS(type != FW6_TYPE_RSSCPL); 346 347 loc = (uintptr_t *)&t4_fw_msg_handler[type]; 348 atomic_store_rel_ptr(loc, (uintptr_t)h); 349 } 350 351 void 352 t4_register_cpl_handler(int opcode, cpl_handler_t h) 353 { 354 uintptr_t *loc; 355 356 MPASS(opcode < nitems(t4_cpl_handler)); 357 MPASS(h == NULL || t4_cpl_handler[opcode] == NULL); 358 359 loc = (uintptr_t *)&t4_cpl_handler[opcode]; 360 atomic_store_rel_ptr(loc, (uintptr_t)h); 361 } 362 363 static int 364 set_tcb_rpl_handler(struct sge_iq *iq, const struct rss_header *rss, 365 struct mbuf *m) 366 { 367 const struct cpl_set_tcb_rpl *cpl = (const void *)(rss + 1); 368 u_int tid; 369 int cookie; 370 371 MPASS(m == NULL); 372 373 tid = GET_TID(cpl); 374 if (is_hpftid(iq->adapter, tid) || is_ftid(iq->adapter, tid)) { 375 /* 376 * The return code for filter-write is put in the CPL cookie so 377 * we have to rely on the hardware tid (is_ftid) to determine 378 * that this is a response to a filter. 379 */ 380 cookie = CPL_COOKIE_FILTER; 381 } else { 382 cookie = G_COOKIE(cpl->cookie); 383 } 384 MPASS(cookie > CPL_COOKIE_RESERVED); 385 MPASS(cookie < nitems(set_tcb_rpl_handlers)); 386 387 return (set_tcb_rpl_handlers[cookie](iq, rss, m)); 388 } 389 390 static int 391 l2t_write_rpl_handler(struct sge_iq *iq, const struct rss_header *rss, 392 struct mbuf *m) 393 { 394 const struct cpl_l2t_write_rpl *rpl = (const void *)(rss + 1); 395 unsigned int cookie; 396 397 MPASS(m == NULL); 398 399 cookie = GET_TID(rpl) & F_SYNC_WR ? CPL_COOKIE_TOM : CPL_COOKIE_FILTER; 400 return (l2t_write_rpl_handlers[cookie](iq, rss, m)); 401 } 402 403 static int 404 act_open_rpl_handler(struct sge_iq *iq, const struct rss_header *rss, 405 struct mbuf *m) 406 { 407 const struct cpl_act_open_rpl *cpl = (const void *)(rss + 1); 408 u_int cookie = G_TID_COOKIE(G_AOPEN_ATID(be32toh(cpl->atid_status))); 409 410 MPASS(m == NULL); 411 MPASS(cookie != CPL_COOKIE_RESERVED); 412 413 return (act_open_rpl_handlers[cookie](iq, rss, m)); 414 } 415 416 static int 417 abort_rpl_rss_handler(struct sge_iq *iq, const struct rss_header *rss, 418 struct mbuf *m) 419 { 420 struct adapter *sc = iq->adapter; 421 u_int cookie; 422 423 MPASS(m == NULL); 424 if (is_hashfilter(sc)) 425 cookie = CPL_COOKIE_HASHFILTER; 426 else 427 cookie = CPL_COOKIE_TOM; 428 429 return (abort_rpl_rss_handlers[cookie](iq, rss, m)); 430 } 431 432 static int 433 fw4_ack_handler(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 434 { 435 struct adapter *sc = iq->adapter; 436 const struct cpl_fw4_ack *cpl = (const void *)(rss + 1); 437 unsigned int tid = G_CPL_FW4_ACK_FLOWID(be32toh(OPCODE_TID(cpl))); 438 u_int cookie; 439 440 MPASS(m == NULL); 441 if (is_etid(sc, tid)) 442 cookie = CPL_COOKIE_ETHOFLD; 443 else 444 cookie = CPL_COOKIE_TOM; 445 446 return (fw4_ack_handlers[cookie](iq, rss, m)); 447 } 448 449 static void 450 t4_init_shared_cpl_handlers(void) 451 { 452 453 t4_register_cpl_handler(CPL_SET_TCB_RPL, set_tcb_rpl_handler); 454 t4_register_cpl_handler(CPL_L2T_WRITE_RPL, l2t_write_rpl_handler); 455 t4_register_cpl_handler(CPL_ACT_OPEN_RPL, act_open_rpl_handler); 456 t4_register_cpl_handler(CPL_ABORT_RPL_RSS, abort_rpl_rss_handler); 457 t4_register_cpl_handler(CPL_FW4_ACK, fw4_ack_handler); 458 } 459 460 void 461 t4_register_shared_cpl_handler(int opcode, cpl_handler_t h, int cookie) 462 { 463 uintptr_t *loc; 464 465 MPASS(opcode < nitems(t4_cpl_handler)); 466 MPASS(cookie > CPL_COOKIE_RESERVED); 467 MPASS(cookie < NUM_CPL_COOKIES); 468 MPASS(t4_cpl_handler[opcode] != NULL); 469 470 switch (opcode) { 471 case CPL_SET_TCB_RPL: 472 loc = (uintptr_t *)&set_tcb_rpl_handlers[cookie]; 473 break; 474 case CPL_L2T_WRITE_RPL: 475 loc = (uintptr_t *)&l2t_write_rpl_handlers[cookie]; 476 break; 477 case CPL_ACT_OPEN_RPL: 478 loc = (uintptr_t *)&act_open_rpl_handlers[cookie]; 479 break; 480 case CPL_ABORT_RPL_RSS: 481 loc = (uintptr_t *)&abort_rpl_rss_handlers[cookie]; 482 break; 483 case CPL_FW4_ACK: 484 loc = (uintptr_t *)&fw4_ack_handlers[cookie]; 485 break; 486 default: 487 MPASS(0); 488 return; 489 } 490 MPASS(h == NULL || *loc == (uintptr_t)NULL); 491 atomic_store_rel_ptr(loc, (uintptr_t)h); 492 } 493 494 /* 495 * Called on MOD_LOAD. Validates and calculates the SGE tunables. 496 */ 497 void 498 t4_sge_modload(void) 499 { 500 501 if (fl_pktshift < 0 || fl_pktshift > 7) { 502 printf("Invalid hw.cxgbe.fl_pktshift value (%d)," 503 " using 0 instead.\n", fl_pktshift); 504 fl_pktshift = 0; 505 } 506 507 if (spg_len != 64 && spg_len != 128) { 508 int len; 509 510 #if defined(__i386__) || defined(__amd64__) 511 len = cpu_clflush_line_size > 64 ? 128 : 64; 512 #else 513 len = 64; 514 #endif 515 if (spg_len != -1) { 516 printf("Invalid hw.cxgbe.spg_len value (%d)," 517 " using %d instead.\n", spg_len, len); 518 } 519 spg_len = len; 520 } 521 522 if (cong_drop < -1 || cong_drop > 1) { 523 printf("Invalid hw.cxgbe.cong_drop value (%d)," 524 " using 0 instead.\n", cong_drop); 525 cong_drop = 0; 526 } 527 528 if (tscale != 1 && (tscale < 3 || tscale > 17)) { 529 printf("Invalid hw.cxgbe.tscale value (%d)," 530 " using 1 instead.\n", tscale); 531 tscale = 1; 532 } 533 534 extfree_refs = counter_u64_alloc(M_WAITOK); 535 extfree_rels = counter_u64_alloc(M_WAITOK); 536 counter_u64_zero(extfree_refs); 537 counter_u64_zero(extfree_rels); 538 539 t4_init_shared_cpl_handlers(); 540 t4_register_cpl_handler(CPL_FW4_MSG, handle_fw_msg); 541 t4_register_cpl_handler(CPL_FW6_MSG, handle_fw_msg); 542 t4_register_cpl_handler(CPL_SGE_EGR_UPDATE, handle_sge_egr_update); 543 #ifdef RATELIMIT 544 t4_register_shared_cpl_handler(CPL_FW4_ACK, ethofld_fw4_ack, 545 CPL_COOKIE_ETHOFLD); 546 #endif 547 t4_register_fw_msg_handler(FW6_TYPE_CMD_RPL, t4_handle_fw_rpl); 548 t4_register_fw_msg_handler(FW6_TYPE_WRERR_RPL, t4_handle_wrerr_rpl); 549 } 550 551 void 552 t4_sge_modunload(void) 553 { 554 555 counter_u64_free(extfree_refs); 556 counter_u64_free(extfree_rels); 557 } 558 559 uint64_t 560 t4_sge_extfree_refs(void) 561 { 562 uint64_t refs, rels; 563 564 rels = counter_u64_fetch(extfree_rels); 565 refs = counter_u64_fetch(extfree_refs); 566 567 return (refs - rels); 568 } 569 570 /* max 4096 */ 571 #define MAX_PACK_BOUNDARY 512 572 573 static inline void 574 setup_pad_and_pack_boundaries(struct adapter *sc) 575 { 576 uint32_t v, m; 577 int pad, pack, pad_shift; 578 579 pad_shift = chip_id(sc) > CHELSIO_T5 ? X_T6_INGPADBOUNDARY_SHIFT : 580 X_INGPADBOUNDARY_SHIFT; 581 pad = fl_pad; 582 if (fl_pad < (1 << pad_shift) || 583 fl_pad > (1 << (pad_shift + M_INGPADBOUNDARY)) || 584 !powerof2(fl_pad)) { 585 /* 586 * If there is any chance that we might use buffer packing and 587 * the chip is a T4, then pick 64 as the pad/pack boundary. Set 588 * it to the minimum allowed in all other cases. 589 */ 590 pad = is_t4(sc) && buffer_packing ? 64 : 1 << pad_shift; 591 592 /* 593 * For fl_pad = 0 we'll still write a reasonable value to the 594 * register but all the freelists will opt out of padding. 595 * We'll complain here only if the user tried to set it to a 596 * value greater than 0 that was invalid. 597 */ 598 if (fl_pad > 0) { 599 device_printf(sc->dev, "Invalid hw.cxgbe.fl_pad value" 600 " (%d), using %d instead.\n", fl_pad, pad); 601 } 602 } 603 m = V_INGPADBOUNDARY(M_INGPADBOUNDARY); 604 v = V_INGPADBOUNDARY(ilog2(pad) - pad_shift); 605 t4_set_reg_field(sc, A_SGE_CONTROL, m, v); 606 607 if (is_t4(sc)) { 608 if (fl_pack != -1 && fl_pack != pad) { 609 /* Complain but carry on. */ 610 device_printf(sc->dev, "hw.cxgbe.fl_pack (%d) ignored," 611 " using %d instead.\n", fl_pack, pad); 612 } 613 return; 614 } 615 616 pack = fl_pack; 617 if (fl_pack < 16 || fl_pack == 32 || fl_pack > 4096 || 618 !powerof2(fl_pack)) { 619 if (sc->params.pci.mps > MAX_PACK_BOUNDARY) 620 pack = MAX_PACK_BOUNDARY; 621 else 622 pack = max(sc->params.pci.mps, CACHE_LINE_SIZE); 623 MPASS(powerof2(pack)); 624 if (pack < 16) 625 pack = 16; 626 if (pack == 32) 627 pack = 64; 628 if (pack > 4096) 629 pack = 4096; 630 if (fl_pack != -1) { 631 device_printf(sc->dev, "Invalid hw.cxgbe.fl_pack value" 632 " (%d), using %d instead.\n", fl_pack, pack); 633 } 634 } 635 m = V_INGPACKBOUNDARY(M_INGPACKBOUNDARY); 636 if (pack == 16) 637 v = V_INGPACKBOUNDARY(0); 638 else 639 v = V_INGPACKBOUNDARY(ilog2(pack) - 5); 640 641 MPASS(!is_t4(sc)); /* T4 doesn't have SGE_CONTROL2 */ 642 t4_set_reg_field(sc, A_SGE_CONTROL2, m, v); 643 } 644 645 /* 646 * adap->params.vpd.cclk must be set up before this is called. 647 */ 648 void 649 t4_tweak_chip_settings(struct adapter *sc) 650 { 651 int i, reg; 652 uint32_t v, m; 653 int intr_timer[SGE_NTIMERS] = {1, 5, 10, 50, 100, 200}; 654 int timer_max = M_TIMERVALUE0 * 1000 / sc->params.vpd.cclk; 655 int intr_pktcount[SGE_NCOUNTERS] = {1, 8, 16, 32}; /* 63 max */ 656 uint16_t indsz = min(RX_COPY_THRESHOLD - 1, M_INDICATESIZE); 657 static int sw_buf_sizes[] = { 658 MCLBYTES, 659 #if MJUMPAGESIZE != MCLBYTES 660 MJUMPAGESIZE, 661 #endif 662 MJUM9BYTES, 663 MJUM16BYTES 664 }; 665 666 KASSERT(sc->flags & MASTER_PF, 667 ("%s: trying to change chip settings when not master.", __func__)); 668 669 m = V_PKTSHIFT(M_PKTSHIFT) | F_RXPKTCPLMODE | F_EGRSTATUSPAGESIZE; 670 v = V_PKTSHIFT(fl_pktshift) | F_RXPKTCPLMODE | 671 V_EGRSTATUSPAGESIZE(spg_len == 128); 672 t4_set_reg_field(sc, A_SGE_CONTROL, m, v); 673 674 setup_pad_and_pack_boundaries(sc); 675 676 v = V_HOSTPAGESIZEPF0(PAGE_SHIFT - 10) | 677 V_HOSTPAGESIZEPF1(PAGE_SHIFT - 10) | 678 V_HOSTPAGESIZEPF2(PAGE_SHIFT - 10) | 679 V_HOSTPAGESIZEPF3(PAGE_SHIFT - 10) | 680 V_HOSTPAGESIZEPF4(PAGE_SHIFT - 10) | 681 V_HOSTPAGESIZEPF5(PAGE_SHIFT - 10) | 682 V_HOSTPAGESIZEPF6(PAGE_SHIFT - 10) | 683 V_HOSTPAGESIZEPF7(PAGE_SHIFT - 10); 684 t4_write_reg(sc, A_SGE_HOST_PAGE_SIZE, v); 685 686 t4_write_reg(sc, A_SGE_FL_BUFFER_SIZE0, 4096); 687 t4_write_reg(sc, A_SGE_FL_BUFFER_SIZE1, 65536); 688 reg = A_SGE_FL_BUFFER_SIZE2; 689 for (i = 0; i < nitems(sw_buf_sizes); i++) { 690 MPASS(reg <= A_SGE_FL_BUFFER_SIZE15); 691 t4_write_reg(sc, reg, sw_buf_sizes[i]); 692 reg += 4; 693 MPASS(reg <= A_SGE_FL_BUFFER_SIZE15); 694 t4_write_reg(sc, reg, sw_buf_sizes[i] - CL_METADATA_SIZE); 695 reg += 4; 696 } 697 698 v = V_THRESHOLD_0(intr_pktcount[0]) | V_THRESHOLD_1(intr_pktcount[1]) | 699 V_THRESHOLD_2(intr_pktcount[2]) | V_THRESHOLD_3(intr_pktcount[3]); 700 t4_write_reg(sc, A_SGE_INGRESS_RX_THRESHOLD, v); 701 702 KASSERT(intr_timer[0] <= timer_max, 703 ("%s: not a single usable timer (%d, %d)", __func__, intr_timer[0], 704 timer_max)); 705 for (i = 1; i < nitems(intr_timer); i++) { 706 KASSERT(intr_timer[i] >= intr_timer[i - 1], 707 ("%s: timers not listed in increasing order (%d)", 708 __func__, i)); 709 710 while (intr_timer[i] > timer_max) { 711 if (i == nitems(intr_timer) - 1) { 712 intr_timer[i] = timer_max; 713 break; 714 } 715 intr_timer[i] += intr_timer[i - 1]; 716 intr_timer[i] /= 2; 717 } 718 } 719 720 v = V_TIMERVALUE0(us_to_core_ticks(sc, intr_timer[0])) | 721 V_TIMERVALUE1(us_to_core_ticks(sc, intr_timer[1])); 722 t4_write_reg(sc, A_SGE_TIMER_VALUE_0_AND_1, v); 723 v = V_TIMERVALUE2(us_to_core_ticks(sc, intr_timer[2])) | 724 V_TIMERVALUE3(us_to_core_ticks(sc, intr_timer[3])); 725 t4_write_reg(sc, A_SGE_TIMER_VALUE_2_AND_3, v); 726 v = V_TIMERVALUE4(us_to_core_ticks(sc, intr_timer[4])) | 727 V_TIMERVALUE5(us_to_core_ticks(sc, intr_timer[5])); 728 t4_write_reg(sc, A_SGE_TIMER_VALUE_4_AND_5, v); 729 730 if (chip_id(sc) >= CHELSIO_T6) { 731 m = V_TSCALE(M_TSCALE); 732 if (tscale == 1) 733 v = 0; 734 else 735 v = V_TSCALE(tscale - 2); 736 t4_set_reg_field(sc, A_SGE_ITP_CONTROL, m, v); 737 738 if (sc->debug_flags & DF_DISABLE_TCB_CACHE) { 739 m = V_RDTHRESHOLD(M_RDTHRESHOLD) | F_WRTHRTHRESHEN | 740 V_WRTHRTHRESH(M_WRTHRTHRESH); 741 t4_tp_pio_read(sc, &v, 1, A_TP_CMM_CONFIG, 1); 742 v &= ~m; 743 v |= V_RDTHRESHOLD(1) | F_WRTHRTHRESHEN | 744 V_WRTHRTHRESH(16); 745 t4_tp_pio_write(sc, &v, 1, A_TP_CMM_CONFIG, 1); 746 } 747 } 748 749 /* 4K, 16K, 64K, 256K DDP "page sizes" for TDDP */ 750 v = V_HPZ0(0) | V_HPZ1(2) | V_HPZ2(4) | V_HPZ3(6); 751 t4_write_reg(sc, A_ULP_RX_TDDP_PSZ, v); 752 753 /* 754 * 4K, 8K, 16K, 64K DDP "page sizes" for iSCSI DDP. These have been 755 * chosen with MAXPHYS = 128K in mind. The largest DDP buffer that we 756 * may have to deal with is MAXPHYS + 1 page. 757 */ 758 v = V_HPZ0(0) | V_HPZ1(1) | V_HPZ2(2) | V_HPZ3(4); 759 t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, v); 760 761 /* We use multiple DDP page sizes both in plain-TOE and ISCSI modes. */ 762 m = v = F_TDDPTAGTCB | F_ISCSITAGTCB; 763 t4_set_reg_field(sc, A_ULP_RX_CTL, m, v); 764 765 m = V_INDICATESIZE(M_INDICATESIZE) | F_REARMDDPOFFSET | 766 F_RESETDDPOFFSET; 767 v = V_INDICATESIZE(indsz) | F_REARMDDPOFFSET | F_RESETDDPOFFSET; 768 t4_set_reg_field(sc, A_TP_PARA_REG5, m, v); 769 } 770 771 /* 772 * SGE wants the buffer to be at least 64B and then a multiple of 16. Its 773 * address mut be 16B aligned. If padding is in use the buffer's start and end 774 * need to be aligned to the pad boundary as well. We'll just make sure that 775 * the size is a multiple of the pad boundary here, it is up to the buffer 776 * allocation code to make sure the start of the buffer is aligned. 777 */ 778 static inline int 779 hwsz_ok(struct adapter *sc, int hwsz) 780 { 781 int mask = fl_pad ? sc->params.sge.pad_boundary - 1 : 16 - 1; 782 783 return (hwsz >= 64 && (hwsz & mask) == 0); 784 } 785 786 /* 787 * XXX: driver really should be able to deal with unexpected settings. 788 */ 789 int 790 t4_read_chip_settings(struct adapter *sc) 791 { 792 struct sge *s = &sc->sge; 793 struct sge_params *sp = &sc->params.sge; 794 int i, j, n, rc = 0; 795 uint32_t m, v, r; 796 uint16_t indsz = min(RX_COPY_THRESHOLD - 1, M_INDICATESIZE); 797 static int sw_buf_sizes[] = { /* Sorted by size */ 798 MCLBYTES, 799 #if MJUMPAGESIZE != MCLBYTES 800 MJUMPAGESIZE, 801 #endif 802 MJUM9BYTES, 803 MJUM16BYTES 804 }; 805 struct rx_buf_info *rxb; 806 807 m = F_RXPKTCPLMODE; 808 v = F_RXPKTCPLMODE; 809 r = sc->params.sge.sge_control; 810 if ((r & m) != v) { 811 device_printf(sc->dev, "invalid SGE_CONTROL(0x%x)\n", r); 812 rc = EINVAL; 813 } 814 815 /* 816 * If this changes then every single use of PAGE_SHIFT in the driver 817 * needs to be carefully reviewed for PAGE_SHIFT vs sp->page_shift. 818 */ 819 if (sp->page_shift != PAGE_SHIFT) { 820 device_printf(sc->dev, "invalid SGE_HOST_PAGE_SIZE(0x%x)\n", r); 821 rc = EINVAL; 822 } 823 824 s->safe_zidx = -1; 825 rxb = &s->rx_buf_info[0]; 826 for (i = 0; i < SW_ZONE_SIZES; i++, rxb++) { 827 rxb->size1 = sw_buf_sizes[i]; 828 rxb->zone = m_getzone(rxb->size1); 829 rxb->type = m_gettype(rxb->size1); 830 rxb->size2 = 0; 831 rxb->hwidx1 = -1; 832 rxb->hwidx2 = -1; 833 for (j = 0; j < SGE_FLBUF_SIZES; j++) { 834 int hwsize = sp->sge_fl_buffer_size[j]; 835 836 if (!hwsz_ok(sc, hwsize)) 837 continue; 838 839 /* hwidx for size1 */ 840 if (rxb->hwidx1 == -1 && rxb->size1 == hwsize) 841 rxb->hwidx1 = j; 842 843 /* hwidx for size2 (buffer packing) */ 844 if (rxb->size1 - CL_METADATA_SIZE < hwsize) 845 continue; 846 n = rxb->size1 - hwsize - CL_METADATA_SIZE; 847 if (n == 0) { 848 rxb->hwidx2 = j; 849 rxb->size2 = hwsize; 850 break; /* stop looking */ 851 } 852 if (rxb->hwidx2 != -1) { 853 if (n < sp->sge_fl_buffer_size[rxb->hwidx2] - 854 hwsize - CL_METADATA_SIZE) { 855 rxb->hwidx2 = j; 856 rxb->size2 = hwsize; 857 } 858 } else if (n <= 2 * CL_METADATA_SIZE) { 859 rxb->hwidx2 = j; 860 rxb->size2 = hwsize; 861 } 862 } 863 if (rxb->hwidx2 != -1) 864 sc->flags |= BUF_PACKING_OK; 865 if (s->safe_zidx == -1 && rxb->size1 == safest_rx_cluster) 866 s->safe_zidx = i; 867 } 868 869 if (sc->flags & IS_VF) 870 return (0); 871 872 v = V_HPZ0(0) | V_HPZ1(2) | V_HPZ2(4) | V_HPZ3(6); 873 r = t4_read_reg(sc, A_ULP_RX_TDDP_PSZ); 874 if (r != v) { 875 device_printf(sc->dev, "invalid ULP_RX_TDDP_PSZ(0x%x)\n", r); 876 rc = EINVAL; 877 } 878 879 m = v = F_TDDPTAGTCB; 880 r = t4_read_reg(sc, A_ULP_RX_CTL); 881 if ((r & m) != v) { 882 device_printf(sc->dev, "invalid ULP_RX_CTL(0x%x)\n", r); 883 rc = EINVAL; 884 } 885 886 m = V_INDICATESIZE(M_INDICATESIZE) | F_REARMDDPOFFSET | 887 F_RESETDDPOFFSET; 888 v = V_INDICATESIZE(indsz) | F_REARMDDPOFFSET | F_RESETDDPOFFSET; 889 r = t4_read_reg(sc, A_TP_PARA_REG5); 890 if ((r & m) != v) { 891 device_printf(sc->dev, "invalid TP_PARA_REG5(0x%x)\n", r); 892 rc = EINVAL; 893 } 894 895 t4_init_tp_params(sc, 1); 896 897 t4_read_mtu_tbl(sc, sc->params.mtus, NULL); 898 t4_load_mtus(sc, sc->params.mtus, sc->params.a_wnd, sc->params.b_wnd); 899 900 return (rc); 901 } 902 903 int 904 t4_create_dma_tag(struct adapter *sc) 905 { 906 int rc; 907 908 rc = bus_dma_tag_create(bus_get_dma_tag(sc->dev), 1, 0, 909 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, BUS_SPACE_MAXSIZE, 910 BUS_SPACE_UNRESTRICTED, BUS_SPACE_MAXSIZE, BUS_DMA_ALLOCNOW, NULL, 911 NULL, &sc->dmat); 912 if (rc != 0) { 913 device_printf(sc->dev, 914 "failed to create main DMA tag: %d\n", rc); 915 } 916 917 return (rc); 918 } 919 920 void 921 t4_sge_sysctls(struct adapter *sc, struct sysctl_ctx_list *ctx, 922 struct sysctl_oid_list *children) 923 { 924 struct sge_params *sp = &sc->params.sge; 925 926 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "buffer_sizes", 927 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT, sc, 0, 928 sysctl_bufsizes, "A", "freelist buffer sizes"); 929 930 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "fl_pktshift", CTLFLAG_RD, 931 NULL, sp->fl_pktshift, "payload DMA offset in rx buffer (bytes)"); 932 933 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "fl_pad", CTLFLAG_RD, 934 NULL, sp->pad_boundary, "payload pad boundary (bytes)"); 935 936 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "spg_len", CTLFLAG_RD, 937 NULL, sp->spg_len, "status page size (bytes)"); 938 939 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "cong_drop", CTLFLAG_RD, 940 NULL, cong_drop, "congestion drop setting"); 941 942 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "fl_pack", CTLFLAG_RD, 943 NULL, sp->pack_boundary, "payload pack boundary (bytes)"); 944 } 945 946 int 947 t4_destroy_dma_tag(struct adapter *sc) 948 { 949 if (sc->dmat) 950 bus_dma_tag_destroy(sc->dmat); 951 952 return (0); 953 } 954 955 /* 956 * Allocate and initialize the firmware event queue, control queues, and special 957 * purpose rx queues owned by the adapter. 958 * 959 * Returns errno on failure. Resources allocated up to that point may still be 960 * allocated. Caller is responsible for cleanup in case this function fails. 961 */ 962 int 963 t4_setup_adapter_queues(struct adapter *sc) 964 { 965 struct sysctl_oid *oid; 966 struct sysctl_oid_list *children; 967 int rc, i; 968 969 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 970 971 sysctl_ctx_init(&sc->ctx); 972 sc->flags |= ADAP_SYSCTL_CTX; 973 974 /* 975 * Firmware event queue 976 */ 977 rc = alloc_fwq(sc); 978 if (rc != 0) 979 return (rc); 980 981 /* 982 * That's all for the VF driver. 983 */ 984 if (sc->flags & IS_VF) 985 return (rc); 986 987 oid = device_get_sysctl_tree(sc->dev); 988 children = SYSCTL_CHILDREN(oid); 989 990 /* 991 * XXX: General purpose rx queues, one per port. 992 */ 993 994 /* 995 * Control queues, one per port. 996 */ 997 oid = SYSCTL_ADD_NODE(&sc->ctx, children, OID_AUTO, "ctrlq", 998 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "control queues"); 999 for_each_port(sc, i) { 1000 struct sge_wrq *ctrlq = &sc->sge.ctrlq[i]; 1001 1002 rc = alloc_ctrlq(sc, ctrlq, i, oid); 1003 if (rc != 0) 1004 return (rc); 1005 } 1006 1007 return (rc); 1008 } 1009 1010 /* 1011 * Idempotent 1012 */ 1013 int 1014 t4_teardown_adapter_queues(struct adapter *sc) 1015 { 1016 int i; 1017 1018 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 1019 1020 /* Do this before freeing the queue */ 1021 if (sc->flags & ADAP_SYSCTL_CTX) { 1022 sysctl_ctx_free(&sc->ctx); 1023 sc->flags &= ~ADAP_SYSCTL_CTX; 1024 } 1025 1026 if (!(sc->flags & IS_VF)) { 1027 for_each_port(sc, i) 1028 free_wrq(sc, &sc->sge.ctrlq[i]); 1029 } 1030 free_fwq(sc); 1031 1032 return (0); 1033 } 1034 1035 /* Maximum payload that can be delivered with a single iq descriptor */ 1036 static inline int 1037 mtu_to_max_payload(struct adapter *sc, int mtu) 1038 { 1039 1040 /* large enough even when hw VLAN extraction is disabled */ 1041 return (sc->params.sge.fl_pktshift + ETHER_HDR_LEN + 1042 ETHER_VLAN_ENCAP_LEN + mtu); 1043 } 1044 1045 int 1046 t4_setup_vi_queues(struct vi_info *vi) 1047 { 1048 int rc = 0, i, intr_idx, iqidx; 1049 struct sge_rxq *rxq; 1050 struct sge_txq *txq; 1051 #ifdef TCP_OFFLOAD 1052 struct sge_ofld_rxq *ofld_rxq; 1053 #endif 1054 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 1055 struct sge_wrq *ofld_txq; 1056 #endif 1057 #ifdef DEV_NETMAP 1058 int saved_idx; 1059 struct sge_nm_rxq *nm_rxq; 1060 struct sge_nm_txq *nm_txq; 1061 #endif 1062 char name[16]; 1063 struct port_info *pi = vi->pi; 1064 struct adapter *sc = pi->adapter; 1065 struct ifnet *ifp = vi->ifp; 1066 struct sysctl_oid *oid = device_get_sysctl_tree(vi->dev); 1067 struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); 1068 int maxp, mtu = ifp->if_mtu; 1069 1070 /* Interrupt vector to start from (when using multiple vectors) */ 1071 intr_idx = vi->first_intr; 1072 1073 #ifdef DEV_NETMAP 1074 saved_idx = intr_idx; 1075 if (ifp->if_capabilities & IFCAP_NETMAP) { 1076 1077 /* netmap is supported with direct interrupts only. */ 1078 MPASS(!forwarding_intr_to_fwq(sc)); 1079 1080 /* 1081 * We don't have buffers to back the netmap rx queues 1082 * right now so we create the queues in a way that 1083 * doesn't set off any congestion signal in the chip. 1084 */ 1085 oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "nm_rxq", 1086 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "rx queues"); 1087 for_each_nm_rxq(vi, i, nm_rxq) { 1088 rc = alloc_nm_rxq(vi, nm_rxq, intr_idx, i, oid); 1089 if (rc != 0) 1090 goto done; 1091 intr_idx++; 1092 } 1093 1094 oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "nm_txq", 1095 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "tx queues"); 1096 for_each_nm_txq(vi, i, nm_txq) { 1097 iqidx = vi->first_nm_rxq + (i % vi->nnmrxq); 1098 rc = alloc_nm_txq(vi, nm_txq, iqidx, i, oid); 1099 if (rc != 0) 1100 goto done; 1101 } 1102 } 1103 1104 /* Normal rx queues and netmap rx queues share the same interrupts. */ 1105 intr_idx = saved_idx; 1106 #endif 1107 1108 /* 1109 * Allocate rx queues first because a default iqid is required when 1110 * creating a tx queue. 1111 */ 1112 maxp = mtu_to_max_payload(sc, mtu); 1113 oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "rxq", 1114 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "rx queues"); 1115 for_each_rxq(vi, i, rxq) { 1116 1117 init_iq(&rxq->iq, sc, vi->tmr_idx, vi->pktc_idx, vi->qsize_rxq); 1118 1119 snprintf(name, sizeof(name), "%s rxq%d-fl", 1120 device_get_nameunit(vi->dev), i); 1121 init_fl(sc, &rxq->fl, vi->qsize_rxq / 8, maxp, name); 1122 1123 rc = alloc_rxq(vi, rxq, 1124 forwarding_intr_to_fwq(sc) ? -1 : intr_idx, i, oid); 1125 if (rc != 0) 1126 goto done; 1127 intr_idx++; 1128 } 1129 #ifdef DEV_NETMAP 1130 if (ifp->if_capabilities & IFCAP_NETMAP) 1131 intr_idx = saved_idx + max(vi->nrxq, vi->nnmrxq); 1132 #endif 1133 #ifdef TCP_OFFLOAD 1134 oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "ofld_rxq", 1135 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "rx queues for offloaded TCP connections"); 1136 for_each_ofld_rxq(vi, i, ofld_rxq) { 1137 1138 init_iq(&ofld_rxq->iq, sc, vi->ofld_tmr_idx, vi->ofld_pktc_idx, 1139 vi->qsize_rxq); 1140 1141 snprintf(name, sizeof(name), "%s ofld_rxq%d-fl", 1142 device_get_nameunit(vi->dev), i); 1143 init_fl(sc, &ofld_rxq->fl, vi->qsize_rxq / 8, maxp, name); 1144 1145 rc = alloc_ofld_rxq(vi, ofld_rxq, 1146 forwarding_intr_to_fwq(sc) ? -1 : intr_idx, i, oid); 1147 if (rc != 0) 1148 goto done; 1149 intr_idx++; 1150 } 1151 #endif 1152 1153 /* 1154 * Now the tx queues. 1155 */ 1156 oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "txq", 1157 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "tx queues"); 1158 for_each_txq(vi, i, txq) { 1159 iqidx = vi->first_rxq + (i % vi->nrxq); 1160 snprintf(name, sizeof(name), "%s txq%d", 1161 device_get_nameunit(vi->dev), i); 1162 init_eq(sc, &txq->eq, EQ_ETH, vi->qsize_txq, pi->tx_chan, 1163 sc->sge.rxq[iqidx].iq.cntxt_id, name); 1164 1165 rc = alloc_txq(vi, txq, i, oid); 1166 if (rc != 0) 1167 goto done; 1168 } 1169 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 1170 oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "ofld_txq", 1171 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "tx queues for TOE/ETHOFLD"); 1172 for_each_ofld_txq(vi, i, ofld_txq) { 1173 struct sysctl_oid *oid2; 1174 1175 snprintf(name, sizeof(name), "%s ofld_txq%d", 1176 device_get_nameunit(vi->dev), i); 1177 if (vi->nofldrxq > 0) { 1178 iqidx = vi->first_ofld_rxq + (i % vi->nofldrxq); 1179 init_eq(sc, &ofld_txq->eq, EQ_OFLD, vi->qsize_txq, 1180 pi->tx_chan, sc->sge.ofld_rxq[iqidx].iq.cntxt_id, 1181 name); 1182 } else { 1183 iqidx = vi->first_rxq + (i % vi->nrxq); 1184 init_eq(sc, &ofld_txq->eq, EQ_OFLD, vi->qsize_txq, 1185 pi->tx_chan, sc->sge.rxq[iqidx].iq.cntxt_id, name); 1186 } 1187 1188 snprintf(name, sizeof(name), "%d", i); 1189 oid2 = SYSCTL_ADD_NODE(&vi->ctx, SYSCTL_CHILDREN(oid), OID_AUTO, 1190 name, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "offload tx queue"); 1191 1192 rc = alloc_wrq(sc, vi, ofld_txq, oid2); 1193 if (rc != 0) 1194 goto done; 1195 } 1196 #endif 1197 done: 1198 if (rc) 1199 t4_teardown_vi_queues(vi); 1200 1201 return (rc); 1202 } 1203 1204 /* 1205 * Idempotent 1206 */ 1207 int 1208 t4_teardown_vi_queues(struct vi_info *vi) 1209 { 1210 int i; 1211 struct sge_rxq *rxq; 1212 struct sge_txq *txq; 1213 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 1214 struct port_info *pi = vi->pi; 1215 struct adapter *sc = pi->adapter; 1216 struct sge_wrq *ofld_txq; 1217 #endif 1218 #ifdef TCP_OFFLOAD 1219 struct sge_ofld_rxq *ofld_rxq; 1220 #endif 1221 #ifdef DEV_NETMAP 1222 struct sge_nm_rxq *nm_rxq; 1223 struct sge_nm_txq *nm_txq; 1224 #endif 1225 1226 /* Do this before freeing the queues */ 1227 if (vi->flags & VI_SYSCTL_CTX) { 1228 sysctl_ctx_free(&vi->ctx); 1229 vi->flags &= ~VI_SYSCTL_CTX; 1230 } 1231 1232 #ifdef DEV_NETMAP 1233 if (vi->ifp->if_capabilities & IFCAP_NETMAP) { 1234 for_each_nm_txq(vi, i, nm_txq) { 1235 free_nm_txq(vi, nm_txq); 1236 } 1237 1238 for_each_nm_rxq(vi, i, nm_rxq) { 1239 free_nm_rxq(vi, nm_rxq); 1240 } 1241 } 1242 #endif 1243 1244 /* 1245 * Take down all the tx queues first, as they reference the rx queues 1246 * (for egress updates, etc.). 1247 */ 1248 1249 for_each_txq(vi, i, txq) { 1250 free_txq(vi, txq); 1251 } 1252 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 1253 for_each_ofld_txq(vi, i, ofld_txq) { 1254 free_wrq(sc, ofld_txq); 1255 } 1256 #endif 1257 1258 /* 1259 * Then take down the rx queues. 1260 */ 1261 1262 for_each_rxq(vi, i, rxq) { 1263 free_rxq(vi, rxq); 1264 } 1265 #ifdef TCP_OFFLOAD 1266 for_each_ofld_rxq(vi, i, ofld_rxq) { 1267 free_ofld_rxq(vi, ofld_rxq); 1268 } 1269 #endif 1270 1271 return (0); 1272 } 1273 1274 /* 1275 * Interrupt handler when the driver is using only 1 interrupt. This is a very 1276 * unusual scenario. 1277 * 1278 * a) Deals with errors, if any. 1279 * b) Services firmware event queue, which is taking interrupts for all other 1280 * queues. 1281 */ 1282 void 1283 t4_intr_all(void *arg) 1284 { 1285 struct adapter *sc = arg; 1286 struct sge_iq *fwq = &sc->sge.fwq; 1287 1288 MPASS(sc->intr_count == 1); 1289 1290 if (sc->intr_type == INTR_INTX) 1291 t4_write_reg(sc, MYPF_REG(A_PCIE_PF_CLI), 0); 1292 1293 t4_intr_err(arg); 1294 t4_intr_evt(fwq); 1295 } 1296 1297 /* 1298 * Interrupt handler for errors (installed directly when multiple interrupts are 1299 * being used, or called by t4_intr_all). 1300 */ 1301 void 1302 t4_intr_err(void *arg) 1303 { 1304 struct adapter *sc = arg; 1305 uint32_t v; 1306 const bool verbose = (sc->debug_flags & DF_VERBOSE_SLOWINTR) != 0; 1307 1308 if (sc->flags & ADAP_ERR) 1309 return; 1310 1311 v = t4_read_reg(sc, MYPF_REG(A_PL_PF_INT_CAUSE)); 1312 if (v & F_PFSW) { 1313 sc->swintr++; 1314 t4_write_reg(sc, MYPF_REG(A_PL_PF_INT_CAUSE), v); 1315 } 1316 1317 t4_slow_intr_handler(sc, verbose); 1318 } 1319 1320 /* 1321 * Interrupt handler for iq-only queues. The firmware event queue is the only 1322 * such queue right now. 1323 */ 1324 void 1325 t4_intr_evt(void *arg) 1326 { 1327 struct sge_iq *iq = arg; 1328 1329 if (atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_BUSY)) { 1330 service_iq(iq, 0); 1331 (void) atomic_cmpset_int(&iq->state, IQS_BUSY, IQS_IDLE); 1332 } 1333 } 1334 1335 /* 1336 * Interrupt handler for iq+fl queues. 1337 */ 1338 void 1339 t4_intr(void *arg) 1340 { 1341 struct sge_iq *iq = arg; 1342 1343 if (atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_BUSY)) { 1344 service_iq_fl(iq, 0); 1345 (void) atomic_cmpset_int(&iq->state, IQS_BUSY, IQS_IDLE); 1346 } 1347 } 1348 1349 #ifdef DEV_NETMAP 1350 /* 1351 * Interrupt handler for netmap rx queues. 1352 */ 1353 void 1354 t4_nm_intr(void *arg) 1355 { 1356 struct sge_nm_rxq *nm_rxq = arg; 1357 1358 if (atomic_cmpset_int(&nm_rxq->nm_state, NM_ON, NM_BUSY)) { 1359 service_nm_rxq(nm_rxq); 1360 (void) atomic_cmpset_int(&nm_rxq->nm_state, NM_BUSY, NM_ON); 1361 } 1362 } 1363 1364 /* 1365 * Interrupt handler for vectors shared between NIC and netmap rx queues. 1366 */ 1367 void 1368 t4_vi_intr(void *arg) 1369 { 1370 struct irq *irq = arg; 1371 1372 MPASS(irq->nm_rxq != NULL); 1373 t4_nm_intr(irq->nm_rxq); 1374 1375 MPASS(irq->rxq != NULL); 1376 t4_intr(irq->rxq); 1377 } 1378 #endif 1379 1380 /* 1381 * Deals with interrupts on an iq-only (no freelist) queue. 1382 */ 1383 static int 1384 service_iq(struct sge_iq *iq, int budget) 1385 { 1386 struct sge_iq *q; 1387 struct adapter *sc = iq->adapter; 1388 struct iq_desc *d = &iq->desc[iq->cidx]; 1389 int ndescs = 0, limit; 1390 int rsp_type; 1391 uint32_t lq; 1392 STAILQ_HEAD(, sge_iq) iql = STAILQ_HEAD_INITIALIZER(iql); 1393 1394 KASSERT(iq->state == IQS_BUSY, ("%s: iq %p not BUSY", __func__, iq)); 1395 KASSERT((iq->flags & IQ_HAS_FL) == 0, 1396 ("%s: called for iq %p with fl (iq->flags 0x%x)", __func__, iq, 1397 iq->flags)); 1398 MPASS((iq->flags & IQ_ADJ_CREDIT) == 0); 1399 MPASS((iq->flags & IQ_LRO_ENABLED) == 0); 1400 1401 limit = budget ? budget : iq->qsize / 16; 1402 1403 /* 1404 * We always come back and check the descriptor ring for new indirect 1405 * interrupts and other responses after running a single handler. 1406 */ 1407 for (;;) { 1408 while ((d->rsp.u.type_gen & F_RSPD_GEN) == iq->gen) { 1409 1410 rmb(); 1411 1412 rsp_type = G_RSPD_TYPE(d->rsp.u.type_gen); 1413 lq = be32toh(d->rsp.pldbuflen_qid); 1414 1415 switch (rsp_type) { 1416 case X_RSPD_TYPE_FLBUF: 1417 panic("%s: data for an iq (%p) with no freelist", 1418 __func__, iq); 1419 1420 /* NOTREACHED */ 1421 1422 case X_RSPD_TYPE_CPL: 1423 KASSERT(d->rss.opcode < NUM_CPL_CMDS, 1424 ("%s: bad opcode %02x.", __func__, 1425 d->rss.opcode)); 1426 t4_cpl_handler[d->rss.opcode](iq, &d->rss, NULL); 1427 break; 1428 1429 case X_RSPD_TYPE_INTR: 1430 /* 1431 * There are 1K interrupt-capable queues (qids 0 1432 * through 1023). A response type indicating a 1433 * forwarded interrupt with a qid >= 1K is an 1434 * iWARP async notification. 1435 */ 1436 if (__predict_true(lq >= 1024)) { 1437 t4_an_handler(iq, &d->rsp); 1438 break; 1439 } 1440 1441 q = sc->sge.iqmap[lq - sc->sge.iq_start - 1442 sc->sge.iq_base]; 1443 if (atomic_cmpset_int(&q->state, IQS_IDLE, 1444 IQS_BUSY)) { 1445 if (service_iq_fl(q, q->qsize / 16) == 0) { 1446 (void) atomic_cmpset_int(&q->state, 1447 IQS_BUSY, IQS_IDLE); 1448 } else { 1449 STAILQ_INSERT_TAIL(&iql, q, 1450 link); 1451 } 1452 } 1453 break; 1454 1455 default: 1456 KASSERT(0, 1457 ("%s: illegal response type %d on iq %p", 1458 __func__, rsp_type, iq)); 1459 log(LOG_ERR, 1460 "%s: illegal response type %d on iq %p", 1461 device_get_nameunit(sc->dev), rsp_type, iq); 1462 break; 1463 } 1464 1465 d++; 1466 if (__predict_false(++iq->cidx == iq->sidx)) { 1467 iq->cidx = 0; 1468 iq->gen ^= F_RSPD_GEN; 1469 d = &iq->desc[0]; 1470 } 1471 if (__predict_false(++ndescs == limit)) { 1472 t4_write_reg(sc, sc->sge_gts_reg, 1473 V_CIDXINC(ndescs) | 1474 V_INGRESSQID(iq->cntxt_id) | 1475 V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX))); 1476 ndescs = 0; 1477 1478 if (budget) { 1479 return (EINPROGRESS); 1480 } 1481 } 1482 } 1483 1484 if (STAILQ_EMPTY(&iql)) 1485 break; 1486 1487 /* 1488 * Process the head only, and send it to the back of the list if 1489 * it's still not done. 1490 */ 1491 q = STAILQ_FIRST(&iql); 1492 STAILQ_REMOVE_HEAD(&iql, link); 1493 if (service_iq_fl(q, q->qsize / 8) == 0) 1494 (void) atomic_cmpset_int(&q->state, IQS_BUSY, IQS_IDLE); 1495 else 1496 STAILQ_INSERT_TAIL(&iql, q, link); 1497 } 1498 1499 t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) | 1500 V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_params)); 1501 1502 return (0); 1503 } 1504 1505 static inline int 1506 sort_before_lro(struct lro_ctrl *lro) 1507 { 1508 1509 return (lro->lro_mbuf_max != 0); 1510 } 1511 1512 static inline uint64_t 1513 last_flit_to_ns(struct adapter *sc, uint64_t lf) 1514 { 1515 uint64_t n = be64toh(lf) & 0xfffffffffffffff; /* 60b, not 64b. */ 1516 1517 if (n > UINT64_MAX / 1000000) 1518 return (n / sc->params.vpd.cclk * 1000000); 1519 else 1520 return (n * 1000000 / sc->params.vpd.cclk); 1521 } 1522 1523 static inline void 1524 move_to_next_rxbuf(struct sge_fl *fl) 1525 { 1526 1527 fl->rx_offset = 0; 1528 if (__predict_false((++fl->cidx & 7) == 0)) { 1529 uint16_t cidx = fl->cidx >> 3; 1530 1531 if (__predict_false(cidx == fl->sidx)) 1532 fl->cidx = cidx = 0; 1533 fl->hw_cidx = cidx; 1534 } 1535 } 1536 1537 /* 1538 * Deals with interrupts on an iq+fl queue. 1539 */ 1540 static int 1541 service_iq_fl(struct sge_iq *iq, int budget) 1542 { 1543 struct sge_rxq *rxq = iq_to_rxq(iq); 1544 struct sge_fl *fl; 1545 struct adapter *sc = iq->adapter; 1546 struct iq_desc *d = &iq->desc[iq->cidx]; 1547 int ndescs, limit; 1548 int rsp_type, starved; 1549 uint32_t lq; 1550 uint16_t fl_hw_cidx; 1551 struct mbuf *m0; 1552 #if defined(INET) || defined(INET6) 1553 const struct timeval lro_timeout = {0, sc->lro_timeout}; 1554 struct lro_ctrl *lro = &rxq->lro; 1555 #endif 1556 1557 KASSERT(iq->state == IQS_BUSY, ("%s: iq %p not BUSY", __func__, iq)); 1558 MPASS(iq->flags & IQ_HAS_FL); 1559 1560 ndescs = 0; 1561 #if defined(INET) || defined(INET6) 1562 if (iq->flags & IQ_ADJ_CREDIT) { 1563 MPASS(sort_before_lro(lro)); 1564 iq->flags &= ~IQ_ADJ_CREDIT; 1565 if ((d->rsp.u.type_gen & F_RSPD_GEN) != iq->gen) { 1566 tcp_lro_flush_all(lro); 1567 t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(1) | 1568 V_INGRESSQID((u32)iq->cntxt_id) | 1569 V_SEINTARM(iq->intr_params)); 1570 return (0); 1571 } 1572 ndescs = 1; 1573 } 1574 #else 1575 MPASS((iq->flags & IQ_ADJ_CREDIT) == 0); 1576 #endif 1577 1578 limit = budget ? budget : iq->qsize / 16; 1579 fl = &rxq->fl; 1580 fl_hw_cidx = fl->hw_cidx; /* stable snapshot */ 1581 while ((d->rsp.u.type_gen & F_RSPD_GEN) == iq->gen) { 1582 1583 rmb(); 1584 1585 m0 = NULL; 1586 rsp_type = G_RSPD_TYPE(d->rsp.u.type_gen); 1587 lq = be32toh(d->rsp.pldbuflen_qid); 1588 1589 switch (rsp_type) { 1590 case X_RSPD_TYPE_FLBUF: 1591 if (lq & F_RSPD_NEWBUF) { 1592 if (fl->rx_offset > 0) 1593 move_to_next_rxbuf(fl); 1594 lq = G_RSPD_LEN(lq); 1595 } 1596 if (IDXDIFF(fl->hw_cidx, fl_hw_cidx, fl->sidx) > 4) { 1597 FL_LOCK(fl); 1598 refill_fl(sc, fl, 64); 1599 FL_UNLOCK(fl); 1600 fl_hw_cidx = fl->hw_cidx; 1601 } 1602 1603 if (d->rss.opcode == CPL_RX_PKT) { 1604 if (__predict_true(eth_rx(sc, rxq, d, lq) == 0)) 1605 break; 1606 goto out; 1607 } 1608 m0 = get_fl_payload(sc, fl, lq); 1609 if (__predict_false(m0 == NULL)) 1610 goto out; 1611 1612 /* fall through */ 1613 1614 case X_RSPD_TYPE_CPL: 1615 KASSERT(d->rss.opcode < NUM_CPL_CMDS, 1616 ("%s: bad opcode %02x.", __func__, d->rss.opcode)); 1617 t4_cpl_handler[d->rss.opcode](iq, &d->rss, m0); 1618 break; 1619 1620 case X_RSPD_TYPE_INTR: 1621 1622 /* 1623 * There are 1K interrupt-capable queues (qids 0 1624 * through 1023). A response type indicating a 1625 * forwarded interrupt with a qid >= 1K is an 1626 * iWARP async notification. That is the only 1627 * acceptable indirect interrupt on this queue. 1628 */ 1629 if (__predict_false(lq < 1024)) { 1630 panic("%s: indirect interrupt on iq_fl %p " 1631 "with qid %u", __func__, iq, lq); 1632 } 1633 1634 t4_an_handler(iq, &d->rsp); 1635 break; 1636 1637 default: 1638 KASSERT(0, ("%s: illegal response type %d on iq %p", 1639 __func__, rsp_type, iq)); 1640 log(LOG_ERR, "%s: illegal response type %d on iq %p", 1641 device_get_nameunit(sc->dev), rsp_type, iq); 1642 break; 1643 } 1644 1645 d++; 1646 if (__predict_false(++iq->cidx == iq->sidx)) { 1647 iq->cidx = 0; 1648 iq->gen ^= F_RSPD_GEN; 1649 d = &iq->desc[0]; 1650 } 1651 if (__predict_false(++ndescs == limit)) { 1652 t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) | 1653 V_INGRESSQID(iq->cntxt_id) | 1654 V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX))); 1655 1656 #if defined(INET) || defined(INET6) 1657 if (iq->flags & IQ_LRO_ENABLED && 1658 !sort_before_lro(lro) && 1659 sc->lro_timeout != 0) { 1660 tcp_lro_flush_inactive(lro, &lro_timeout); 1661 } 1662 #endif 1663 if (budget) 1664 return (EINPROGRESS); 1665 ndescs = 0; 1666 } 1667 } 1668 out: 1669 #if defined(INET) || defined(INET6) 1670 if (iq->flags & IQ_LRO_ENABLED) { 1671 if (ndescs > 0 && lro->lro_mbuf_count > 8) { 1672 MPASS(sort_before_lro(lro)); 1673 /* hold back one credit and don't flush LRO state */ 1674 iq->flags |= IQ_ADJ_CREDIT; 1675 ndescs--; 1676 } else { 1677 tcp_lro_flush_all(lro); 1678 } 1679 } 1680 #endif 1681 1682 t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) | 1683 V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_params)); 1684 1685 FL_LOCK(fl); 1686 starved = refill_fl(sc, fl, 64); 1687 FL_UNLOCK(fl); 1688 if (__predict_false(starved != 0)) 1689 add_fl_to_sfl(sc, fl); 1690 1691 return (0); 1692 } 1693 1694 static inline struct cluster_metadata * 1695 cl_metadata(struct fl_sdesc *sd) 1696 { 1697 1698 return ((void *)(sd->cl + sd->moff)); 1699 } 1700 1701 static void 1702 rxb_free(struct mbuf *m) 1703 { 1704 struct cluster_metadata *clm = m->m_ext.ext_arg1; 1705 1706 uma_zfree(clm->zone, clm->cl); 1707 counter_u64_add(extfree_rels, 1); 1708 } 1709 1710 /* 1711 * The mbuf returned comes from zone_muf and carries the payload in one of these 1712 * ways 1713 * a) complete frame inside the mbuf 1714 * b) m_cljset (for clusters without metadata) 1715 * d) m_extaddref (cluster with metadata) 1716 */ 1717 static struct mbuf * 1718 get_scatter_segment(struct adapter *sc, struct sge_fl *fl, int fr_offset, 1719 int remaining) 1720 { 1721 struct mbuf *m; 1722 struct fl_sdesc *sd = &fl->sdesc[fl->cidx]; 1723 struct rx_buf_info *rxb = &sc->sge.rx_buf_info[sd->zidx]; 1724 struct cluster_metadata *clm; 1725 int len, blen; 1726 caddr_t payload; 1727 1728 if (fl->flags & FL_BUF_PACKING) { 1729 u_int l, pad; 1730 1731 blen = rxb->size2 - fl->rx_offset; /* max possible in this buf */ 1732 len = min(remaining, blen); 1733 payload = sd->cl + fl->rx_offset; 1734 1735 l = fr_offset + len; 1736 pad = roundup2(l, fl->buf_boundary) - l; 1737 if (fl->rx_offset + len + pad < rxb->size2) 1738 blen = len + pad; 1739 MPASS(fl->rx_offset + blen <= rxb->size2); 1740 } else { 1741 MPASS(fl->rx_offset == 0); /* not packing */ 1742 blen = rxb->size1; 1743 len = min(remaining, blen); 1744 payload = sd->cl; 1745 } 1746 1747 if (fr_offset == 0) { 1748 m = m_gethdr(M_NOWAIT, MT_DATA); 1749 if (__predict_false(m == NULL)) 1750 return (NULL); 1751 m->m_pkthdr.len = remaining; 1752 } else { 1753 m = m_get(M_NOWAIT, MT_DATA); 1754 if (__predict_false(m == NULL)) 1755 return (NULL); 1756 } 1757 m->m_len = len; 1758 1759 if (sc->sc_do_rxcopy && len < RX_COPY_THRESHOLD) { 1760 /* copy data to mbuf */ 1761 bcopy(payload, mtod(m, caddr_t), len); 1762 if (fl->flags & FL_BUF_PACKING) { 1763 fl->rx_offset += blen; 1764 MPASS(fl->rx_offset <= rxb->size2); 1765 if (fl->rx_offset < rxb->size2) 1766 return (m); /* without advancing the cidx */ 1767 } 1768 } else if (fl->flags & FL_BUF_PACKING) { 1769 clm = cl_metadata(sd); 1770 if (sd->nmbuf++ == 0) { 1771 clm->refcount = 1; 1772 clm->zone = rxb->zone; 1773 clm->cl = sd->cl; 1774 counter_u64_add(extfree_refs, 1); 1775 } 1776 m_extaddref(m, payload, blen, &clm->refcount, rxb_free, clm, 1777 NULL); 1778 1779 fl->rx_offset += blen; 1780 MPASS(fl->rx_offset <= rxb->size2); 1781 if (fl->rx_offset < rxb->size2) 1782 return (m); /* without advancing the cidx */ 1783 } else { 1784 m_cljset(m, sd->cl, rxb->type); 1785 sd->cl = NULL; /* consumed, not a recycle candidate */ 1786 } 1787 1788 move_to_next_rxbuf(fl); 1789 1790 return (m); 1791 } 1792 1793 static struct mbuf * 1794 get_fl_payload(struct adapter *sc, struct sge_fl *fl, const u_int plen) 1795 { 1796 struct mbuf *m0, *m, **pnext; 1797 u_int remaining; 1798 1799 if (__predict_false(fl->flags & FL_BUF_RESUME)) { 1800 M_ASSERTPKTHDR(fl->m0); 1801 MPASS(fl->m0->m_pkthdr.len == plen); 1802 MPASS(fl->remaining < plen); 1803 1804 m0 = fl->m0; 1805 pnext = fl->pnext; 1806 remaining = fl->remaining; 1807 fl->flags &= ~FL_BUF_RESUME; 1808 goto get_segment; 1809 } 1810 1811 /* 1812 * Payload starts at rx_offset in the current hw buffer. Its length is 1813 * 'len' and it may span multiple hw buffers. 1814 */ 1815 1816 m0 = get_scatter_segment(sc, fl, 0, plen); 1817 if (m0 == NULL) 1818 return (NULL); 1819 remaining = plen - m0->m_len; 1820 pnext = &m0->m_next; 1821 while (remaining > 0) { 1822 get_segment: 1823 MPASS(fl->rx_offset == 0); 1824 m = get_scatter_segment(sc, fl, plen - remaining, remaining); 1825 if (__predict_false(m == NULL)) { 1826 fl->m0 = m0; 1827 fl->pnext = pnext; 1828 fl->remaining = remaining; 1829 fl->flags |= FL_BUF_RESUME; 1830 return (NULL); 1831 } 1832 *pnext = m; 1833 pnext = &m->m_next; 1834 remaining -= m->m_len; 1835 } 1836 *pnext = NULL; 1837 1838 M_ASSERTPKTHDR(m0); 1839 return (m0); 1840 } 1841 1842 static int 1843 skip_scatter_segment(struct adapter *sc, struct sge_fl *fl, int fr_offset, 1844 int remaining) 1845 { 1846 struct fl_sdesc *sd = &fl->sdesc[fl->cidx]; 1847 struct rx_buf_info *rxb = &sc->sge.rx_buf_info[sd->zidx]; 1848 int len, blen; 1849 1850 if (fl->flags & FL_BUF_PACKING) { 1851 u_int l, pad; 1852 1853 blen = rxb->size2 - fl->rx_offset; /* max possible in this buf */ 1854 len = min(remaining, blen); 1855 1856 l = fr_offset + len; 1857 pad = roundup2(l, fl->buf_boundary) - l; 1858 if (fl->rx_offset + len + pad < rxb->size2) 1859 blen = len + pad; 1860 fl->rx_offset += blen; 1861 MPASS(fl->rx_offset <= rxb->size2); 1862 if (fl->rx_offset < rxb->size2) 1863 return (len); /* without advancing the cidx */ 1864 } else { 1865 MPASS(fl->rx_offset == 0); /* not packing */ 1866 blen = rxb->size1; 1867 len = min(remaining, blen); 1868 } 1869 move_to_next_rxbuf(fl); 1870 return (len); 1871 } 1872 1873 static inline void 1874 skip_fl_payload(struct adapter *sc, struct sge_fl *fl, int plen) 1875 { 1876 int remaining, fr_offset, len; 1877 1878 fr_offset = 0; 1879 remaining = plen; 1880 while (remaining > 0) { 1881 len = skip_scatter_segment(sc, fl, fr_offset, remaining); 1882 fr_offset += len; 1883 remaining -= len; 1884 } 1885 } 1886 1887 static inline int 1888 get_segment_len(struct adapter *sc, struct sge_fl *fl, int plen) 1889 { 1890 int len; 1891 struct fl_sdesc *sd = &fl->sdesc[fl->cidx]; 1892 struct rx_buf_info *rxb = &sc->sge.rx_buf_info[sd->zidx]; 1893 1894 if (fl->flags & FL_BUF_PACKING) 1895 len = rxb->size2 - fl->rx_offset; 1896 else 1897 len = rxb->size1; 1898 1899 return (min(plen, len)); 1900 } 1901 1902 static int 1903 eth_rx(struct adapter *sc, struct sge_rxq *rxq, const struct iq_desc *d, 1904 u_int plen) 1905 { 1906 struct mbuf *m0; 1907 struct ifnet *ifp = rxq->ifp; 1908 struct sge_fl *fl = &rxq->fl; 1909 struct vi_info *vi = ifp->if_softc; 1910 const struct cpl_rx_pkt *cpl; 1911 #if defined(INET) || defined(INET6) 1912 struct lro_ctrl *lro = &rxq->lro; 1913 #endif 1914 static const int sw_hashtype[4][2] = { 1915 {M_HASHTYPE_NONE, M_HASHTYPE_NONE}, 1916 {M_HASHTYPE_RSS_IPV4, M_HASHTYPE_RSS_IPV6}, 1917 {M_HASHTYPE_RSS_TCP_IPV4, M_HASHTYPE_RSS_TCP_IPV6}, 1918 {M_HASHTYPE_RSS_UDP_IPV4, M_HASHTYPE_RSS_UDP_IPV6}, 1919 }; 1920 1921 MPASS(plen > sc->params.sge.fl_pktshift); 1922 if (vi->pfil != NULL && PFIL_HOOKED_IN(vi->pfil) && 1923 __predict_true((fl->flags & FL_BUF_RESUME) == 0)) { 1924 struct fl_sdesc *sd = &fl->sdesc[fl->cidx]; 1925 caddr_t frame; 1926 int rc, slen; 1927 1928 slen = get_segment_len(sc, fl, plen) - 1929 sc->params.sge.fl_pktshift; 1930 frame = sd->cl + fl->rx_offset + sc->params.sge.fl_pktshift; 1931 CURVNET_SET_QUIET(ifp->if_vnet); 1932 rc = pfil_run_hooks(vi->pfil, frame, ifp, 1933 slen | PFIL_MEMPTR | PFIL_IN, NULL); 1934 CURVNET_RESTORE(); 1935 if (rc == PFIL_DROPPED || rc == PFIL_CONSUMED) { 1936 skip_fl_payload(sc, fl, plen); 1937 return (0); 1938 } 1939 if (rc == PFIL_REALLOCED) { 1940 skip_fl_payload(sc, fl, plen); 1941 m0 = pfil_mem2mbuf(frame); 1942 goto have_mbuf; 1943 } 1944 } 1945 1946 m0 = get_fl_payload(sc, fl, plen); 1947 if (__predict_false(m0 == NULL)) 1948 return (ENOMEM); 1949 1950 m0->m_pkthdr.len -= sc->params.sge.fl_pktshift; 1951 m0->m_len -= sc->params.sge.fl_pktshift; 1952 m0->m_data += sc->params.sge.fl_pktshift; 1953 1954 have_mbuf: 1955 m0->m_pkthdr.rcvif = ifp; 1956 M_HASHTYPE_SET(m0, sw_hashtype[d->rss.hash_type][d->rss.ipv6]); 1957 m0->m_pkthdr.flowid = be32toh(d->rss.hash_val); 1958 1959 cpl = (const void *)(&d->rss + 1); 1960 if (cpl->csum_calc && !(cpl->err_vec & sc->params.tp.err_vec_mask)) { 1961 if (ifp->if_capenable & IFCAP_RXCSUM && 1962 cpl->l2info & htobe32(F_RXF_IP)) { 1963 m0->m_pkthdr.csum_flags = (CSUM_IP_CHECKED | 1964 CSUM_IP_VALID | CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 1965 rxq->rxcsum++; 1966 } else if (ifp->if_capenable & IFCAP_RXCSUM_IPV6 && 1967 cpl->l2info & htobe32(F_RXF_IP6)) { 1968 m0->m_pkthdr.csum_flags = (CSUM_DATA_VALID_IPV6 | 1969 CSUM_PSEUDO_HDR); 1970 rxq->rxcsum++; 1971 } 1972 1973 if (__predict_false(cpl->ip_frag)) 1974 m0->m_pkthdr.csum_data = be16toh(cpl->csum); 1975 else 1976 m0->m_pkthdr.csum_data = 0xffff; 1977 } 1978 1979 if (cpl->vlan_ex) { 1980 m0->m_pkthdr.ether_vtag = be16toh(cpl->vlan); 1981 m0->m_flags |= M_VLANTAG; 1982 rxq->vlan_extraction++; 1983 } 1984 1985 if (rxq->iq.flags & IQ_RX_TIMESTAMP) { 1986 /* 1987 * Fill up rcv_tstmp but do not set M_TSTMP. 1988 * rcv_tstmp is not in the format that the 1989 * kernel expects and we don't want to mislead 1990 * it. For now this is only for custom code 1991 * that knows how to interpret cxgbe's stamp. 1992 */ 1993 m0->m_pkthdr.rcv_tstmp = 1994 last_flit_to_ns(sc, d->rsp.u.last_flit); 1995 #ifdef notyet 1996 m0->m_flags |= M_TSTMP; 1997 #endif 1998 } 1999 2000 #ifdef NUMA 2001 m0->m_pkthdr.numa_domain = ifp->if_numa_domain; 2002 #endif 2003 #if defined(INET) || defined(INET6) 2004 if (rxq->iq.flags & IQ_LRO_ENABLED && 2005 (M_HASHTYPE_GET(m0) == M_HASHTYPE_RSS_TCP_IPV4 || 2006 M_HASHTYPE_GET(m0) == M_HASHTYPE_RSS_TCP_IPV6)) { 2007 if (sort_before_lro(lro)) { 2008 tcp_lro_queue_mbuf(lro, m0); 2009 return (0); /* queued for sort, then LRO */ 2010 } 2011 if (tcp_lro_rx(lro, m0, 0) == 0) 2012 return (0); /* queued for LRO */ 2013 } 2014 #endif 2015 ifp->if_input(ifp, m0); 2016 2017 return (0); 2018 } 2019 2020 /* 2021 * Must drain the wrq or make sure that someone else will. 2022 */ 2023 static void 2024 wrq_tx_drain(void *arg, int n) 2025 { 2026 struct sge_wrq *wrq = arg; 2027 struct sge_eq *eq = &wrq->eq; 2028 2029 EQ_LOCK(eq); 2030 if (TAILQ_EMPTY(&wrq->incomplete_wrs) && !STAILQ_EMPTY(&wrq->wr_list)) 2031 drain_wrq_wr_list(wrq->adapter, wrq); 2032 EQ_UNLOCK(eq); 2033 } 2034 2035 static void 2036 drain_wrq_wr_list(struct adapter *sc, struct sge_wrq *wrq) 2037 { 2038 struct sge_eq *eq = &wrq->eq; 2039 u_int available, dbdiff; /* # of hardware descriptors */ 2040 u_int n; 2041 struct wrqe *wr; 2042 struct fw_eth_tx_pkt_wr *dst; /* any fw WR struct will do */ 2043 2044 EQ_LOCK_ASSERT_OWNED(eq); 2045 MPASS(TAILQ_EMPTY(&wrq->incomplete_wrs)); 2046 wr = STAILQ_FIRST(&wrq->wr_list); 2047 MPASS(wr != NULL); /* Must be called with something useful to do */ 2048 MPASS(eq->pidx == eq->dbidx); 2049 dbdiff = 0; 2050 2051 do { 2052 eq->cidx = read_hw_cidx(eq); 2053 if (eq->pidx == eq->cidx) 2054 available = eq->sidx - 1; 2055 else 2056 available = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1; 2057 2058 MPASS(wr->wrq == wrq); 2059 n = howmany(wr->wr_len, EQ_ESIZE); 2060 if (available < n) 2061 break; 2062 2063 dst = (void *)&eq->desc[eq->pidx]; 2064 if (__predict_true(eq->sidx - eq->pidx > n)) { 2065 /* Won't wrap, won't end exactly at the status page. */ 2066 bcopy(&wr->wr[0], dst, wr->wr_len); 2067 eq->pidx += n; 2068 } else { 2069 int first_portion = (eq->sidx - eq->pidx) * EQ_ESIZE; 2070 2071 bcopy(&wr->wr[0], dst, first_portion); 2072 if (wr->wr_len > first_portion) { 2073 bcopy(&wr->wr[first_portion], &eq->desc[0], 2074 wr->wr_len - first_portion); 2075 } 2076 eq->pidx = n - (eq->sidx - eq->pidx); 2077 } 2078 wrq->tx_wrs_copied++; 2079 2080 if (available < eq->sidx / 4 && 2081 atomic_cmpset_int(&eq->equiq, 0, 1)) { 2082 /* 2083 * XXX: This is not 100% reliable with some 2084 * types of WRs. But this is a very unusual 2085 * situation for an ofld/ctrl queue anyway. 2086 */ 2087 dst->equiq_to_len16 |= htobe32(F_FW_WR_EQUIQ | 2088 F_FW_WR_EQUEQ); 2089 } 2090 2091 dbdiff += n; 2092 if (dbdiff >= 16) { 2093 ring_eq_db(sc, eq, dbdiff); 2094 dbdiff = 0; 2095 } 2096 2097 STAILQ_REMOVE_HEAD(&wrq->wr_list, link); 2098 free_wrqe(wr); 2099 MPASS(wrq->nwr_pending > 0); 2100 wrq->nwr_pending--; 2101 MPASS(wrq->ndesc_needed >= n); 2102 wrq->ndesc_needed -= n; 2103 } while ((wr = STAILQ_FIRST(&wrq->wr_list)) != NULL); 2104 2105 if (dbdiff) 2106 ring_eq_db(sc, eq, dbdiff); 2107 } 2108 2109 /* 2110 * Doesn't fail. Holds on to work requests it can't send right away. 2111 */ 2112 void 2113 t4_wrq_tx_locked(struct adapter *sc, struct sge_wrq *wrq, struct wrqe *wr) 2114 { 2115 #ifdef INVARIANTS 2116 struct sge_eq *eq = &wrq->eq; 2117 #endif 2118 2119 EQ_LOCK_ASSERT_OWNED(eq); 2120 MPASS(wr != NULL); 2121 MPASS(wr->wr_len > 0 && wr->wr_len <= SGE_MAX_WR_LEN); 2122 MPASS((wr->wr_len & 0x7) == 0); 2123 2124 STAILQ_INSERT_TAIL(&wrq->wr_list, wr, link); 2125 wrq->nwr_pending++; 2126 wrq->ndesc_needed += howmany(wr->wr_len, EQ_ESIZE); 2127 2128 if (!TAILQ_EMPTY(&wrq->incomplete_wrs)) 2129 return; /* commit_wrq_wr will drain wr_list as well. */ 2130 2131 drain_wrq_wr_list(sc, wrq); 2132 2133 /* Doorbell must have caught up to the pidx. */ 2134 MPASS(eq->pidx == eq->dbidx); 2135 } 2136 2137 void 2138 t4_update_fl_bufsize(struct ifnet *ifp) 2139 { 2140 struct vi_info *vi = ifp->if_softc; 2141 struct adapter *sc = vi->adapter; 2142 struct sge_rxq *rxq; 2143 #ifdef TCP_OFFLOAD 2144 struct sge_ofld_rxq *ofld_rxq; 2145 #endif 2146 struct sge_fl *fl; 2147 int i, maxp, mtu = ifp->if_mtu; 2148 2149 maxp = mtu_to_max_payload(sc, mtu); 2150 for_each_rxq(vi, i, rxq) { 2151 fl = &rxq->fl; 2152 2153 FL_LOCK(fl); 2154 fl->zidx = find_refill_source(sc, maxp, 2155 fl->flags & FL_BUF_PACKING); 2156 FL_UNLOCK(fl); 2157 } 2158 #ifdef TCP_OFFLOAD 2159 for_each_ofld_rxq(vi, i, ofld_rxq) { 2160 fl = &ofld_rxq->fl; 2161 2162 FL_LOCK(fl); 2163 fl->zidx = find_refill_source(sc, maxp, 2164 fl->flags & FL_BUF_PACKING); 2165 FL_UNLOCK(fl); 2166 } 2167 #endif 2168 } 2169 2170 static inline int 2171 mbuf_nsegs(struct mbuf *m) 2172 { 2173 2174 M_ASSERTPKTHDR(m); 2175 KASSERT(m->m_pkthdr.l5hlen > 0, 2176 ("%s: mbuf %p missing information on # of segments.", __func__, m)); 2177 2178 return (m->m_pkthdr.l5hlen); 2179 } 2180 2181 static inline void 2182 set_mbuf_nsegs(struct mbuf *m, uint8_t nsegs) 2183 { 2184 2185 M_ASSERTPKTHDR(m); 2186 m->m_pkthdr.l5hlen = nsegs; 2187 } 2188 2189 static inline int 2190 mbuf_cflags(struct mbuf *m) 2191 { 2192 2193 M_ASSERTPKTHDR(m); 2194 return (m->m_pkthdr.PH_loc.eight[4]); 2195 } 2196 2197 static inline void 2198 set_mbuf_cflags(struct mbuf *m, uint8_t flags) 2199 { 2200 2201 M_ASSERTPKTHDR(m); 2202 m->m_pkthdr.PH_loc.eight[4] = flags; 2203 } 2204 2205 static inline int 2206 mbuf_len16(struct mbuf *m) 2207 { 2208 int n; 2209 2210 M_ASSERTPKTHDR(m); 2211 n = m->m_pkthdr.PH_loc.eight[0]; 2212 if (!(mbuf_cflags(m) & MC_TLS)) 2213 MPASS(n > 0 && n <= SGE_MAX_WR_LEN / 16); 2214 2215 return (n); 2216 } 2217 2218 static inline void 2219 set_mbuf_len16(struct mbuf *m, uint8_t len16) 2220 { 2221 2222 M_ASSERTPKTHDR(m); 2223 m->m_pkthdr.PH_loc.eight[0] = len16; 2224 } 2225 2226 #ifdef RATELIMIT 2227 static inline int 2228 mbuf_eo_nsegs(struct mbuf *m) 2229 { 2230 2231 M_ASSERTPKTHDR(m); 2232 return (m->m_pkthdr.PH_loc.eight[1]); 2233 } 2234 2235 static inline void 2236 set_mbuf_eo_nsegs(struct mbuf *m, uint8_t nsegs) 2237 { 2238 2239 M_ASSERTPKTHDR(m); 2240 m->m_pkthdr.PH_loc.eight[1] = nsegs; 2241 } 2242 2243 static inline int 2244 mbuf_eo_len16(struct mbuf *m) 2245 { 2246 int n; 2247 2248 M_ASSERTPKTHDR(m); 2249 n = m->m_pkthdr.PH_loc.eight[2]; 2250 MPASS(n > 0 && n <= SGE_MAX_WR_LEN / 16); 2251 2252 return (n); 2253 } 2254 2255 static inline void 2256 set_mbuf_eo_len16(struct mbuf *m, uint8_t len16) 2257 { 2258 2259 M_ASSERTPKTHDR(m); 2260 m->m_pkthdr.PH_loc.eight[2] = len16; 2261 } 2262 2263 static inline int 2264 mbuf_eo_tsclk_tsoff(struct mbuf *m) 2265 { 2266 2267 M_ASSERTPKTHDR(m); 2268 return (m->m_pkthdr.PH_loc.eight[3]); 2269 } 2270 2271 static inline void 2272 set_mbuf_eo_tsclk_tsoff(struct mbuf *m, uint8_t tsclk_tsoff) 2273 { 2274 2275 M_ASSERTPKTHDR(m); 2276 m->m_pkthdr.PH_loc.eight[3] = tsclk_tsoff; 2277 } 2278 2279 static inline int 2280 needs_eo(struct cxgbe_snd_tag *cst) 2281 { 2282 2283 return (cst != NULL && cst->type == IF_SND_TAG_TYPE_RATE_LIMIT); 2284 } 2285 #endif 2286 2287 /* 2288 * Try to allocate an mbuf to contain a raw work request. To make it 2289 * easy to construct the work request, don't allocate a chain but a 2290 * single mbuf. 2291 */ 2292 struct mbuf * 2293 alloc_wr_mbuf(int len, int how) 2294 { 2295 struct mbuf *m; 2296 2297 if (len <= MHLEN) 2298 m = m_gethdr(how, MT_DATA); 2299 else if (len <= MCLBYTES) 2300 m = m_getcl(how, MT_DATA, M_PKTHDR); 2301 else 2302 m = NULL; 2303 if (m == NULL) 2304 return (NULL); 2305 m->m_pkthdr.len = len; 2306 m->m_len = len; 2307 set_mbuf_cflags(m, MC_RAW_WR); 2308 set_mbuf_len16(m, howmany(len, 16)); 2309 return (m); 2310 } 2311 2312 static inline int 2313 needs_hwcsum(struct mbuf *m) 2314 { 2315 2316 M_ASSERTPKTHDR(m); 2317 2318 return (m->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP | CSUM_IP | 2319 CSUM_TSO | CSUM_UDP_IPV6 | CSUM_TCP_IPV6)); 2320 } 2321 2322 static inline int 2323 needs_tso(struct mbuf *m) 2324 { 2325 2326 M_ASSERTPKTHDR(m); 2327 2328 return (m->m_pkthdr.csum_flags & CSUM_TSO); 2329 } 2330 2331 static inline int 2332 needs_l3_csum(struct mbuf *m) 2333 { 2334 2335 M_ASSERTPKTHDR(m); 2336 2337 return (m->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TSO)); 2338 } 2339 2340 static inline int 2341 needs_tcp_csum(struct mbuf *m) 2342 { 2343 2344 M_ASSERTPKTHDR(m); 2345 return (m->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_TCP_IPV6 | CSUM_TSO)); 2346 } 2347 2348 #ifdef RATELIMIT 2349 static inline int 2350 needs_l4_csum(struct mbuf *m) 2351 { 2352 2353 M_ASSERTPKTHDR(m); 2354 2355 return (m->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP | CSUM_UDP_IPV6 | 2356 CSUM_TCP_IPV6 | CSUM_TSO)); 2357 } 2358 2359 static inline int 2360 needs_udp_csum(struct mbuf *m) 2361 { 2362 2363 M_ASSERTPKTHDR(m); 2364 return (m->m_pkthdr.csum_flags & (CSUM_UDP | CSUM_UDP_IPV6)); 2365 } 2366 #endif 2367 2368 static inline int 2369 needs_vlan_insertion(struct mbuf *m) 2370 { 2371 2372 M_ASSERTPKTHDR(m); 2373 2374 return (m->m_flags & M_VLANTAG); 2375 } 2376 2377 static void * 2378 m_advance(struct mbuf **pm, int *poffset, int len) 2379 { 2380 struct mbuf *m = *pm; 2381 int offset = *poffset; 2382 uintptr_t p = 0; 2383 2384 MPASS(len > 0); 2385 2386 for (;;) { 2387 if (offset + len < m->m_len) { 2388 offset += len; 2389 p = mtod(m, uintptr_t) + offset; 2390 break; 2391 } 2392 len -= m->m_len - offset; 2393 m = m->m_next; 2394 offset = 0; 2395 MPASS(m != NULL); 2396 } 2397 *poffset = offset; 2398 *pm = m; 2399 return ((void *)p); 2400 } 2401 2402 static inline int 2403 count_mbuf_ext_pgs(struct mbuf *m, int skip, vm_paddr_t *nextaddr) 2404 { 2405 vm_paddr_t paddr; 2406 int i, len, off, pglen, pgoff, seglen, segoff; 2407 int nsegs = 0; 2408 2409 M_ASSERTEXTPG(m); 2410 off = mtod(m, vm_offset_t); 2411 len = m->m_len; 2412 off += skip; 2413 len -= skip; 2414 2415 if (m->m_epg_hdrlen != 0) { 2416 if (off >= m->m_epg_hdrlen) { 2417 off -= m->m_epg_hdrlen; 2418 } else { 2419 seglen = m->m_epg_hdrlen - off; 2420 segoff = off; 2421 seglen = min(seglen, len); 2422 off = 0; 2423 len -= seglen; 2424 paddr = pmap_kextract( 2425 (vm_offset_t)&m->m_epg_hdr[segoff]); 2426 if (*nextaddr != paddr) 2427 nsegs++; 2428 *nextaddr = paddr + seglen; 2429 } 2430 } 2431 pgoff = m->m_epg_1st_off; 2432 for (i = 0; i < m->m_epg_npgs && len > 0; i++) { 2433 pglen = m_epg_pagelen(m, i, pgoff); 2434 if (off >= pglen) { 2435 off -= pglen; 2436 pgoff = 0; 2437 continue; 2438 } 2439 seglen = pglen - off; 2440 segoff = pgoff + off; 2441 off = 0; 2442 seglen = min(seglen, len); 2443 len -= seglen; 2444 paddr = m->m_epg_pa[i] + segoff; 2445 if (*nextaddr != paddr) 2446 nsegs++; 2447 *nextaddr = paddr + seglen; 2448 pgoff = 0; 2449 }; 2450 if (len != 0) { 2451 seglen = min(len, m->m_epg_trllen - off); 2452 len -= seglen; 2453 paddr = pmap_kextract((vm_offset_t)&m->m_epg_trail[off]); 2454 if (*nextaddr != paddr) 2455 nsegs++; 2456 *nextaddr = paddr + seglen; 2457 } 2458 2459 return (nsegs); 2460 } 2461 2462 2463 /* 2464 * Can deal with empty mbufs in the chain that have m_len = 0, but the chain 2465 * must have at least one mbuf that's not empty. It is possible for this 2466 * routine to return 0 if skip accounts for all the contents of the mbuf chain. 2467 */ 2468 static inline int 2469 count_mbuf_nsegs(struct mbuf *m, int skip, uint8_t *cflags) 2470 { 2471 vm_paddr_t nextaddr, paddr; 2472 vm_offset_t va; 2473 int len, nsegs; 2474 2475 M_ASSERTPKTHDR(m); 2476 MPASS(m->m_pkthdr.len > 0); 2477 MPASS(m->m_pkthdr.len >= skip); 2478 2479 nsegs = 0; 2480 nextaddr = 0; 2481 for (; m; m = m->m_next) { 2482 len = m->m_len; 2483 if (__predict_false(len == 0)) 2484 continue; 2485 if (skip >= len) { 2486 skip -= len; 2487 continue; 2488 } 2489 if ((m->m_flags & M_EXTPG) != 0) { 2490 *cflags |= MC_NOMAP; 2491 nsegs += count_mbuf_ext_pgs(m, skip, &nextaddr); 2492 skip = 0; 2493 continue; 2494 } 2495 va = mtod(m, vm_offset_t) + skip; 2496 len -= skip; 2497 skip = 0; 2498 paddr = pmap_kextract(va); 2499 nsegs += sglist_count((void *)(uintptr_t)va, len); 2500 if (paddr == nextaddr) 2501 nsegs--; 2502 nextaddr = pmap_kextract(va + len - 1) + 1; 2503 } 2504 2505 return (nsegs); 2506 } 2507 2508 /* 2509 * Analyze the mbuf to determine its tx needs. The mbuf passed in may change: 2510 * a) caller can assume it's been freed if this function returns with an error. 2511 * b) it may get defragged up if the gather list is too long for the hardware. 2512 */ 2513 int 2514 parse_pkt(struct adapter *sc, struct mbuf **mp) 2515 { 2516 struct mbuf *m0 = *mp, *m; 2517 int rc, nsegs, defragged = 0, offset; 2518 struct ether_header *eh; 2519 void *l3hdr; 2520 #if defined(INET) || defined(INET6) 2521 struct tcphdr *tcp; 2522 #endif 2523 #if defined(KERN_TLS) || defined(RATELIMIT) 2524 struct cxgbe_snd_tag *cst; 2525 #endif 2526 uint16_t eh_type; 2527 uint8_t cflags; 2528 2529 cflags = 0; 2530 M_ASSERTPKTHDR(m0); 2531 if (__predict_false(m0->m_pkthdr.len < ETHER_HDR_LEN)) { 2532 rc = EINVAL; 2533 fail: 2534 m_freem(m0); 2535 *mp = NULL; 2536 return (rc); 2537 } 2538 restart: 2539 /* 2540 * First count the number of gather list segments in the payload. 2541 * Defrag the mbuf if nsegs exceeds the hardware limit. 2542 */ 2543 M_ASSERTPKTHDR(m0); 2544 MPASS(m0->m_pkthdr.len > 0); 2545 nsegs = count_mbuf_nsegs(m0, 0, &cflags); 2546 #if defined(KERN_TLS) || defined(RATELIMIT) 2547 if (m0->m_pkthdr.csum_flags & CSUM_SND_TAG) 2548 cst = mst_to_cst(m0->m_pkthdr.snd_tag); 2549 else 2550 cst = NULL; 2551 #endif 2552 #ifdef KERN_TLS 2553 if (cst != NULL && cst->type == IF_SND_TAG_TYPE_TLS) { 2554 int len16; 2555 2556 cflags |= MC_TLS; 2557 set_mbuf_cflags(m0, cflags); 2558 rc = t6_ktls_parse_pkt(m0, &nsegs, &len16); 2559 if (rc != 0) 2560 goto fail; 2561 set_mbuf_nsegs(m0, nsegs); 2562 set_mbuf_len16(m0, len16); 2563 return (0); 2564 } 2565 #endif 2566 if (nsegs > (needs_tso(m0) ? TX_SGL_SEGS_TSO : TX_SGL_SEGS)) { 2567 if (defragged++ > 0 || (m = m_defrag(m0, M_NOWAIT)) == NULL) { 2568 rc = EFBIG; 2569 goto fail; 2570 } 2571 *mp = m0 = m; /* update caller's copy after defrag */ 2572 goto restart; 2573 } 2574 2575 if (__predict_false(nsegs > 2 && m0->m_pkthdr.len <= MHLEN && 2576 !(cflags & MC_NOMAP))) { 2577 m0 = m_pullup(m0, m0->m_pkthdr.len); 2578 if (m0 == NULL) { 2579 /* Should have left well enough alone. */ 2580 rc = EFBIG; 2581 goto fail; 2582 } 2583 *mp = m0; /* update caller's copy after pullup */ 2584 goto restart; 2585 } 2586 set_mbuf_nsegs(m0, nsegs); 2587 set_mbuf_cflags(m0, cflags); 2588 if (sc->flags & IS_VF) 2589 set_mbuf_len16(m0, txpkt_vm_len16(nsegs, needs_tso(m0))); 2590 else 2591 set_mbuf_len16(m0, txpkt_len16(nsegs, needs_tso(m0))); 2592 2593 #ifdef RATELIMIT 2594 /* 2595 * Ethofld is limited to TCP and UDP for now, and only when L4 hw 2596 * checksumming is enabled. needs_l4_csum happens to check for all the 2597 * right things. 2598 */ 2599 if (__predict_false(needs_eo(cst) && !needs_l4_csum(m0))) { 2600 m_snd_tag_rele(m0->m_pkthdr.snd_tag); 2601 m0->m_pkthdr.snd_tag = NULL; 2602 m0->m_pkthdr.csum_flags &= ~CSUM_SND_TAG; 2603 cst = NULL; 2604 } 2605 #endif 2606 2607 if (!needs_hwcsum(m0) 2608 #ifdef RATELIMIT 2609 && !needs_eo(cst) 2610 #endif 2611 ) 2612 return (0); 2613 2614 m = m0; 2615 eh = mtod(m, struct ether_header *); 2616 eh_type = ntohs(eh->ether_type); 2617 if (eh_type == ETHERTYPE_VLAN) { 2618 struct ether_vlan_header *evh = (void *)eh; 2619 2620 eh_type = ntohs(evh->evl_proto); 2621 m0->m_pkthdr.l2hlen = sizeof(*evh); 2622 } else 2623 m0->m_pkthdr.l2hlen = sizeof(*eh); 2624 2625 offset = 0; 2626 l3hdr = m_advance(&m, &offset, m0->m_pkthdr.l2hlen); 2627 2628 switch (eh_type) { 2629 #ifdef INET6 2630 case ETHERTYPE_IPV6: 2631 { 2632 struct ip6_hdr *ip6 = l3hdr; 2633 2634 MPASS(!needs_tso(m0) || ip6->ip6_nxt == IPPROTO_TCP); 2635 2636 m0->m_pkthdr.l3hlen = sizeof(*ip6); 2637 break; 2638 } 2639 #endif 2640 #ifdef INET 2641 case ETHERTYPE_IP: 2642 { 2643 struct ip *ip = l3hdr; 2644 2645 m0->m_pkthdr.l3hlen = ip->ip_hl * 4; 2646 break; 2647 } 2648 #endif 2649 default: 2650 panic("%s: ethertype 0x%04x unknown. if_cxgbe must be compiled" 2651 " with the same INET/INET6 options as the kernel.", 2652 __func__, eh_type); 2653 } 2654 2655 #if defined(INET) || defined(INET6) 2656 if (needs_tcp_csum(m0)) { 2657 tcp = m_advance(&m, &offset, m0->m_pkthdr.l3hlen); 2658 m0->m_pkthdr.l4hlen = tcp->th_off * 4; 2659 #ifdef RATELIMIT 2660 if (tsclk >= 0 && *(uint32_t *)(tcp + 1) == ntohl(0x0101080a)) { 2661 set_mbuf_eo_tsclk_tsoff(m0, 2662 V_FW_ETH_TX_EO_WR_TSCLK(tsclk) | 2663 V_FW_ETH_TX_EO_WR_TSOFF(sizeof(*tcp) / 2 + 1)); 2664 } else 2665 set_mbuf_eo_tsclk_tsoff(m0, 0); 2666 } else if (needs_udp_csum(m0)) { 2667 m0->m_pkthdr.l4hlen = sizeof(struct udphdr); 2668 #endif 2669 } 2670 #ifdef RATELIMIT 2671 if (needs_eo(cst)) { 2672 u_int immhdrs; 2673 2674 /* EO WRs have the headers in the WR and not the GL. */ 2675 immhdrs = m0->m_pkthdr.l2hlen + m0->m_pkthdr.l3hlen + 2676 m0->m_pkthdr.l4hlen; 2677 cflags = 0; 2678 nsegs = count_mbuf_nsegs(m0, immhdrs, &cflags); 2679 MPASS(cflags == mbuf_cflags(m0)); 2680 set_mbuf_eo_nsegs(m0, nsegs); 2681 set_mbuf_eo_len16(m0, 2682 txpkt_eo_len16(nsegs, immhdrs, needs_tso(m0))); 2683 } 2684 #endif 2685 #endif 2686 MPASS(m0 == *mp); 2687 return (0); 2688 } 2689 2690 void * 2691 start_wrq_wr(struct sge_wrq *wrq, int len16, struct wrq_cookie *cookie) 2692 { 2693 struct sge_eq *eq = &wrq->eq; 2694 struct adapter *sc = wrq->adapter; 2695 int ndesc, available; 2696 struct wrqe *wr; 2697 void *w; 2698 2699 MPASS(len16 > 0); 2700 ndesc = tx_len16_to_desc(len16); 2701 MPASS(ndesc > 0 && ndesc <= SGE_MAX_WR_NDESC); 2702 2703 EQ_LOCK(eq); 2704 2705 if (TAILQ_EMPTY(&wrq->incomplete_wrs) && !STAILQ_EMPTY(&wrq->wr_list)) 2706 drain_wrq_wr_list(sc, wrq); 2707 2708 if (!STAILQ_EMPTY(&wrq->wr_list)) { 2709 slowpath: 2710 EQ_UNLOCK(eq); 2711 wr = alloc_wrqe(len16 * 16, wrq); 2712 if (__predict_false(wr == NULL)) 2713 return (NULL); 2714 cookie->pidx = -1; 2715 cookie->ndesc = ndesc; 2716 return (&wr->wr); 2717 } 2718 2719 eq->cidx = read_hw_cidx(eq); 2720 if (eq->pidx == eq->cidx) 2721 available = eq->sidx - 1; 2722 else 2723 available = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1; 2724 if (available < ndesc) 2725 goto slowpath; 2726 2727 cookie->pidx = eq->pidx; 2728 cookie->ndesc = ndesc; 2729 TAILQ_INSERT_TAIL(&wrq->incomplete_wrs, cookie, link); 2730 2731 w = &eq->desc[eq->pidx]; 2732 IDXINCR(eq->pidx, ndesc, eq->sidx); 2733 if (__predict_false(cookie->pidx + ndesc > eq->sidx)) { 2734 w = &wrq->ss[0]; 2735 wrq->ss_pidx = cookie->pidx; 2736 wrq->ss_len = len16 * 16; 2737 } 2738 2739 EQ_UNLOCK(eq); 2740 2741 return (w); 2742 } 2743 2744 void 2745 commit_wrq_wr(struct sge_wrq *wrq, void *w, struct wrq_cookie *cookie) 2746 { 2747 struct sge_eq *eq = &wrq->eq; 2748 struct adapter *sc = wrq->adapter; 2749 int ndesc, pidx; 2750 struct wrq_cookie *prev, *next; 2751 2752 if (cookie->pidx == -1) { 2753 struct wrqe *wr = __containerof(w, struct wrqe, wr); 2754 2755 t4_wrq_tx(sc, wr); 2756 return; 2757 } 2758 2759 if (__predict_false(w == &wrq->ss[0])) { 2760 int n = (eq->sidx - wrq->ss_pidx) * EQ_ESIZE; 2761 2762 MPASS(wrq->ss_len > n); /* WR had better wrap around. */ 2763 bcopy(&wrq->ss[0], &eq->desc[wrq->ss_pidx], n); 2764 bcopy(&wrq->ss[n], &eq->desc[0], wrq->ss_len - n); 2765 wrq->tx_wrs_ss++; 2766 } else 2767 wrq->tx_wrs_direct++; 2768 2769 EQ_LOCK(eq); 2770 ndesc = cookie->ndesc; /* Can be more than SGE_MAX_WR_NDESC here. */ 2771 pidx = cookie->pidx; 2772 MPASS(pidx >= 0 && pidx < eq->sidx); 2773 prev = TAILQ_PREV(cookie, wrq_incomplete_wrs, link); 2774 next = TAILQ_NEXT(cookie, link); 2775 if (prev == NULL) { 2776 MPASS(pidx == eq->dbidx); 2777 if (next == NULL || ndesc >= 16) { 2778 int available; 2779 struct fw_eth_tx_pkt_wr *dst; /* any fw WR struct will do */ 2780 2781 /* 2782 * Note that the WR via which we'll request tx updates 2783 * is at pidx and not eq->pidx, which has moved on 2784 * already. 2785 */ 2786 dst = (void *)&eq->desc[pidx]; 2787 available = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1; 2788 if (available < eq->sidx / 4 && 2789 atomic_cmpset_int(&eq->equiq, 0, 1)) { 2790 /* 2791 * XXX: This is not 100% reliable with some 2792 * types of WRs. But this is a very unusual 2793 * situation for an ofld/ctrl queue anyway. 2794 */ 2795 dst->equiq_to_len16 |= htobe32(F_FW_WR_EQUIQ | 2796 F_FW_WR_EQUEQ); 2797 } 2798 2799 ring_eq_db(wrq->adapter, eq, ndesc); 2800 } else { 2801 MPASS(IDXDIFF(next->pidx, pidx, eq->sidx) == ndesc); 2802 next->pidx = pidx; 2803 next->ndesc += ndesc; 2804 } 2805 } else { 2806 MPASS(IDXDIFF(pidx, prev->pidx, eq->sidx) == prev->ndesc); 2807 prev->ndesc += ndesc; 2808 } 2809 TAILQ_REMOVE(&wrq->incomplete_wrs, cookie, link); 2810 2811 if (TAILQ_EMPTY(&wrq->incomplete_wrs) && !STAILQ_EMPTY(&wrq->wr_list)) 2812 drain_wrq_wr_list(sc, wrq); 2813 2814 #ifdef INVARIANTS 2815 if (TAILQ_EMPTY(&wrq->incomplete_wrs)) { 2816 /* Doorbell must have caught up to the pidx. */ 2817 MPASS(wrq->eq.pidx == wrq->eq.dbidx); 2818 } 2819 #endif 2820 EQ_UNLOCK(eq); 2821 } 2822 2823 static u_int 2824 can_resume_eth_tx(struct mp_ring *r) 2825 { 2826 struct sge_eq *eq = r->cookie; 2827 2828 return (total_available_tx_desc(eq) > eq->sidx / 8); 2829 } 2830 2831 static inline bool 2832 cannot_use_txpkts(struct mbuf *m) 2833 { 2834 /* maybe put a GL limit too, to avoid silliness? */ 2835 2836 return (needs_tso(m) || (mbuf_cflags(m) & (MC_RAW_WR | MC_TLS)) != 0); 2837 } 2838 2839 static inline int 2840 discard_tx(struct sge_eq *eq) 2841 { 2842 2843 return ((eq->flags & (EQ_ENABLED | EQ_QFLUSH)) != EQ_ENABLED); 2844 } 2845 2846 static inline int 2847 wr_can_update_eq(void *p) 2848 { 2849 struct fw_eth_tx_pkts_wr *wr = p; 2850 2851 switch (G_FW_WR_OP(be32toh(wr->op_pkd))) { 2852 case FW_ULPTX_WR: 2853 case FW_ETH_TX_PKT_WR: 2854 case FW_ETH_TX_PKTS_WR: 2855 case FW_ETH_TX_PKTS2_WR: 2856 case FW_ETH_TX_PKT_VM_WR: 2857 case FW_ETH_TX_PKTS_VM_WR: 2858 return (1); 2859 default: 2860 return (0); 2861 } 2862 } 2863 2864 static inline void 2865 set_txupdate_flags(struct sge_txq *txq, u_int avail, 2866 struct fw_eth_tx_pkt_wr *wr) 2867 { 2868 struct sge_eq *eq = &txq->eq; 2869 struct txpkts *txp = &txq->txp; 2870 2871 if ((txp->npkt > 0 || avail < eq->sidx / 2) && 2872 atomic_cmpset_int(&eq->equiq, 0, 1)) { 2873 wr->equiq_to_len16 |= htobe32(F_FW_WR_EQUEQ | F_FW_WR_EQUIQ); 2874 eq->equeqidx = eq->pidx; 2875 } else if (IDXDIFF(eq->pidx, eq->equeqidx, eq->sidx) >= 32) { 2876 wr->equiq_to_len16 |= htobe32(F_FW_WR_EQUEQ); 2877 eq->equeqidx = eq->pidx; 2878 } 2879 } 2880 2881 /* 2882 * r->items[cidx] to r->items[pidx], with a wraparound at r->size, are ready to 2883 * be consumed. Return the actual number consumed. 0 indicates a stall. 2884 */ 2885 static u_int 2886 eth_tx(struct mp_ring *r, u_int cidx, u_int pidx, bool *coalescing) 2887 { 2888 struct sge_txq *txq = r->cookie; 2889 struct ifnet *ifp = txq->ifp; 2890 struct sge_eq *eq = &txq->eq; 2891 struct txpkts *txp = &txq->txp; 2892 struct vi_info *vi = ifp->if_softc; 2893 struct adapter *sc = vi->adapter; 2894 u_int total, remaining; /* # of packets */ 2895 u_int n, avail, dbdiff; /* # of hardware descriptors */ 2896 int i, rc; 2897 struct mbuf *m0; 2898 bool snd; 2899 void *wr; /* start of the last WR written to the ring */ 2900 2901 TXQ_LOCK_ASSERT_OWNED(txq); 2902 2903 remaining = IDXDIFF(pidx, cidx, r->size); 2904 if (__predict_false(discard_tx(eq))) { 2905 for (i = 0; i < txp->npkt; i++) 2906 m_freem(txp->mb[i]); 2907 txp->npkt = 0; 2908 while (cidx != pidx) { 2909 m0 = r->items[cidx]; 2910 m_freem(m0); 2911 if (++cidx == r->size) 2912 cidx = 0; 2913 } 2914 reclaim_tx_descs(txq, eq->sidx); 2915 *coalescing = false; 2916 return (remaining); /* emptied */ 2917 } 2918 2919 /* How many hardware descriptors do we have readily available. */ 2920 if (eq->pidx == eq->cidx) { 2921 avail = eq->sidx - 1; 2922 if (txp->score++ >= 5) 2923 txp->score = 5; /* tx is completely idle, reset. */ 2924 } else 2925 avail = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1; 2926 2927 total = 0; 2928 if (remaining == 0) { 2929 if (txp->score-- == 1) /* egr_update had to drain txpkts */ 2930 txp->score = 1; 2931 goto send_txpkts; 2932 } 2933 2934 dbdiff = 0; 2935 MPASS(remaining > 0); 2936 while (remaining > 0) { 2937 m0 = r->items[cidx]; 2938 M_ASSERTPKTHDR(m0); 2939 MPASS(m0->m_nextpkt == NULL); 2940 2941 if (avail < 2 * SGE_MAX_WR_NDESC) 2942 avail += reclaim_tx_descs(txq, 64); 2943 2944 if (txp->npkt > 0 || remaining > 1 || txp->score > 3 || 2945 atomic_load_int(&txq->eq.equiq) != 0) { 2946 if (sc->flags & IS_VF) 2947 rc = add_to_txpkts_vf(sc, txq, m0, avail, &snd); 2948 else 2949 rc = add_to_txpkts_pf(sc, txq, m0, avail, &snd); 2950 } else { 2951 snd = false; 2952 rc = EINVAL; 2953 } 2954 if (snd) { 2955 MPASS(txp->npkt > 0); 2956 for (i = 0; i < txp->npkt; i++) 2957 ETHER_BPF_MTAP(ifp, txp->mb[i]); 2958 if (txp->npkt > 1) { 2959 if (txp->score++ >= 10) 2960 txp->score = 10; 2961 MPASS(avail >= tx_len16_to_desc(txp->len16)); 2962 if (sc->flags & IS_VF) 2963 n = write_txpkts_vm_wr(sc, txq); 2964 else 2965 n = write_txpkts_wr(sc, txq); 2966 } else { 2967 MPASS(avail >= 2968 tx_len16_to_desc(mbuf_len16(txp->mb[0]))); 2969 if (sc->flags & IS_VF) 2970 n = write_txpkt_vm_wr(sc, txq, 2971 txp->mb[0]); 2972 else 2973 n = write_txpkt_wr(sc, txq, txp->mb[0], 2974 avail); 2975 } 2976 MPASS(n <= SGE_MAX_WR_NDESC); 2977 avail -= n; 2978 dbdiff += n; 2979 wr = &eq->desc[eq->pidx]; 2980 IDXINCR(eq->pidx, n, eq->sidx); 2981 txp->npkt = 0; /* emptied */ 2982 } 2983 if (rc == 0) { 2984 /* m0 was coalesced into txq->txpkts. */ 2985 goto next_mbuf; 2986 } 2987 if (rc == EAGAIN) { 2988 /* 2989 * m0 is suitable for tx coalescing but could not be 2990 * combined with the existing txq->txpkts, which has now 2991 * been transmitted. Start a new txpkts with m0. 2992 */ 2993 MPASS(snd); 2994 MPASS(txp->npkt == 0); 2995 continue; 2996 } 2997 2998 MPASS(rc != 0 && rc != EAGAIN); 2999 MPASS(txp->npkt == 0); 3000 wr = &eq->desc[eq->pidx]; 3001 if (mbuf_cflags(m0) & MC_RAW_WR) { 3002 n = write_raw_wr(txq, wr, m0, avail); 3003 #ifdef KERN_TLS 3004 } else if (mbuf_cflags(m0) & MC_TLS) { 3005 ETHER_BPF_MTAP(ifp, m0); 3006 n = t6_ktls_write_wr(txq, wr, m0, mbuf_nsegs(m0), 3007 avail); 3008 #endif 3009 } else { 3010 n = tx_len16_to_desc(mbuf_len16(m0)); 3011 if (__predict_false(avail < n)) { 3012 avail += reclaim_tx_descs(txq, 32); 3013 if (avail < n) 3014 break; /* out of descriptors */ 3015 } 3016 ETHER_BPF_MTAP(ifp, m0); 3017 if (sc->flags & IS_VF) 3018 n = write_txpkt_vm_wr(sc, txq, m0); 3019 else 3020 n = write_txpkt_wr(sc, txq, m0, avail); 3021 } 3022 MPASS(n >= 1 && n <= avail); 3023 if (!(mbuf_cflags(m0) & MC_TLS)) 3024 MPASS(n <= SGE_MAX_WR_NDESC); 3025 3026 avail -= n; 3027 dbdiff += n; 3028 IDXINCR(eq->pidx, n, eq->sidx); 3029 3030 if (dbdiff >= 512 / EQ_ESIZE) { /* X_FETCHBURSTMAX_512B */ 3031 if (wr_can_update_eq(wr)) 3032 set_txupdate_flags(txq, avail, wr); 3033 ring_eq_db(sc, eq, dbdiff); 3034 avail += reclaim_tx_descs(txq, 32); 3035 dbdiff = 0; 3036 } 3037 next_mbuf: 3038 total++; 3039 remaining--; 3040 if (__predict_false(++cidx == r->size)) 3041 cidx = 0; 3042 } 3043 if (dbdiff != 0) { 3044 if (wr_can_update_eq(wr)) 3045 set_txupdate_flags(txq, avail, wr); 3046 ring_eq_db(sc, eq, dbdiff); 3047 reclaim_tx_descs(txq, 32); 3048 } else if (eq->pidx == eq->cidx && txp->npkt > 0 && 3049 atomic_load_int(&txq->eq.equiq) == 0) { 3050 /* 3051 * If nothing was submitted to the chip for tx (it was coalesced 3052 * into txpkts instead) and there is no tx update outstanding 3053 * then we need to send txpkts now. 3054 */ 3055 send_txpkts: 3056 MPASS(txp->npkt > 0); 3057 for (i = 0; i < txp->npkt; i++) 3058 ETHER_BPF_MTAP(ifp, txp->mb[i]); 3059 if (txp->npkt > 1) { 3060 MPASS(avail >= tx_len16_to_desc(txp->len16)); 3061 if (sc->flags & IS_VF) 3062 n = write_txpkts_vm_wr(sc, txq); 3063 else 3064 n = write_txpkts_wr(sc, txq); 3065 } else { 3066 MPASS(avail >= 3067 tx_len16_to_desc(mbuf_len16(txp->mb[0]))); 3068 if (sc->flags & IS_VF) 3069 n = write_txpkt_vm_wr(sc, txq, txp->mb[0]); 3070 else 3071 n = write_txpkt_wr(sc, txq, txp->mb[0], avail); 3072 } 3073 MPASS(n <= SGE_MAX_WR_NDESC); 3074 wr = &eq->desc[eq->pidx]; 3075 IDXINCR(eq->pidx, n, eq->sidx); 3076 txp->npkt = 0; /* emptied */ 3077 3078 MPASS(wr_can_update_eq(wr)); 3079 set_txupdate_flags(txq, avail - n, wr); 3080 ring_eq_db(sc, eq, n); 3081 reclaim_tx_descs(txq, 32); 3082 } 3083 *coalescing = txp->npkt > 0; 3084 3085 return (total); 3086 } 3087 3088 static inline void 3089 init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, int pktc_idx, 3090 int qsize) 3091 { 3092 3093 KASSERT(tmr_idx >= 0 && tmr_idx < SGE_NTIMERS, 3094 ("%s: bad tmr_idx %d", __func__, tmr_idx)); 3095 KASSERT(pktc_idx < SGE_NCOUNTERS, /* -ve is ok, means don't use */ 3096 ("%s: bad pktc_idx %d", __func__, pktc_idx)); 3097 3098 iq->flags = 0; 3099 iq->adapter = sc; 3100 iq->intr_params = V_QINTR_TIMER_IDX(tmr_idx); 3101 iq->intr_pktc_idx = SGE_NCOUNTERS - 1; 3102 if (pktc_idx >= 0) { 3103 iq->intr_params |= F_QINTR_CNT_EN; 3104 iq->intr_pktc_idx = pktc_idx; 3105 } 3106 iq->qsize = roundup2(qsize, 16); /* See FW_IQ_CMD/iqsize */ 3107 iq->sidx = iq->qsize - sc->params.sge.spg_len / IQ_ESIZE; 3108 } 3109 3110 static inline void 3111 init_fl(struct adapter *sc, struct sge_fl *fl, int qsize, int maxp, char *name) 3112 { 3113 3114 fl->qsize = qsize; 3115 fl->sidx = qsize - sc->params.sge.spg_len / EQ_ESIZE; 3116 strlcpy(fl->lockname, name, sizeof(fl->lockname)); 3117 if (sc->flags & BUF_PACKING_OK && 3118 ((!is_t4(sc) && buffer_packing) || /* T5+: enabled unless 0 */ 3119 (is_t4(sc) && buffer_packing == 1)))/* T4: disabled unless 1 */ 3120 fl->flags |= FL_BUF_PACKING; 3121 fl->zidx = find_refill_source(sc, maxp, fl->flags & FL_BUF_PACKING); 3122 fl->safe_zidx = sc->sge.safe_zidx; 3123 } 3124 3125 static inline void 3126 init_eq(struct adapter *sc, struct sge_eq *eq, int eqtype, int qsize, 3127 uint8_t tx_chan, uint16_t iqid, char *name) 3128 { 3129 KASSERT(eqtype <= EQ_TYPEMASK, ("%s: bad qtype %d", __func__, eqtype)); 3130 3131 eq->flags = eqtype & EQ_TYPEMASK; 3132 eq->tx_chan = tx_chan; 3133 eq->iqid = iqid; 3134 eq->sidx = qsize - sc->params.sge.spg_len / EQ_ESIZE; 3135 strlcpy(eq->lockname, name, sizeof(eq->lockname)); 3136 } 3137 3138 static int 3139 alloc_ring(struct adapter *sc, size_t len, bus_dma_tag_t *tag, 3140 bus_dmamap_t *map, bus_addr_t *pa, void **va) 3141 { 3142 int rc; 3143 3144 rc = bus_dma_tag_create(sc->dmat, 512, 0, BUS_SPACE_MAXADDR, 3145 BUS_SPACE_MAXADDR, NULL, NULL, len, 1, len, 0, NULL, NULL, tag); 3146 if (rc != 0) { 3147 device_printf(sc->dev, "cannot allocate DMA tag: %d\n", rc); 3148 goto done; 3149 } 3150 3151 rc = bus_dmamem_alloc(*tag, va, 3152 BUS_DMA_WAITOK | BUS_DMA_COHERENT | BUS_DMA_ZERO, map); 3153 if (rc != 0) { 3154 device_printf(sc->dev, "cannot allocate DMA memory: %d\n", rc); 3155 goto done; 3156 } 3157 3158 rc = bus_dmamap_load(*tag, *map, *va, len, oneseg_dma_callback, pa, 0); 3159 if (rc != 0) { 3160 device_printf(sc->dev, "cannot load DMA map: %d\n", rc); 3161 goto done; 3162 } 3163 done: 3164 if (rc) 3165 free_ring(sc, *tag, *map, *pa, *va); 3166 3167 return (rc); 3168 } 3169 3170 static int 3171 free_ring(struct adapter *sc, bus_dma_tag_t tag, bus_dmamap_t map, 3172 bus_addr_t pa, void *va) 3173 { 3174 if (pa) 3175 bus_dmamap_unload(tag, map); 3176 if (va) 3177 bus_dmamem_free(tag, va, map); 3178 if (tag) 3179 bus_dma_tag_destroy(tag); 3180 3181 return (0); 3182 } 3183 3184 /* 3185 * Allocates the ring for an ingress queue and an optional freelist. If the 3186 * freelist is specified it will be allocated and then associated with the 3187 * ingress queue. 3188 * 3189 * Returns errno on failure. Resources allocated up to that point may still be 3190 * allocated. Caller is responsible for cleanup in case this function fails. 3191 * 3192 * If the ingress queue will take interrupts directly then the intr_idx 3193 * specifies the vector, starting from 0. -1 means the interrupts for this 3194 * queue should be forwarded to the fwq. 3195 */ 3196 static int 3197 alloc_iq_fl(struct vi_info *vi, struct sge_iq *iq, struct sge_fl *fl, 3198 int intr_idx, int cong) 3199 { 3200 int rc, i, cntxt_id; 3201 size_t len; 3202 struct fw_iq_cmd c; 3203 struct port_info *pi = vi->pi; 3204 struct adapter *sc = iq->adapter; 3205 struct sge_params *sp = &sc->params.sge; 3206 __be32 v = 0; 3207 3208 len = iq->qsize * IQ_ESIZE; 3209 rc = alloc_ring(sc, len, &iq->desc_tag, &iq->desc_map, &iq->ba, 3210 (void **)&iq->desc); 3211 if (rc != 0) 3212 return (rc); 3213 3214 bzero(&c, sizeof(c)); 3215 c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST | 3216 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(sc->pf) | 3217 V_FW_IQ_CMD_VFN(0)); 3218 3219 c.alloc_to_len16 = htobe32(F_FW_IQ_CMD_ALLOC | F_FW_IQ_CMD_IQSTART | 3220 FW_LEN16(c)); 3221 3222 /* Special handling for firmware event queue */ 3223 if (iq == &sc->sge.fwq) 3224 v |= F_FW_IQ_CMD_IQASYNCH; 3225 3226 if (intr_idx < 0) { 3227 /* Forwarded interrupts, all headed to fwq */ 3228 v |= F_FW_IQ_CMD_IQANDST; 3229 v |= V_FW_IQ_CMD_IQANDSTINDEX(sc->sge.fwq.cntxt_id); 3230 } else { 3231 KASSERT(intr_idx < sc->intr_count, 3232 ("%s: invalid direct intr_idx %d", __func__, intr_idx)); 3233 v |= V_FW_IQ_CMD_IQANDSTINDEX(intr_idx); 3234 } 3235 3236 c.type_to_iqandstindex = htobe32(v | 3237 V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) | 3238 V_FW_IQ_CMD_VIID(vi->viid) | 3239 V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT)); 3240 c.iqdroprss_to_iqesize = htobe16(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) | 3241 F_FW_IQ_CMD_IQGTSMODE | 3242 V_FW_IQ_CMD_IQINTCNTTHRESH(iq->intr_pktc_idx) | 3243 V_FW_IQ_CMD_IQESIZE(ilog2(IQ_ESIZE) - 4)); 3244 c.iqsize = htobe16(iq->qsize); 3245 c.iqaddr = htobe64(iq->ba); 3246 if (cong >= 0) 3247 c.iqns_to_fl0congen = htobe32(F_FW_IQ_CMD_IQFLINTCONGEN); 3248 3249 if (fl) { 3250 mtx_init(&fl->fl_lock, fl->lockname, NULL, MTX_DEF); 3251 3252 len = fl->qsize * EQ_ESIZE; 3253 rc = alloc_ring(sc, len, &fl->desc_tag, &fl->desc_map, 3254 &fl->ba, (void **)&fl->desc); 3255 if (rc) 3256 return (rc); 3257 3258 /* Allocate space for one software descriptor per buffer. */ 3259 rc = alloc_fl_sdesc(fl); 3260 if (rc != 0) { 3261 device_printf(sc->dev, 3262 "failed to setup fl software descriptors: %d\n", 3263 rc); 3264 return (rc); 3265 } 3266 3267 if (fl->flags & FL_BUF_PACKING) { 3268 fl->lowat = roundup2(sp->fl_starve_threshold2, 8); 3269 fl->buf_boundary = sp->pack_boundary; 3270 } else { 3271 fl->lowat = roundup2(sp->fl_starve_threshold, 8); 3272 fl->buf_boundary = 16; 3273 } 3274 if (fl_pad && fl->buf_boundary < sp->pad_boundary) 3275 fl->buf_boundary = sp->pad_boundary; 3276 3277 c.iqns_to_fl0congen |= 3278 htobe32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) | 3279 F_FW_IQ_CMD_FL0FETCHRO | F_FW_IQ_CMD_FL0DATARO | 3280 (fl_pad ? F_FW_IQ_CMD_FL0PADEN : 0) | 3281 (fl->flags & FL_BUF_PACKING ? F_FW_IQ_CMD_FL0PACKEN : 3282 0)); 3283 if (cong >= 0) { 3284 c.iqns_to_fl0congen |= 3285 htobe32(V_FW_IQ_CMD_FL0CNGCHMAP(cong) | 3286 F_FW_IQ_CMD_FL0CONGCIF | 3287 F_FW_IQ_CMD_FL0CONGEN); 3288 } 3289 c.fl0dcaen_to_fl0cidxfthresh = 3290 htobe16(V_FW_IQ_CMD_FL0FBMIN(chip_id(sc) <= CHELSIO_T5 ? 3291 X_FETCHBURSTMIN_128B : X_FETCHBURSTMIN_64B_T6) | 3292 V_FW_IQ_CMD_FL0FBMAX(chip_id(sc) <= CHELSIO_T5 ? 3293 X_FETCHBURSTMAX_512B : X_FETCHBURSTMAX_256B)); 3294 c.fl0size = htobe16(fl->qsize); 3295 c.fl0addr = htobe64(fl->ba); 3296 } 3297 3298 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c); 3299 if (rc != 0) { 3300 device_printf(sc->dev, 3301 "failed to create ingress queue: %d\n", rc); 3302 return (rc); 3303 } 3304 3305 iq->cidx = 0; 3306 iq->gen = F_RSPD_GEN; 3307 iq->intr_next = iq->intr_params; 3308 iq->cntxt_id = be16toh(c.iqid); 3309 iq->abs_id = be16toh(c.physiqid); 3310 iq->flags |= IQ_ALLOCATED; 3311 3312 cntxt_id = iq->cntxt_id - sc->sge.iq_start; 3313 if (cntxt_id >= sc->sge.niq) { 3314 panic ("%s: iq->cntxt_id (%d) more than the max (%d)", __func__, 3315 cntxt_id, sc->sge.niq - 1); 3316 } 3317 sc->sge.iqmap[cntxt_id] = iq; 3318 3319 if (fl) { 3320 u_int qid; 3321 3322 iq->flags |= IQ_HAS_FL; 3323 fl->cntxt_id = be16toh(c.fl0id); 3324 fl->pidx = fl->cidx = 0; 3325 3326 cntxt_id = fl->cntxt_id - sc->sge.eq_start; 3327 if (cntxt_id >= sc->sge.neq) { 3328 panic("%s: fl->cntxt_id (%d) more than the max (%d)", 3329 __func__, cntxt_id, sc->sge.neq - 1); 3330 } 3331 sc->sge.eqmap[cntxt_id] = (void *)fl; 3332 3333 qid = fl->cntxt_id; 3334 if (isset(&sc->doorbells, DOORBELL_UDB)) { 3335 uint32_t s_qpp = sc->params.sge.eq_s_qpp; 3336 uint32_t mask = (1 << s_qpp) - 1; 3337 volatile uint8_t *udb; 3338 3339 udb = sc->udbs_base + UDBS_DB_OFFSET; 3340 udb += (qid >> s_qpp) << PAGE_SHIFT; 3341 qid &= mask; 3342 if (qid < PAGE_SIZE / UDBS_SEG_SIZE) { 3343 udb += qid << UDBS_SEG_SHIFT; 3344 qid = 0; 3345 } 3346 fl->udb = (volatile void *)udb; 3347 } 3348 fl->dbval = V_QID(qid) | sc->chip_params->sge_fl_db; 3349 3350 FL_LOCK(fl); 3351 /* Enough to make sure the SGE doesn't think it's starved */ 3352 refill_fl(sc, fl, fl->lowat); 3353 FL_UNLOCK(fl); 3354 } 3355 3356 if (chip_id(sc) >= CHELSIO_T5 && !(sc->flags & IS_VF) && cong >= 0) { 3357 uint32_t param, val; 3358 3359 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 3360 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) | 3361 V_FW_PARAMS_PARAM_YZ(iq->cntxt_id); 3362 if (cong == 0) 3363 val = 1 << 19; 3364 else { 3365 val = 2 << 19; 3366 for (i = 0; i < 4; i++) { 3367 if (cong & (1 << i)) 3368 val |= 1 << (i << 2); 3369 } 3370 } 3371 3372 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 3373 if (rc != 0) { 3374 /* report error but carry on */ 3375 device_printf(sc->dev, 3376 "failed to set congestion manager context for " 3377 "ingress queue %d: %d\n", iq->cntxt_id, rc); 3378 } 3379 } 3380 3381 /* Enable IQ interrupts */ 3382 atomic_store_rel_int(&iq->state, IQS_IDLE); 3383 t4_write_reg(sc, sc->sge_gts_reg, V_SEINTARM(iq->intr_params) | 3384 V_INGRESSQID(iq->cntxt_id)); 3385 3386 return (0); 3387 } 3388 3389 static int 3390 free_iq_fl(struct vi_info *vi, struct sge_iq *iq, struct sge_fl *fl) 3391 { 3392 int rc; 3393 struct adapter *sc = iq->adapter; 3394 device_t dev; 3395 3396 if (sc == NULL) 3397 return (0); /* nothing to do */ 3398 3399 dev = vi ? vi->dev : sc->dev; 3400 3401 if (iq->flags & IQ_ALLOCATED) { 3402 rc = -t4_iq_free(sc, sc->mbox, sc->pf, 0, 3403 FW_IQ_TYPE_FL_INT_CAP, iq->cntxt_id, 3404 fl ? fl->cntxt_id : 0xffff, 0xffff); 3405 if (rc != 0) { 3406 device_printf(dev, 3407 "failed to free queue %p: %d\n", iq, rc); 3408 return (rc); 3409 } 3410 iq->flags &= ~IQ_ALLOCATED; 3411 } 3412 3413 free_ring(sc, iq->desc_tag, iq->desc_map, iq->ba, iq->desc); 3414 3415 bzero(iq, sizeof(*iq)); 3416 3417 if (fl) { 3418 free_ring(sc, fl->desc_tag, fl->desc_map, fl->ba, 3419 fl->desc); 3420 3421 if (fl->sdesc) 3422 free_fl_sdesc(sc, fl); 3423 3424 if (mtx_initialized(&fl->fl_lock)) 3425 mtx_destroy(&fl->fl_lock); 3426 3427 bzero(fl, sizeof(*fl)); 3428 } 3429 3430 return (0); 3431 } 3432 3433 static void 3434 add_iq_sysctls(struct sysctl_ctx_list *ctx, struct sysctl_oid *oid, 3435 struct sge_iq *iq) 3436 { 3437 struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); 3438 3439 SYSCTL_ADD_UAUTO(ctx, children, OID_AUTO, "ba", CTLFLAG_RD, &iq->ba, 3440 "bus address of descriptor ring"); 3441 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "dmalen", CTLFLAG_RD, NULL, 3442 iq->qsize * IQ_ESIZE, "descriptor ring size in bytes"); 3443 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "abs_id", 3444 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, &iq->abs_id, 0, 3445 sysctl_uint16, "I", "absolute id of the queue"); 3446 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cntxt_id", 3447 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, &iq->cntxt_id, 0, 3448 sysctl_uint16, "I", "SGE context id of the queue"); 3449 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cidx", 3450 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, &iq->cidx, 0, 3451 sysctl_uint16, "I", "consumer index"); 3452 } 3453 3454 static void 3455 add_fl_sysctls(struct adapter *sc, struct sysctl_ctx_list *ctx, 3456 struct sysctl_oid *oid, struct sge_fl *fl) 3457 { 3458 struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); 3459 3460 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "fl", 3461 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "freelist"); 3462 children = SYSCTL_CHILDREN(oid); 3463 3464 SYSCTL_ADD_UAUTO(ctx, children, OID_AUTO, "ba", CTLFLAG_RD, 3465 &fl->ba, "bus address of descriptor ring"); 3466 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "dmalen", CTLFLAG_RD, NULL, 3467 fl->sidx * EQ_ESIZE + sc->params.sge.spg_len, 3468 "desc ring size in bytes"); 3469 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cntxt_id", 3470 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, &fl->cntxt_id, 0, 3471 sysctl_uint16, "I", "SGE context id of the freelist"); 3472 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "padding", CTLFLAG_RD, NULL, 3473 fl_pad ? 1 : 0, "padding enabled"); 3474 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "packing", CTLFLAG_RD, NULL, 3475 fl->flags & FL_BUF_PACKING ? 1 : 0, "packing enabled"); 3476 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cidx", CTLFLAG_RD, &fl->cidx, 3477 0, "consumer index"); 3478 if (fl->flags & FL_BUF_PACKING) { 3479 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "rx_offset", 3480 CTLFLAG_RD, &fl->rx_offset, 0, "packing rx offset"); 3481 } 3482 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "pidx", CTLFLAG_RD, &fl->pidx, 3483 0, "producer index"); 3484 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "cluster_allocated", 3485 CTLFLAG_RD, &fl->cl_allocated, "# of clusters allocated"); 3486 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "cluster_recycled", 3487 CTLFLAG_RD, &fl->cl_recycled, "# of clusters recycled"); 3488 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "cluster_fast_recycled", 3489 CTLFLAG_RD, &fl->cl_fast_recycled, "# of clusters recycled (fast)"); 3490 } 3491 3492 static int 3493 alloc_fwq(struct adapter *sc) 3494 { 3495 int rc, intr_idx; 3496 struct sge_iq *fwq = &sc->sge.fwq; 3497 struct sysctl_oid *oid = device_get_sysctl_tree(sc->dev); 3498 struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); 3499 3500 init_iq(fwq, sc, 0, 0, FW_IQ_QSIZE); 3501 if (sc->flags & IS_VF) 3502 intr_idx = 0; 3503 else 3504 intr_idx = sc->intr_count > 1 ? 1 : 0; 3505 rc = alloc_iq_fl(&sc->port[0]->vi[0], fwq, NULL, intr_idx, -1); 3506 if (rc != 0) { 3507 device_printf(sc->dev, 3508 "failed to create firmware event queue: %d\n", rc); 3509 return (rc); 3510 } 3511 3512 oid = SYSCTL_ADD_NODE(&sc->ctx, children, OID_AUTO, "fwq", 3513 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "firmware event queue"); 3514 add_iq_sysctls(&sc->ctx, oid, fwq); 3515 3516 return (0); 3517 } 3518 3519 static int 3520 free_fwq(struct adapter *sc) 3521 { 3522 return free_iq_fl(NULL, &sc->sge.fwq, NULL); 3523 } 3524 3525 static int 3526 alloc_ctrlq(struct adapter *sc, struct sge_wrq *ctrlq, int idx, 3527 struct sysctl_oid *oid) 3528 { 3529 int rc; 3530 char name[16]; 3531 struct sysctl_oid_list *children; 3532 3533 snprintf(name, sizeof(name), "%s ctrlq%d", device_get_nameunit(sc->dev), 3534 idx); 3535 init_eq(sc, &ctrlq->eq, EQ_CTRL, CTRL_EQ_QSIZE, sc->port[idx]->tx_chan, 3536 sc->sge.fwq.cntxt_id, name); 3537 3538 children = SYSCTL_CHILDREN(oid); 3539 snprintf(name, sizeof(name), "%d", idx); 3540 oid = SYSCTL_ADD_NODE(&sc->ctx, children, OID_AUTO, name, 3541 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "ctrl queue"); 3542 rc = alloc_wrq(sc, NULL, ctrlq, oid); 3543 3544 return (rc); 3545 } 3546 3547 int 3548 tnl_cong(struct port_info *pi, int drop) 3549 { 3550 3551 if (drop == -1) 3552 return (-1); 3553 else if (drop == 1) 3554 return (0); 3555 else 3556 return (pi->rx_e_chan_map); 3557 } 3558 3559 static int 3560 alloc_rxq(struct vi_info *vi, struct sge_rxq *rxq, int intr_idx, int idx, 3561 struct sysctl_oid *oid) 3562 { 3563 int rc; 3564 struct adapter *sc = vi->adapter; 3565 struct sysctl_oid_list *children; 3566 char name[16]; 3567 3568 rc = alloc_iq_fl(vi, &rxq->iq, &rxq->fl, intr_idx, 3569 tnl_cong(vi->pi, cong_drop)); 3570 if (rc != 0) 3571 return (rc); 3572 3573 if (idx == 0) 3574 sc->sge.iq_base = rxq->iq.abs_id - rxq->iq.cntxt_id; 3575 else 3576 KASSERT(rxq->iq.cntxt_id + sc->sge.iq_base == rxq->iq.abs_id, 3577 ("iq_base mismatch")); 3578 KASSERT(sc->sge.iq_base == 0 || sc->flags & IS_VF, 3579 ("PF with non-zero iq_base")); 3580 3581 /* 3582 * The freelist is just barely above the starvation threshold right now, 3583 * fill it up a bit more. 3584 */ 3585 FL_LOCK(&rxq->fl); 3586 refill_fl(sc, &rxq->fl, 128); 3587 FL_UNLOCK(&rxq->fl); 3588 3589 #if defined(INET) || defined(INET6) 3590 rc = tcp_lro_init_args(&rxq->lro, vi->ifp, lro_entries, lro_mbufs); 3591 if (rc != 0) 3592 return (rc); 3593 MPASS(rxq->lro.ifp == vi->ifp); /* also indicates LRO init'ed */ 3594 3595 if (vi->ifp->if_capenable & IFCAP_LRO) 3596 rxq->iq.flags |= IQ_LRO_ENABLED; 3597 #endif 3598 if (vi->ifp->if_capenable & IFCAP_HWRXTSTMP) 3599 rxq->iq.flags |= IQ_RX_TIMESTAMP; 3600 rxq->ifp = vi->ifp; 3601 3602 children = SYSCTL_CHILDREN(oid); 3603 3604 snprintf(name, sizeof(name), "%d", idx); 3605 oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, name, 3606 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "rx queue"); 3607 children = SYSCTL_CHILDREN(oid); 3608 3609 add_iq_sysctls(&vi->ctx, oid, &rxq->iq); 3610 #if defined(INET) || defined(INET6) 3611 SYSCTL_ADD_U64(&vi->ctx, children, OID_AUTO, "lro_queued", CTLFLAG_RD, 3612 &rxq->lro.lro_queued, 0, NULL); 3613 SYSCTL_ADD_U64(&vi->ctx, children, OID_AUTO, "lro_flushed", CTLFLAG_RD, 3614 &rxq->lro.lro_flushed, 0, NULL); 3615 #endif 3616 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "rxcsum", CTLFLAG_RD, 3617 &rxq->rxcsum, "# of times hardware assisted with checksum"); 3618 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "vlan_extraction", 3619 CTLFLAG_RD, &rxq->vlan_extraction, 3620 "# of times hardware extracted 802.1Q tag"); 3621 3622 add_fl_sysctls(sc, &vi->ctx, oid, &rxq->fl); 3623 3624 return (rc); 3625 } 3626 3627 static int 3628 free_rxq(struct vi_info *vi, struct sge_rxq *rxq) 3629 { 3630 int rc; 3631 3632 #if defined(INET) || defined(INET6) 3633 if (rxq->lro.ifp) { 3634 tcp_lro_free(&rxq->lro); 3635 rxq->lro.ifp = NULL; 3636 } 3637 #endif 3638 3639 rc = free_iq_fl(vi, &rxq->iq, &rxq->fl); 3640 if (rc == 0) 3641 bzero(rxq, sizeof(*rxq)); 3642 3643 return (rc); 3644 } 3645 3646 #ifdef TCP_OFFLOAD 3647 static int 3648 alloc_ofld_rxq(struct vi_info *vi, struct sge_ofld_rxq *ofld_rxq, 3649 int intr_idx, int idx, struct sysctl_oid *oid) 3650 { 3651 struct port_info *pi = vi->pi; 3652 int rc; 3653 struct sysctl_oid_list *children; 3654 char name[16]; 3655 3656 rc = alloc_iq_fl(vi, &ofld_rxq->iq, &ofld_rxq->fl, intr_idx, 0); 3657 if (rc != 0) 3658 return (rc); 3659 3660 children = SYSCTL_CHILDREN(oid); 3661 3662 snprintf(name, sizeof(name), "%d", idx); 3663 oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, name, 3664 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "rx queue"); 3665 add_iq_sysctls(&vi->ctx, oid, &ofld_rxq->iq); 3666 add_fl_sysctls(pi->adapter, &vi->ctx, oid, &ofld_rxq->fl); 3667 3668 return (rc); 3669 } 3670 3671 static int 3672 free_ofld_rxq(struct vi_info *vi, struct sge_ofld_rxq *ofld_rxq) 3673 { 3674 int rc; 3675 3676 rc = free_iq_fl(vi, &ofld_rxq->iq, &ofld_rxq->fl); 3677 if (rc == 0) 3678 bzero(ofld_rxq, sizeof(*ofld_rxq)); 3679 3680 return (rc); 3681 } 3682 #endif 3683 3684 #ifdef DEV_NETMAP 3685 static int 3686 alloc_nm_rxq(struct vi_info *vi, struct sge_nm_rxq *nm_rxq, int intr_idx, 3687 int idx, struct sysctl_oid *oid) 3688 { 3689 int rc; 3690 struct sysctl_oid_list *children; 3691 struct sysctl_ctx_list *ctx; 3692 char name[16]; 3693 size_t len; 3694 struct adapter *sc = vi->adapter; 3695 struct netmap_adapter *na = NA(vi->ifp); 3696 3697 MPASS(na != NULL); 3698 3699 len = vi->qsize_rxq * IQ_ESIZE; 3700 rc = alloc_ring(sc, len, &nm_rxq->iq_desc_tag, &nm_rxq->iq_desc_map, 3701 &nm_rxq->iq_ba, (void **)&nm_rxq->iq_desc); 3702 if (rc != 0) 3703 return (rc); 3704 3705 len = na->num_rx_desc * EQ_ESIZE + sc->params.sge.spg_len; 3706 rc = alloc_ring(sc, len, &nm_rxq->fl_desc_tag, &nm_rxq->fl_desc_map, 3707 &nm_rxq->fl_ba, (void **)&nm_rxq->fl_desc); 3708 if (rc != 0) 3709 return (rc); 3710 3711 nm_rxq->vi = vi; 3712 nm_rxq->nid = idx; 3713 nm_rxq->iq_cidx = 0; 3714 nm_rxq->iq_sidx = vi->qsize_rxq - sc->params.sge.spg_len / IQ_ESIZE; 3715 nm_rxq->iq_gen = F_RSPD_GEN; 3716 nm_rxq->fl_pidx = nm_rxq->fl_cidx = 0; 3717 nm_rxq->fl_sidx = na->num_rx_desc; 3718 nm_rxq->fl_sidx2 = nm_rxq->fl_sidx; /* copy for rxsync cacheline */ 3719 nm_rxq->intr_idx = intr_idx; 3720 nm_rxq->iq_cntxt_id = INVALID_NM_RXQ_CNTXT_ID; 3721 3722 ctx = &vi->ctx; 3723 children = SYSCTL_CHILDREN(oid); 3724 3725 snprintf(name, sizeof(name), "%d", idx); 3726 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, name, 3727 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "rx queue"); 3728 children = SYSCTL_CHILDREN(oid); 3729 3730 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "abs_id", 3731 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, &nm_rxq->iq_abs_id, 3732 0, sysctl_uint16, "I", "absolute id of the queue"); 3733 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cntxt_id", 3734 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, &nm_rxq->iq_cntxt_id, 3735 0, sysctl_uint16, "I", "SGE context id of the queue"); 3736 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cidx", 3737 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, &nm_rxq->iq_cidx, 0, 3738 sysctl_uint16, "I", "consumer index"); 3739 3740 children = SYSCTL_CHILDREN(oid); 3741 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "fl", 3742 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "freelist"); 3743 children = SYSCTL_CHILDREN(oid); 3744 3745 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cntxt_id", 3746 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, &nm_rxq->fl_cntxt_id, 3747 0, sysctl_uint16, "I", "SGE context id of the freelist"); 3748 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cidx", CTLFLAG_RD, 3749 &nm_rxq->fl_cidx, 0, "consumer index"); 3750 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "pidx", CTLFLAG_RD, 3751 &nm_rxq->fl_pidx, 0, "producer index"); 3752 3753 return (rc); 3754 } 3755 3756 3757 static int 3758 free_nm_rxq(struct vi_info *vi, struct sge_nm_rxq *nm_rxq) 3759 { 3760 struct adapter *sc = vi->adapter; 3761 3762 if (vi->flags & VI_INIT_DONE) 3763 MPASS(nm_rxq->iq_cntxt_id == INVALID_NM_RXQ_CNTXT_ID); 3764 else 3765 MPASS(nm_rxq->iq_cntxt_id == 0); 3766 3767 free_ring(sc, nm_rxq->iq_desc_tag, nm_rxq->iq_desc_map, nm_rxq->iq_ba, 3768 nm_rxq->iq_desc); 3769 free_ring(sc, nm_rxq->fl_desc_tag, nm_rxq->fl_desc_map, nm_rxq->fl_ba, 3770 nm_rxq->fl_desc); 3771 3772 return (0); 3773 } 3774 3775 static int 3776 alloc_nm_txq(struct vi_info *vi, struct sge_nm_txq *nm_txq, int iqidx, int idx, 3777 struct sysctl_oid *oid) 3778 { 3779 int rc; 3780 size_t len; 3781 struct port_info *pi = vi->pi; 3782 struct adapter *sc = pi->adapter; 3783 struct netmap_adapter *na = NA(vi->ifp); 3784 char name[16]; 3785 struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); 3786 3787 len = na->num_tx_desc * EQ_ESIZE + sc->params.sge.spg_len; 3788 rc = alloc_ring(sc, len, &nm_txq->desc_tag, &nm_txq->desc_map, 3789 &nm_txq->ba, (void **)&nm_txq->desc); 3790 if (rc) 3791 return (rc); 3792 3793 nm_txq->pidx = nm_txq->cidx = 0; 3794 nm_txq->sidx = na->num_tx_desc; 3795 nm_txq->nid = idx; 3796 nm_txq->iqidx = iqidx; 3797 nm_txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT) | 3798 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(sc->pf) | 3799 V_TXPKT_VF(vi->vin) | V_TXPKT_VF_VLD(vi->vfvld)); 3800 if (sc->params.fw_vers >= FW_VERSION32(1, 24, 11, 0)) 3801 nm_txq->op_pkd = htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS2_WR)); 3802 else 3803 nm_txq->op_pkd = htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR)); 3804 nm_txq->cntxt_id = INVALID_NM_TXQ_CNTXT_ID; 3805 3806 snprintf(name, sizeof(name), "%d", idx); 3807 oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, name, 3808 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "netmap tx queue"); 3809 children = SYSCTL_CHILDREN(oid); 3810 3811 SYSCTL_ADD_UINT(&vi->ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD, 3812 &nm_txq->cntxt_id, 0, "SGE context id of the queue"); 3813 SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "cidx", 3814 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, &nm_txq->cidx, 0, 3815 sysctl_uint16, "I", "consumer index"); 3816 SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "pidx", 3817 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, &nm_txq->pidx, 0, 3818 sysctl_uint16, "I", "producer index"); 3819 3820 return (rc); 3821 } 3822 3823 static int 3824 free_nm_txq(struct vi_info *vi, struct sge_nm_txq *nm_txq) 3825 { 3826 struct adapter *sc = vi->adapter; 3827 3828 if (vi->flags & VI_INIT_DONE) 3829 MPASS(nm_txq->cntxt_id == INVALID_NM_TXQ_CNTXT_ID); 3830 else 3831 MPASS(nm_txq->cntxt_id == 0); 3832 3833 free_ring(sc, nm_txq->desc_tag, nm_txq->desc_map, nm_txq->ba, 3834 nm_txq->desc); 3835 3836 return (0); 3837 } 3838 #endif 3839 3840 /* 3841 * Returns a reasonable automatic cidx flush threshold for a given queue size. 3842 */ 3843 static u_int 3844 qsize_to_fthresh(int qsize) 3845 { 3846 u_int fthresh; 3847 3848 while (!powerof2(qsize)) 3849 qsize++; 3850 fthresh = ilog2(qsize); 3851 if (fthresh > X_CIDXFLUSHTHRESH_128) 3852 fthresh = X_CIDXFLUSHTHRESH_128; 3853 3854 return (fthresh); 3855 } 3856 3857 static int 3858 ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq) 3859 { 3860 int rc, cntxt_id; 3861 struct fw_eq_ctrl_cmd c; 3862 int qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE; 3863 3864 bzero(&c, sizeof(c)); 3865 3866 c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_CTRL_CMD) | F_FW_CMD_REQUEST | 3867 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_CTRL_CMD_PFN(sc->pf) | 3868 V_FW_EQ_CTRL_CMD_VFN(0)); 3869 c.alloc_to_len16 = htobe32(F_FW_EQ_CTRL_CMD_ALLOC | 3870 F_FW_EQ_CTRL_CMD_EQSTART | FW_LEN16(c)); 3871 c.cmpliqid_eqid = htonl(V_FW_EQ_CTRL_CMD_CMPLIQID(eq->iqid)); 3872 c.physeqid_pkd = htobe32(0); 3873 c.fetchszm_to_iqid = 3874 htobe32(V_FW_EQ_CTRL_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) | 3875 V_FW_EQ_CTRL_CMD_PCIECHN(eq->tx_chan) | 3876 F_FW_EQ_CTRL_CMD_FETCHRO | V_FW_EQ_CTRL_CMD_IQID(eq->iqid)); 3877 c.dcaen_to_eqsize = 3878 htobe32(V_FW_EQ_CTRL_CMD_FBMIN(chip_id(sc) <= CHELSIO_T5 ? 3879 X_FETCHBURSTMIN_64B : X_FETCHBURSTMIN_64B_T6) | 3880 V_FW_EQ_CTRL_CMD_FBMAX(X_FETCHBURSTMAX_512B) | 3881 V_FW_EQ_CTRL_CMD_CIDXFTHRESH(qsize_to_fthresh(qsize)) | 3882 V_FW_EQ_CTRL_CMD_EQSIZE(qsize)); 3883 c.eqaddr = htobe64(eq->ba); 3884 3885 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c); 3886 if (rc != 0) { 3887 device_printf(sc->dev, 3888 "failed to create control queue %d: %d\n", eq->tx_chan, rc); 3889 return (rc); 3890 } 3891 eq->flags |= EQ_ALLOCATED; 3892 3893 eq->cntxt_id = G_FW_EQ_CTRL_CMD_EQID(be32toh(c.cmpliqid_eqid)); 3894 cntxt_id = eq->cntxt_id - sc->sge.eq_start; 3895 if (cntxt_id >= sc->sge.neq) 3896 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__, 3897 cntxt_id, sc->sge.neq - 1); 3898 sc->sge.eqmap[cntxt_id] = eq; 3899 3900 return (rc); 3901 } 3902 3903 static int 3904 eth_eq_alloc(struct adapter *sc, struct vi_info *vi, struct sge_eq *eq) 3905 { 3906 int rc, cntxt_id; 3907 struct fw_eq_eth_cmd c; 3908 int qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE; 3909 3910 bzero(&c, sizeof(c)); 3911 3912 c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST | 3913 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) | 3914 V_FW_EQ_ETH_CMD_VFN(0)); 3915 c.alloc_to_len16 = htobe32(F_FW_EQ_ETH_CMD_ALLOC | 3916 F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c)); 3917 c.autoequiqe_to_viid = htobe32(F_FW_EQ_ETH_CMD_AUTOEQUIQE | 3918 F_FW_EQ_ETH_CMD_AUTOEQUEQE | V_FW_EQ_ETH_CMD_VIID(vi->viid)); 3919 c.fetchszm_to_iqid = 3920 htobe32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_NONE) | 3921 V_FW_EQ_ETH_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO | 3922 V_FW_EQ_ETH_CMD_IQID(eq->iqid)); 3923 c.dcaen_to_eqsize = 3924 htobe32(V_FW_EQ_ETH_CMD_FBMIN(chip_id(sc) <= CHELSIO_T5 ? 3925 X_FETCHBURSTMIN_64B : X_FETCHBURSTMIN_64B_T6) | 3926 V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) | 3927 V_FW_EQ_ETH_CMD_EQSIZE(qsize)); 3928 c.eqaddr = htobe64(eq->ba); 3929 3930 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c); 3931 if (rc != 0) { 3932 device_printf(vi->dev, 3933 "failed to create Ethernet egress queue: %d\n", rc); 3934 return (rc); 3935 } 3936 eq->flags |= EQ_ALLOCATED; 3937 3938 eq->cntxt_id = G_FW_EQ_ETH_CMD_EQID(be32toh(c.eqid_pkd)); 3939 eq->abs_id = G_FW_EQ_ETH_CMD_PHYSEQID(be32toh(c.physeqid_pkd)); 3940 cntxt_id = eq->cntxt_id - sc->sge.eq_start; 3941 if (cntxt_id >= sc->sge.neq) 3942 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__, 3943 cntxt_id, sc->sge.neq - 1); 3944 sc->sge.eqmap[cntxt_id] = eq; 3945 3946 return (rc); 3947 } 3948 3949 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 3950 static int 3951 ofld_eq_alloc(struct adapter *sc, struct vi_info *vi, struct sge_eq *eq) 3952 { 3953 int rc, cntxt_id; 3954 struct fw_eq_ofld_cmd c; 3955 int qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE; 3956 3957 bzero(&c, sizeof(c)); 3958 3959 c.op_to_vfn = htonl(V_FW_CMD_OP(FW_EQ_OFLD_CMD) | F_FW_CMD_REQUEST | 3960 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_OFLD_CMD_PFN(sc->pf) | 3961 V_FW_EQ_OFLD_CMD_VFN(0)); 3962 c.alloc_to_len16 = htonl(F_FW_EQ_OFLD_CMD_ALLOC | 3963 F_FW_EQ_OFLD_CMD_EQSTART | FW_LEN16(c)); 3964 c.fetchszm_to_iqid = 3965 htonl(V_FW_EQ_OFLD_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) | 3966 V_FW_EQ_OFLD_CMD_PCIECHN(eq->tx_chan) | 3967 F_FW_EQ_OFLD_CMD_FETCHRO | V_FW_EQ_OFLD_CMD_IQID(eq->iqid)); 3968 c.dcaen_to_eqsize = 3969 htobe32(V_FW_EQ_OFLD_CMD_FBMIN(chip_id(sc) <= CHELSIO_T5 ? 3970 X_FETCHBURSTMIN_64B : X_FETCHBURSTMIN_64B_T6) | 3971 V_FW_EQ_OFLD_CMD_FBMAX(X_FETCHBURSTMAX_512B) | 3972 V_FW_EQ_OFLD_CMD_CIDXFTHRESH(qsize_to_fthresh(qsize)) | 3973 V_FW_EQ_OFLD_CMD_EQSIZE(qsize)); 3974 c.eqaddr = htobe64(eq->ba); 3975 3976 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c); 3977 if (rc != 0) { 3978 device_printf(vi->dev, 3979 "failed to create egress queue for TCP offload: %d\n", rc); 3980 return (rc); 3981 } 3982 eq->flags |= EQ_ALLOCATED; 3983 3984 eq->cntxt_id = G_FW_EQ_OFLD_CMD_EQID(be32toh(c.eqid_pkd)); 3985 cntxt_id = eq->cntxt_id - sc->sge.eq_start; 3986 if (cntxt_id >= sc->sge.neq) 3987 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__, 3988 cntxt_id, sc->sge.neq - 1); 3989 sc->sge.eqmap[cntxt_id] = eq; 3990 3991 return (rc); 3992 } 3993 #endif 3994 3995 static int 3996 alloc_eq(struct adapter *sc, struct vi_info *vi, struct sge_eq *eq) 3997 { 3998 int rc, qsize; 3999 size_t len; 4000 4001 mtx_init(&eq->eq_lock, eq->lockname, NULL, MTX_DEF); 4002 4003 qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE; 4004 len = qsize * EQ_ESIZE; 4005 rc = alloc_ring(sc, len, &eq->desc_tag, &eq->desc_map, 4006 &eq->ba, (void **)&eq->desc); 4007 if (rc) 4008 return (rc); 4009 4010 eq->pidx = eq->cidx = eq->dbidx = 0; 4011 /* Note that equeqidx is not used with sge_wrq (OFLD/CTRL) queues. */ 4012 eq->equeqidx = 0; 4013 eq->doorbells = sc->doorbells; 4014 4015 switch (eq->flags & EQ_TYPEMASK) { 4016 case EQ_CTRL: 4017 rc = ctrl_eq_alloc(sc, eq); 4018 break; 4019 4020 case EQ_ETH: 4021 rc = eth_eq_alloc(sc, vi, eq); 4022 break; 4023 4024 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 4025 case EQ_OFLD: 4026 rc = ofld_eq_alloc(sc, vi, eq); 4027 break; 4028 #endif 4029 4030 default: 4031 panic("%s: invalid eq type %d.", __func__, 4032 eq->flags & EQ_TYPEMASK); 4033 } 4034 if (rc != 0) { 4035 device_printf(sc->dev, 4036 "failed to allocate egress queue(%d): %d\n", 4037 eq->flags & EQ_TYPEMASK, rc); 4038 } 4039 4040 if (isset(&eq->doorbells, DOORBELL_UDB) || 4041 isset(&eq->doorbells, DOORBELL_UDBWC) || 4042 isset(&eq->doorbells, DOORBELL_WCWR)) { 4043 uint32_t s_qpp = sc->params.sge.eq_s_qpp; 4044 uint32_t mask = (1 << s_qpp) - 1; 4045 volatile uint8_t *udb; 4046 4047 udb = sc->udbs_base + UDBS_DB_OFFSET; 4048 udb += (eq->cntxt_id >> s_qpp) << PAGE_SHIFT; /* pg offset */ 4049 eq->udb_qid = eq->cntxt_id & mask; /* id in page */ 4050 if (eq->udb_qid >= PAGE_SIZE / UDBS_SEG_SIZE) 4051 clrbit(&eq->doorbells, DOORBELL_WCWR); 4052 else { 4053 udb += eq->udb_qid << UDBS_SEG_SHIFT; /* seg offset */ 4054 eq->udb_qid = 0; 4055 } 4056 eq->udb = (volatile void *)udb; 4057 } 4058 4059 return (rc); 4060 } 4061 4062 static int 4063 free_eq(struct adapter *sc, struct sge_eq *eq) 4064 { 4065 int rc; 4066 4067 if (eq->flags & EQ_ALLOCATED) { 4068 switch (eq->flags & EQ_TYPEMASK) { 4069 case EQ_CTRL: 4070 rc = -t4_ctrl_eq_free(sc, sc->mbox, sc->pf, 0, 4071 eq->cntxt_id); 4072 break; 4073 4074 case EQ_ETH: 4075 rc = -t4_eth_eq_free(sc, sc->mbox, sc->pf, 0, 4076 eq->cntxt_id); 4077 break; 4078 4079 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 4080 case EQ_OFLD: 4081 rc = -t4_ofld_eq_free(sc, sc->mbox, sc->pf, 0, 4082 eq->cntxt_id); 4083 break; 4084 #endif 4085 4086 default: 4087 panic("%s: invalid eq type %d.", __func__, 4088 eq->flags & EQ_TYPEMASK); 4089 } 4090 if (rc != 0) { 4091 device_printf(sc->dev, 4092 "failed to free egress queue (%d): %d\n", 4093 eq->flags & EQ_TYPEMASK, rc); 4094 return (rc); 4095 } 4096 eq->flags &= ~EQ_ALLOCATED; 4097 } 4098 4099 free_ring(sc, eq->desc_tag, eq->desc_map, eq->ba, eq->desc); 4100 4101 if (mtx_initialized(&eq->eq_lock)) 4102 mtx_destroy(&eq->eq_lock); 4103 4104 bzero(eq, sizeof(*eq)); 4105 return (0); 4106 } 4107 4108 static int 4109 alloc_wrq(struct adapter *sc, struct vi_info *vi, struct sge_wrq *wrq, 4110 struct sysctl_oid *oid) 4111 { 4112 int rc; 4113 struct sysctl_ctx_list *ctx = vi ? &vi->ctx : &sc->ctx; 4114 struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); 4115 4116 rc = alloc_eq(sc, vi, &wrq->eq); 4117 if (rc) 4118 return (rc); 4119 4120 wrq->adapter = sc; 4121 TASK_INIT(&wrq->wrq_tx_task, 0, wrq_tx_drain, wrq); 4122 TAILQ_INIT(&wrq->incomplete_wrs); 4123 STAILQ_INIT(&wrq->wr_list); 4124 wrq->nwr_pending = 0; 4125 wrq->ndesc_needed = 0; 4126 4127 SYSCTL_ADD_UAUTO(ctx, children, OID_AUTO, "ba", CTLFLAG_RD, 4128 &wrq->eq.ba, "bus address of descriptor ring"); 4129 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "dmalen", CTLFLAG_RD, NULL, 4130 wrq->eq.sidx * EQ_ESIZE + sc->params.sge.spg_len, 4131 "desc ring size in bytes"); 4132 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD, 4133 &wrq->eq.cntxt_id, 0, "SGE context id of the queue"); 4134 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cidx", 4135 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, &wrq->eq.cidx, 0, 4136 sysctl_uint16, "I", "consumer index"); 4137 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pidx", 4138 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, &wrq->eq.pidx, 0, 4139 sysctl_uint16, "I", "producer index"); 4140 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sidx", CTLFLAG_RD, NULL, 4141 wrq->eq.sidx, "status page index"); 4142 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tx_wrs_direct", CTLFLAG_RD, 4143 &wrq->tx_wrs_direct, "# of work requests (direct)"); 4144 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tx_wrs_copied", CTLFLAG_RD, 4145 &wrq->tx_wrs_copied, "# of work requests (copied)"); 4146 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tx_wrs_sspace", CTLFLAG_RD, 4147 &wrq->tx_wrs_ss, "# of work requests (copied from scratch space)"); 4148 4149 return (rc); 4150 } 4151 4152 static int 4153 free_wrq(struct adapter *sc, struct sge_wrq *wrq) 4154 { 4155 int rc; 4156 4157 rc = free_eq(sc, &wrq->eq); 4158 if (rc) 4159 return (rc); 4160 4161 bzero(wrq, sizeof(*wrq)); 4162 return (0); 4163 } 4164 4165 static int 4166 alloc_txq(struct vi_info *vi, struct sge_txq *txq, int idx, 4167 struct sysctl_oid *oid) 4168 { 4169 int rc; 4170 struct port_info *pi = vi->pi; 4171 struct adapter *sc = pi->adapter; 4172 struct sge_eq *eq = &txq->eq; 4173 struct txpkts *txp; 4174 char name[16]; 4175 struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); 4176 4177 rc = mp_ring_alloc(&txq->r, eq->sidx, txq, eth_tx, can_resume_eth_tx, 4178 M_CXGBE, &eq->eq_lock, M_WAITOK); 4179 if (rc != 0) { 4180 device_printf(sc->dev, "failed to allocate mp_ring: %d\n", rc); 4181 return (rc); 4182 } 4183 4184 rc = alloc_eq(sc, vi, eq); 4185 if (rc != 0) { 4186 mp_ring_free(txq->r); 4187 txq->r = NULL; 4188 return (rc); 4189 } 4190 4191 /* Can't fail after this point. */ 4192 4193 if (idx == 0) 4194 sc->sge.eq_base = eq->abs_id - eq->cntxt_id; 4195 else 4196 KASSERT(eq->cntxt_id + sc->sge.eq_base == eq->abs_id, 4197 ("eq_base mismatch")); 4198 KASSERT(sc->sge.eq_base == 0 || sc->flags & IS_VF, 4199 ("PF with non-zero eq_base")); 4200 4201 TASK_INIT(&txq->tx_reclaim_task, 0, tx_reclaim, eq); 4202 txq->ifp = vi->ifp; 4203 txq->gl = sglist_alloc(TX_SGL_SEGS, M_WAITOK); 4204 if (sc->flags & IS_VF) 4205 txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) | 4206 V_TXPKT_INTF(pi->tx_chan)); 4207 else 4208 txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) | 4209 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(sc->pf) | 4210 V_TXPKT_VF(vi->vin) | V_TXPKT_VF_VLD(vi->vfvld)); 4211 txq->tc_idx = -1; 4212 txq->sdesc = malloc(eq->sidx * sizeof(struct tx_sdesc), M_CXGBE, 4213 M_ZERO | M_WAITOK); 4214 4215 txp = &txq->txp; 4216 txp->score = 5; 4217 MPASS(nitems(txp->mb) >= sc->params.max_pkts_per_eth_tx_pkts_wr); 4218 txq->txp.max_npkt = min(nitems(txp->mb), 4219 sc->params.max_pkts_per_eth_tx_pkts_wr); 4220 4221 snprintf(name, sizeof(name), "%d", idx); 4222 oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, name, 4223 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "tx queue"); 4224 children = SYSCTL_CHILDREN(oid); 4225 4226 SYSCTL_ADD_UAUTO(&vi->ctx, children, OID_AUTO, "ba", CTLFLAG_RD, 4227 &eq->ba, "bus address of descriptor ring"); 4228 SYSCTL_ADD_INT(&vi->ctx, children, OID_AUTO, "dmalen", CTLFLAG_RD, NULL, 4229 eq->sidx * EQ_ESIZE + sc->params.sge.spg_len, 4230 "desc ring size in bytes"); 4231 SYSCTL_ADD_UINT(&vi->ctx, children, OID_AUTO, "abs_id", CTLFLAG_RD, 4232 &eq->abs_id, 0, "absolute id of the queue"); 4233 SYSCTL_ADD_UINT(&vi->ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD, 4234 &eq->cntxt_id, 0, "SGE context id of the queue"); 4235 SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "cidx", 4236 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, &eq->cidx, 0, 4237 sysctl_uint16, "I", "consumer index"); 4238 SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "pidx", 4239 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, &eq->pidx, 0, 4240 sysctl_uint16, "I", "producer index"); 4241 SYSCTL_ADD_INT(&vi->ctx, children, OID_AUTO, "sidx", CTLFLAG_RD, NULL, 4242 eq->sidx, "status page index"); 4243 4244 SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "tc", 4245 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, vi, idx, sysctl_tc, 4246 "I", "traffic class (-1 means none)"); 4247 4248 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txcsum", CTLFLAG_RD, 4249 &txq->txcsum, "# of times hardware assisted with checksum"); 4250 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "vlan_insertion", 4251 CTLFLAG_RD, &txq->vlan_insertion, 4252 "# of times hardware inserted 802.1Q tag"); 4253 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "tso_wrs", CTLFLAG_RD, 4254 &txq->tso_wrs, "# of TSO work requests"); 4255 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "imm_wrs", CTLFLAG_RD, 4256 &txq->imm_wrs, "# of work requests with immediate data"); 4257 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "sgl_wrs", CTLFLAG_RD, 4258 &txq->sgl_wrs, "# of work requests with direct SGL"); 4259 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txpkt_wrs", CTLFLAG_RD, 4260 &txq->txpkt_wrs, "# of txpkt work requests (one pkt/WR)"); 4261 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txpkts0_wrs", 4262 CTLFLAG_RD, &txq->txpkts0_wrs, 4263 "# of txpkts (type 0) work requests"); 4264 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txpkts1_wrs", 4265 CTLFLAG_RD, &txq->txpkts1_wrs, 4266 "# of txpkts (type 1) work requests"); 4267 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txpkts0_pkts", 4268 CTLFLAG_RD, &txq->txpkts0_pkts, 4269 "# of frames tx'd using type0 txpkts work requests"); 4270 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txpkts1_pkts", 4271 CTLFLAG_RD, &txq->txpkts1_pkts, 4272 "# of frames tx'd using type1 txpkts work requests"); 4273 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "raw_wrs", CTLFLAG_RD, 4274 &txq->raw_wrs, "# of raw work requests (non-packets)"); 4275 4276 #ifdef KERN_TLS 4277 if (sc->flags & KERN_TLS_OK) { 4278 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, 4279 "kern_tls_records", CTLFLAG_RD, &txq->kern_tls_records, 4280 "# of NIC TLS records transmitted"); 4281 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, 4282 "kern_tls_short", CTLFLAG_RD, &txq->kern_tls_short, 4283 "# of short NIC TLS records transmitted"); 4284 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, 4285 "kern_tls_partial", CTLFLAG_RD, &txq->kern_tls_partial, 4286 "# of partial NIC TLS records transmitted"); 4287 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, 4288 "kern_tls_full", CTLFLAG_RD, &txq->kern_tls_full, 4289 "# of full NIC TLS records transmitted"); 4290 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, 4291 "kern_tls_octets", CTLFLAG_RD, &txq->kern_tls_octets, 4292 "# of payload octets in transmitted NIC TLS records"); 4293 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, 4294 "kern_tls_waste", CTLFLAG_RD, &txq->kern_tls_waste, 4295 "# of octets DMAd but not transmitted in NIC TLS records"); 4296 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, 4297 "kern_tls_options", CTLFLAG_RD, &txq->kern_tls_options, 4298 "# of NIC TLS options-only packets transmitted"); 4299 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, 4300 "kern_tls_header", CTLFLAG_RD, &txq->kern_tls_header, 4301 "# of NIC TLS header-only packets transmitted"); 4302 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, 4303 "kern_tls_fin", CTLFLAG_RD, &txq->kern_tls_fin, 4304 "# of NIC TLS FIN-only packets transmitted"); 4305 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, 4306 "kern_tls_fin_short", CTLFLAG_RD, &txq->kern_tls_fin_short, 4307 "# of NIC TLS padded FIN packets on short TLS records"); 4308 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, 4309 "kern_tls_cbc", CTLFLAG_RD, &txq->kern_tls_cbc, 4310 "# of NIC TLS sessions using AES-CBC"); 4311 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, 4312 "kern_tls_gcm", CTLFLAG_RD, &txq->kern_tls_gcm, 4313 "# of NIC TLS sessions using AES-GCM"); 4314 } 4315 #endif 4316 mp_ring_sysctls(txq->r, &vi->ctx, children); 4317 4318 return (0); 4319 } 4320 4321 static int 4322 free_txq(struct vi_info *vi, struct sge_txq *txq) 4323 { 4324 int rc; 4325 struct adapter *sc = vi->adapter; 4326 struct sge_eq *eq = &txq->eq; 4327 4328 rc = free_eq(sc, eq); 4329 if (rc) 4330 return (rc); 4331 4332 sglist_free(txq->gl); 4333 free(txq->sdesc, M_CXGBE); 4334 mp_ring_free(txq->r); 4335 4336 bzero(txq, sizeof(*txq)); 4337 return (0); 4338 } 4339 4340 static void 4341 oneseg_dma_callback(void *arg, bus_dma_segment_t *segs, int nseg, int error) 4342 { 4343 bus_addr_t *ba = arg; 4344 4345 KASSERT(nseg == 1, 4346 ("%s meant for single segment mappings only.", __func__)); 4347 4348 *ba = error ? 0 : segs->ds_addr; 4349 } 4350 4351 static inline void 4352 ring_fl_db(struct adapter *sc, struct sge_fl *fl) 4353 { 4354 uint32_t n, v; 4355 4356 n = IDXDIFF(fl->pidx >> 3, fl->dbidx, fl->sidx); 4357 MPASS(n > 0); 4358 4359 wmb(); 4360 v = fl->dbval | V_PIDX(n); 4361 if (fl->udb) 4362 *fl->udb = htole32(v); 4363 else 4364 t4_write_reg(sc, sc->sge_kdoorbell_reg, v); 4365 IDXINCR(fl->dbidx, n, fl->sidx); 4366 } 4367 4368 /* 4369 * Fills up the freelist by allocating up to 'n' buffers. Buffers that are 4370 * recycled do not count towards this allocation budget. 4371 * 4372 * Returns non-zero to indicate that this freelist should be added to the list 4373 * of starving freelists. 4374 */ 4375 static int 4376 refill_fl(struct adapter *sc, struct sge_fl *fl, int n) 4377 { 4378 __be64 *d; 4379 struct fl_sdesc *sd; 4380 uintptr_t pa; 4381 caddr_t cl; 4382 struct rx_buf_info *rxb; 4383 struct cluster_metadata *clm; 4384 uint16_t max_pidx; 4385 uint16_t hw_cidx = fl->hw_cidx; /* stable snapshot */ 4386 4387 FL_LOCK_ASSERT_OWNED(fl); 4388 4389 /* 4390 * We always stop at the beginning of the hardware descriptor that's just 4391 * before the one with the hw cidx. This is to avoid hw pidx = hw cidx, 4392 * which would mean an empty freelist to the chip. 4393 */ 4394 max_pidx = __predict_false(hw_cidx == 0) ? fl->sidx - 1 : hw_cidx - 1; 4395 if (fl->pidx == max_pidx * 8) 4396 return (0); 4397 4398 d = &fl->desc[fl->pidx]; 4399 sd = &fl->sdesc[fl->pidx]; 4400 4401 while (n > 0) { 4402 4403 if (sd->cl != NULL) { 4404 4405 if (sd->nmbuf == 0) { 4406 /* 4407 * Fast recycle without involving any atomics on 4408 * the cluster's metadata (if the cluster has 4409 * metadata). This happens when all frames 4410 * received in the cluster were small enough to 4411 * fit within a single mbuf each. 4412 */ 4413 fl->cl_fast_recycled++; 4414 goto recycled; 4415 } 4416 4417 /* 4418 * Cluster is guaranteed to have metadata. Clusters 4419 * without metadata always take the fast recycle path 4420 * when they're recycled. 4421 */ 4422 clm = cl_metadata(sd); 4423 MPASS(clm != NULL); 4424 4425 if (atomic_fetchadd_int(&clm->refcount, -1) == 1) { 4426 fl->cl_recycled++; 4427 counter_u64_add(extfree_rels, 1); 4428 goto recycled; 4429 } 4430 sd->cl = NULL; /* gave up my reference */ 4431 } 4432 MPASS(sd->cl == NULL); 4433 rxb = &sc->sge.rx_buf_info[fl->zidx]; 4434 cl = uma_zalloc(rxb->zone, M_NOWAIT); 4435 if (__predict_false(cl == NULL)) { 4436 if (fl->zidx != fl->safe_zidx) { 4437 rxb = &sc->sge.rx_buf_info[fl->safe_zidx]; 4438 cl = uma_zalloc(rxb->zone, M_NOWAIT); 4439 } 4440 if (cl == NULL) 4441 break; 4442 } 4443 fl->cl_allocated++; 4444 n--; 4445 4446 pa = pmap_kextract((vm_offset_t)cl); 4447 sd->cl = cl; 4448 sd->zidx = fl->zidx; 4449 4450 if (fl->flags & FL_BUF_PACKING) { 4451 *d = htobe64(pa | rxb->hwidx2); 4452 sd->moff = rxb->size2; 4453 } else { 4454 *d = htobe64(pa | rxb->hwidx1); 4455 sd->moff = 0; 4456 } 4457 recycled: 4458 sd->nmbuf = 0; 4459 d++; 4460 sd++; 4461 if (__predict_false((++fl->pidx & 7) == 0)) { 4462 uint16_t pidx = fl->pidx >> 3; 4463 4464 if (__predict_false(pidx == fl->sidx)) { 4465 fl->pidx = 0; 4466 pidx = 0; 4467 sd = fl->sdesc; 4468 d = fl->desc; 4469 } 4470 if (n < 8 || pidx == max_pidx) 4471 break; 4472 4473 if (IDXDIFF(pidx, fl->dbidx, fl->sidx) >= 4) 4474 ring_fl_db(sc, fl); 4475 } 4476 } 4477 4478 if ((fl->pidx >> 3) != fl->dbidx) 4479 ring_fl_db(sc, fl); 4480 4481 return (FL_RUNNING_LOW(fl) && !(fl->flags & FL_STARVING)); 4482 } 4483 4484 /* 4485 * Attempt to refill all starving freelists. 4486 */ 4487 static void 4488 refill_sfl(void *arg) 4489 { 4490 struct adapter *sc = arg; 4491 struct sge_fl *fl, *fl_temp; 4492 4493 mtx_assert(&sc->sfl_lock, MA_OWNED); 4494 TAILQ_FOREACH_SAFE(fl, &sc->sfl, link, fl_temp) { 4495 FL_LOCK(fl); 4496 refill_fl(sc, fl, 64); 4497 if (FL_NOT_RUNNING_LOW(fl) || fl->flags & FL_DOOMED) { 4498 TAILQ_REMOVE(&sc->sfl, fl, link); 4499 fl->flags &= ~FL_STARVING; 4500 } 4501 FL_UNLOCK(fl); 4502 } 4503 4504 if (!TAILQ_EMPTY(&sc->sfl)) 4505 callout_schedule(&sc->sfl_callout, hz / 5); 4506 } 4507 4508 static int 4509 alloc_fl_sdesc(struct sge_fl *fl) 4510 { 4511 4512 fl->sdesc = malloc(fl->sidx * 8 * sizeof(struct fl_sdesc), M_CXGBE, 4513 M_ZERO | M_WAITOK); 4514 4515 return (0); 4516 } 4517 4518 static void 4519 free_fl_sdesc(struct adapter *sc, struct sge_fl *fl) 4520 { 4521 struct fl_sdesc *sd; 4522 struct cluster_metadata *clm; 4523 int i; 4524 4525 sd = fl->sdesc; 4526 for (i = 0; i < fl->sidx * 8; i++, sd++) { 4527 if (sd->cl == NULL) 4528 continue; 4529 4530 if (sd->nmbuf == 0) 4531 uma_zfree(sc->sge.rx_buf_info[sd->zidx].zone, sd->cl); 4532 else if (fl->flags & FL_BUF_PACKING) { 4533 clm = cl_metadata(sd); 4534 if (atomic_fetchadd_int(&clm->refcount, -1) == 1) { 4535 uma_zfree(sc->sge.rx_buf_info[sd->zidx].zone, 4536 sd->cl); 4537 counter_u64_add(extfree_rels, 1); 4538 } 4539 } 4540 sd->cl = NULL; 4541 } 4542 4543 free(fl->sdesc, M_CXGBE); 4544 fl->sdesc = NULL; 4545 } 4546 4547 static inline void 4548 get_pkt_gl(struct mbuf *m, struct sglist *gl) 4549 { 4550 int rc; 4551 4552 M_ASSERTPKTHDR(m); 4553 4554 sglist_reset(gl); 4555 rc = sglist_append_mbuf(gl, m); 4556 if (__predict_false(rc != 0)) { 4557 panic("%s: mbuf %p (%d segs) was vetted earlier but now fails " 4558 "with %d.", __func__, m, mbuf_nsegs(m), rc); 4559 } 4560 4561 KASSERT(gl->sg_nseg == mbuf_nsegs(m), 4562 ("%s: nsegs changed for mbuf %p from %d to %d", __func__, m, 4563 mbuf_nsegs(m), gl->sg_nseg)); 4564 KASSERT(gl->sg_nseg > 0 && 4565 gl->sg_nseg <= (needs_tso(m) ? TX_SGL_SEGS_TSO : TX_SGL_SEGS), 4566 ("%s: %d segments, should have been 1 <= nsegs <= %d", __func__, 4567 gl->sg_nseg, needs_tso(m) ? TX_SGL_SEGS_TSO : TX_SGL_SEGS)); 4568 } 4569 4570 /* 4571 * len16 for a txpkt WR with a GL. Includes the firmware work request header. 4572 */ 4573 static inline u_int 4574 txpkt_len16(u_int nsegs, u_int tso) 4575 { 4576 u_int n; 4577 4578 MPASS(nsegs > 0); 4579 4580 nsegs--; /* first segment is part of ulptx_sgl */ 4581 n = sizeof(struct fw_eth_tx_pkt_wr) + sizeof(struct cpl_tx_pkt_core) + 4582 sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1)); 4583 if (tso) 4584 n += sizeof(struct cpl_tx_pkt_lso_core); 4585 4586 return (howmany(n, 16)); 4587 } 4588 4589 /* 4590 * len16 for a txpkt_vm WR with a GL. Includes the firmware work 4591 * request header. 4592 */ 4593 static inline u_int 4594 txpkt_vm_len16(u_int nsegs, u_int tso) 4595 { 4596 u_int n; 4597 4598 MPASS(nsegs > 0); 4599 4600 nsegs--; /* first segment is part of ulptx_sgl */ 4601 n = sizeof(struct fw_eth_tx_pkt_vm_wr) + 4602 sizeof(struct cpl_tx_pkt_core) + 4603 sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1)); 4604 if (tso) 4605 n += sizeof(struct cpl_tx_pkt_lso_core); 4606 4607 return (howmany(n, 16)); 4608 } 4609 4610 /* 4611 * len16 for a txpkts type 0 WR with a GL. Does not include the firmware work 4612 * request header. 4613 */ 4614 static inline u_int 4615 txpkts0_len16(u_int nsegs) 4616 { 4617 u_int n; 4618 4619 MPASS(nsegs > 0); 4620 4621 nsegs--; /* first segment is part of ulptx_sgl */ 4622 n = sizeof(struct ulp_txpkt) + sizeof(struct ulptx_idata) + 4623 sizeof(struct cpl_tx_pkt_core) + sizeof(struct ulptx_sgl) + 4624 8 * ((3 * nsegs) / 2 + (nsegs & 1)); 4625 4626 return (howmany(n, 16)); 4627 } 4628 4629 /* 4630 * len16 for a txpkts type 1 WR with a GL. Does not include the firmware work 4631 * request header. 4632 */ 4633 static inline u_int 4634 txpkts1_len16(void) 4635 { 4636 u_int n; 4637 4638 n = sizeof(struct cpl_tx_pkt_core) + sizeof(struct ulptx_sgl); 4639 4640 return (howmany(n, 16)); 4641 } 4642 4643 static inline u_int 4644 imm_payload(u_int ndesc) 4645 { 4646 u_int n; 4647 4648 n = ndesc * EQ_ESIZE - sizeof(struct fw_eth_tx_pkt_wr) - 4649 sizeof(struct cpl_tx_pkt_core); 4650 4651 return (n); 4652 } 4653 4654 static inline uint64_t 4655 csum_to_ctrl(struct adapter *sc, struct mbuf *m) 4656 { 4657 uint64_t ctrl; 4658 int csum_type; 4659 4660 M_ASSERTPKTHDR(m); 4661 4662 if (needs_hwcsum(m) == 0) 4663 return (F_TXPKT_IPCSUM_DIS | F_TXPKT_L4CSUM_DIS); 4664 4665 ctrl = 0; 4666 if (needs_l3_csum(m) == 0) 4667 ctrl |= F_TXPKT_IPCSUM_DIS; 4668 switch (m->m_pkthdr.csum_flags & 4669 (CSUM_IP_TCP | CSUM_IP_UDP | CSUM_IP6_TCP | CSUM_IP6_UDP)) { 4670 case CSUM_IP_TCP: 4671 csum_type = TX_CSUM_TCPIP; 4672 break; 4673 case CSUM_IP_UDP: 4674 csum_type = TX_CSUM_UDPIP; 4675 break; 4676 case CSUM_IP6_TCP: 4677 csum_type = TX_CSUM_TCPIP6; 4678 break; 4679 case CSUM_IP6_UDP: 4680 csum_type = TX_CSUM_UDPIP6; 4681 break; 4682 default: 4683 /* needs_hwcsum told us that at least some hwcsum is needed. */ 4684 MPASS(ctrl == 0); 4685 MPASS(m->m_pkthdr.csum_flags & CSUM_IP); 4686 ctrl |= F_TXPKT_L4CSUM_DIS; 4687 csum_type = TX_CSUM_IP; 4688 break; 4689 } 4690 4691 MPASS(m->m_pkthdr.l2hlen > 0); 4692 MPASS(m->m_pkthdr.l3hlen > 0); 4693 ctrl |= V_TXPKT_CSUM_TYPE(csum_type) | 4694 V_TXPKT_IPHDR_LEN(m->m_pkthdr.l3hlen); 4695 if (chip_id(sc) <= CHELSIO_T5) 4696 ctrl |= V_TXPKT_ETHHDR_LEN(m->m_pkthdr.l2hlen - ETHER_HDR_LEN); 4697 else 4698 ctrl |= V_T6_TXPKT_ETHHDR_LEN(m->m_pkthdr.l2hlen - ETHER_HDR_LEN); 4699 4700 return (ctrl); 4701 } 4702 4703 #define VM_TX_L2HDR_LEN 16 /* ethmacdst to vlantci */ 4704 4705 /* 4706 * Write a VM txpkt WR for this packet to the hardware descriptors, update the 4707 * software descriptor, and advance the pidx. It is guaranteed that enough 4708 * descriptors are available. 4709 * 4710 * The return value is the # of hardware descriptors used. 4711 */ 4712 static u_int 4713 write_txpkt_vm_wr(struct adapter *sc, struct sge_txq *txq, struct mbuf *m0) 4714 { 4715 struct sge_eq *eq; 4716 struct fw_eth_tx_pkt_vm_wr *wr; 4717 struct tx_sdesc *txsd; 4718 struct cpl_tx_pkt_core *cpl; 4719 uint32_t ctrl; /* used in many unrelated places */ 4720 uint64_t ctrl1; 4721 int len16, ndesc, pktlen, nsegs; 4722 caddr_t dst; 4723 4724 TXQ_LOCK_ASSERT_OWNED(txq); 4725 M_ASSERTPKTHDR(m0); 4726 4727 len16 = mbuf_len16(m0); 4728 nsegs = mbuf_nsegs(m0); 4729 pktlen = m0->m_pkthdr.len; 4730 ctrl = sizeof(struct cpl_tx_pkt_core); 4731 if (needs_tso(m0)) 4732 ctrl += sizeof(struct cpl_tx_pkt_lso_core); 4733 ndesc = tx_len16_to_desc(len16); 4734 4735 /* Firmware work request header */ 4736 eq = &txq->eq; 4737 wr = (void *)&eq->desc[eq->pidx]; 4738 wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_VM_WR) | 4739 V_FW_ETH_TX_PKT_WR_IMMDLEN(ctrl)); 4740 4741 ctrl = V_FW_WR_LEN16(len16); 4742 wr->equiq_to_len16 = htobe32(ctrl); 4743 wr->r3[0] = 0; 4744 wr->r3[1] = 0; 4745 4746 /* 4747 * Copy over ethmacdst, ethmacsrc, ethtype, and vlantci. 4748 * vlantci is ignored unless the ethtype is 0x8100, so it's 4749 * simpler to always copy it rather than making it 4750 * conditional. Also, it seems that we do not have to set 4751 * vlantci or fake the ethtype when doing VLAN tag insertion. 4752 */ 4753 m_copydata(m0, 0, VM_TX_L2HDR_LEN, wr->ethmacdst); 4754 4755 if (needs_tso(m0)) { 4756 struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1); 4757 4758 KASSERT(m0->m_pkthdr.l2hlen > 0 && m0->m_pkthdr.l3hlen > 0 && 4759 m0->m_pkthdr.l4hlen > 0, 4760 ("%s: mbuf %p needs TSO but missing header lengths", 4761 __func__, m0)); 4762 4763 ctrl = V_LSO_OPCODE(CPL_TX_PKT_LSO) | F_LSO_FIRST_SLICE | 4764 F_LSO_LAST_SLICE | V_LSO_ETHHDR_LEN((m0->m_pkthdr.l2hlen - 4765 ETHER_HDR_LEN) >> 2) | 4766 V_LSO_IPHDR_LEN(m0->m_pkthdr.l3hlen >> 2) | 4767 V_LSO_TCPHDR_LEN(m0->m_pkthdr.l4hlen >> 2); 4768 if (m0->m_pkthdr.l3hlen == sizeof(struct ip6_hdr)) 4769 ctrl |= F_LSO_IPV6; 4770 4771 lso->lso_ctrl = htobe32(ctrl); 4772 lso->ipid_ofst = htobe16(0); 4773 lso->mss = htobe16(m0->m_pkthdr.tso_segsz); 4774 lso->seqno_offset = htobe32(0); 4775 lso->len = htobe32(pktlen); 4776 4777 cpl = (void *)(lso + 1); 4778 4779 txq->tso_wrs++; 4780 } else 4781 cpl = (void *)(wr + 1); 4782 4783 /* Checksum offload */ 4784 ctrl1 = csum_to_ctrl(sc, m0); 4785 if (ctrl1 != (F_TXPKT_IPCSUM_DIS | F_TXPKT_L4CSUM_DIS)) 4786 txq->txcsum++; /* some hardware assistance provided */ 4787 4788 /* VLAN tag insertion */ 4789 if (needs_vlan_insertion(m0)) { 4790 ctrl1 |= F_TXPKT_VLAN_VLD | 4791 V_TXPKT_VLAN(m0->m_pkthdr.ether_vtag); 4792 txq->vlan_insertion++; 4793 } 4794 4795 /* CPL header */ 4796 cpl->ctrl0 = txq->cpl_ctrl0; 4797 cpl->pack = 0; 4798 cpl->len = htobe16(pktlen); 4799 cpl->ctrl1 = htobe64(ctrl1); 4800 4801 /* SGL */ 4802 dst = (void *)(cpl + 1); 4803 4804 /* 4805 * A packet using TSO will use up an entire descriptor for the 4806 * firmware work request header, LSO CPL, and TX_PKT_XT CPL. 4807 * If this descriptor is the last descriptor in the ring, wrap 4808 * around to the front of the ring explicitly for the start of 4809 * the sgl. 4810 */ 4811 if (dst == (void *)&eq->desc[eq->sidx]) { 4812 dst = (void *)&eq->desc[0]; 4813 write_gl_to_txd(txq, m0, &dst, 0); 4814 } else 4815 write_gl_to_txd(txq, m0, &dst, eq->sidx - ndesc < eq->pidx); 4816 txq->sgl_wrs++; 4817 txq->txpkt_wrs++; 4818 4819 txsd = &txq->sdesc[eq->pidx]; 4820 txsd->m = m0; 4821 txsd->desc_used = ndesc; 4822 4823 return (ndesc); 4824 } 4825 4826 /* 4827 * Write a raw WR to the hardware descriptors, update the software 4828 * descriptor, and advance the pidx. It is guaranteed that enough 4829 * descriptors are available. 4830 * 4831 * The return value is the # of hardware descriptors used. 4832 */ 4833 static u_int 4834 write_raw_wr(struct sge_txq *txq, void *wr, struct mbuf *m0, u_int available) 4835 { 4836 struct sge_eq *eq = &txq->eq; 4837 struct tx_sdesc *txsd; 4838 struct mbuf *m; 4839 caddr_t dst; 4840 int len16, ndesc; 4841 4842 len16 = mbuf_len16(m0); 4843 ndesc = tx_len16_to_desc(len16); 4844 MPASS(ndesc <= available); 4845 4846 dst = wr; 4847 for (m = m0; m != NULL; m = m->m_next) 4848 copy_to_txd(eq, mtod(m, caddr_t), &dst, m->m_len); 4849 4850 txq->raw_wrs++; 4851 4852 txsd = &txq->sdesc[eq->pidx]; 4853 txsd->m = m0; 4854 txsd->desc_used = ndesc; 4855 4856 return (ndesc); 4857 } 4858 4859 /* 4860 * Write a txpkt WR for this packet to the hardware descriptors, update the 4861 * software descriptor, and advance the pidx. It is guaranteed that enough 4862 * descriptors are available. 4863 * 4864 * The return value is the # of hardware descriptors used. 4865 */ 4866 static u_int 4867 write_txpkt_wr(struct adapter *sc, struct sge_txq *txq, struct mbuf *m0, 4868 u_int available) 4869 { 4870 struct sge_eq *eq; 4871 struct fw_eth_tx_pkt_wr *wr; 4872 struct tx_sdesc *txsd; 4873 struct cpl_tx_pkt_core *cpl; 4874 uint32_t ctrl; /* used in many unrelated places */ 4875 uint64_t ctrl1; 4876 int len16, ndesc, pktlen, nsegs; 4877 caddr_t dst; 4878 4879 TXQ_LOCK_ASSERT_OWNED(txq); 4880 M_ASSERTPKTHDR(m0); 4881 4882 len16 = mbuf_len16(m0); 4883 nsegs = mbuf_nsegs(m0); 4884 pktlen = m0->m_pkthdr.len; 4885 ctrl = sizeof(struct cpl_tx_pkt_core); 4886 if (needs_tso(m0)) 4887 ctrl += sizeof(struct cpl_tx_pkt_lso_core); 4888 else if (!(mbuf_cflags(m0) & MC_NOMAP) && pktlen <= imm_payload(2) && 4889 available >= 2) { 4890 /* Immediate data. Recalculate len16 and set nsegs to 0. */ 4891 ctrl += pktlen; 4892 len16 = howmany(sizeof(struct fw_eth_tx_pkt_wr) + 4893 sizeof(struct cpl_tx_pkt_core) + pktlen, 16); 4894 nsegs = 0; 4895 } 4896 ndesc = tx_len16_to_desc(len16); 4897 MPASS(ndesc <= available); 4898 4899 /* Firmware work request header */ 4900 eq = &txq->eq; 4901 wr = (void *)&eq->desc[eq->pidx]; 4902 wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) | 4903 V_FW_ETH_TX_PKT_WR_IMMDLEN(ctrl)); 4904 4905 ctrl = V_FW_WR_LEN16(len16); 4906 wr->equiq_to_len16 = htobe32(ctrl); 4907 wr->r3 = 0; 4908 4909 if (needs_tso(m0)) { 4910 struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1); 4911 4912 KASSERT(m0->m_pkthdr.l2hlen > 0 && m0->m_pkthdr.l3hlen > 0 && 4913 m0->m_pkthdr.l4hlen > 0, 4914 ("%s: mbuf %p needs TSO but missing header lengths", 4915 __func__, m0)); 4916 4917 ctrl = V_LSO_OPCODE(CPL_TX_PKT_LSO) | F_LSO_FIRST_SLICE | 4918 F_LSO_LAST_SLICE | V_LSO_ETHHDR_LEN((m0->m_pkthdr.l2hlen - 4919 ETHER_HDR_LEN) >> 2) | 4920 V_LSO_IPHDR_LEN(m0->m_pkthdr.l3hlen >> 2) | 4921 V_LSO_TCPHDR_LEN(m0->m_pkthdr.l4hlen >> 2); 4922 if (m0->m_pkthdr.l3hlen == sizeof(struct ip6_hdr)) 4923 ctrl |= F_LSO_IPV6; 4924 4925 lso->lso_ctrl = htobe32(ctrl); 4926 lso->ipid_ofst = htobe16(0); 4927 lso->mss = htobe16(m0->m_pkthdr.tso_segsz); 4928 lso->seqno_offset = htobe32(0); 4929 lso->len = htobe32(pktlen); 4930 4931 cpl = (void *)(lso + 1); 4932 4933 txq->tso_wrs++; 4934 } else 4935 cpl = (void *)(wr + 1); 4936 4937 /* Checksum offload */ 4938 ctrl1 = csum_to_ctrl(sc, m0); 4939 if (ctrl1 != (F_TXPKT_IPCSUM_DIS | F_TXPKT_L4CSUM_DIS)) 4940 txq->txcsum++; /* some hardware assistance provided */ 4941 4942 /* VLAN tag insertion */ 4943 if (needs_vlan_insertion(m0)) { 4944 ctrl1 |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN(m0->m_pkthdr.ether_vtag); 4945 txq->vlan_insertion++; 4946 } 4947 4948 /* CPL header */ 4949 cpl->ctrl0 = txq->cpl_ctrl0; 4950 cpl->pack = 0; 4951 cpl->len = htobe16(pktlen); 4952 cpl->ctrl1 = htobe64(ctrl1); 4953 4954 /* SGL */ 4955 dst = (void *)(cpl + 1); 4956 if (nsegs > 0) { 4957 4958 write_gl_to_txd(txq, m0, &dst, eq->sidx - ndesc < eq->pidx); 4959 txq->sgl_wrs++; 4960 } else { 4961 struct mbuf *m; 4962 4963 for (m = m0; m != NULL; m = m->m_next) { 4964 copy_to_txd(eq, mtod(m, caddr_t), &dst, m->m_len); 4965 #ifdef INVARIANTS 4966 pktlen -= m->m_len; 4967 #endif 4968 } 4969 #ifdef INVARIANTS 4970 KASSERT(pktlen == 0, ("%s: %d bytes left.", __func__, pktlen)); 4971 #endif 4972 txq->imm_wrs++; 4973 } 4974 4975 txq->txpkt_wrs++; 4976 4977 txsd = &txq->sdesc[eq->pidx]; 4978 txsd->m = m0; 4979 txsd->desc_used = ndesc; 4980 4981 return (ndesc); 4982 } 4983 4984 static inline bool 4985 cmp_l2hdr(struct txpkts *txp, struct mbuf *m) 4986 { 4987 int len; 4988 4989 MPASS(txp->npkt > 0); 4990 MPASS(m->m_len >= VM_TX_L2HDR_LEN); 4991 4992 if (txp->ethtype == be16toh(ETHERTYPE_VLAN)) 4993 len = VM_TX_L2HDR_LEN; 4994 else 4995 len = sizeof(struct ether_header); 4996 4997 return (memcmp(m->m_data, &txp->ethmacdst[0], len) != 0); 4998 } 4999 5000 static inline void 5001 save_l2hdr(struct txpkts *txp, struct mbuf *m) 5002 { 5003 MPASS(m->m_len >= VM_TX_L2HDR_LEN); 5004 5005 memcpy(&txp->ethmacdst[0], mtod(m, const void *), VM_TX_L2HDR_LEN); 5006 } 5007 5008 static int 5009 add_to_txpkts_vf(struct adapter *sc, struct sge_txq *txq, struct mbuf *m, 5010 int avail, bool *send) 5011 { 5012 struct txpkts *txp = &txq->txp; 5013 5014 MPASS(sc->flags & IS_VF); 5015 5016 /* Cannot have TSO and coalesce at the same time. */ 5017 if (cannot_use_txpkts(m)) { 5018 cannot_coalesce: 5019 *send = txp->npkt > 0; 5020 return (EINVAL); 5021 } 5022 5023 /* VF allows coalescing of type 1 (1 GL) only */ 5024 if (mbuf_nsegs(m) > 1) 5025 goto cannot_coalesce; 5026 5027 *send = false; 5028 if (txp->npkt > 0) { 5029 MPASS(tx_len16_to_desc(txp->len16) <= avail); 5030 MPASS(txp->npkt < txp->max_npkt); 5031 MPASS(txp->wr_type == 1); /* VF supports type 1 only */ 5032 5033 if (tx_len16_to_desc(txp->len16 + txpkts1_len16()) > avail) { 5034 retry_after_send: 5035 *send = true; 5036 return (EAGAIN); 5037 } 5038 if (m->m_pkthdr.len + txp->plen > 65535) 5039 goto retry_after_send; 5040 if (cmp_l2hdr(txp, m)) 5041 goto retry_after_send; 5042 5043 txp->len16 += txpkts1_len16(); 5044 txp->plen += m->m_pkthdr.len; 5045 txp->mb[txp->npkt++] = m; 5046 if (txp->npkt == txp->max_npkt) 5047 *send = true; 5048 } else { 5049 txp->len16 = howmany(sizeof(struct fw_eth_tx_pkts_vm_wr), 16) + 5050 txpkts1_len16(); 5051 if (tx_len16_to_desc(txp->len16) > avail) 5052 goto cannot_coalesce; 5053 txp->npkt = 1; 5054 txp->wr_type = 1; 5055 txp->plen = m->m_pkthdr.len; 5056 txp->mb[0] = m; 5057 save_l2hdr(txp, m); 5058 } 5059 return (0); 5060 } 5061 5062 static int 5063 add_to_txpkts_pf(struct adapter *sc, struct sge_txq *txq, struct mbuf *m, 5064 int avail, bool *send) 5065 { 5066 struct txpkts *txp = &txq->txp; 5067 int nsegs; 5068 5069 MPASS(!(sc->flags & IS_VF)); 5070 5071 /* Cannot have TSO and coalesce at the same time. */ 5072 if (cannot_use_txpkts(m)) { 5073 cannot_coalesce: 5074 *send = txp->npkt > 0; 5075 return (EINVAL); 5076 } 5077 5078 *send = false; 5079 nsegs = mbuf_nsegs(m); 5080 if (txp->npkt == 0) { 5081 if (m->m_pkthdr.len > 65535) 5082 goto cannot_coalesce; 5083 if (nsegs > 1) { 5084 txp->wr_type = 0; 5085 txp->len16 = 5086 howmany(sizeof(struct fw_eth_tx_pkts_wr), 16) + 5087 txpkts0_len16(nsegs); 5088 } else { 5089 txp->wr_type = 1; 5090 txp->len16 = 5091 howmany(sizeof(struct fw_eth_tx_pkts_wr), 16) + 5092 txpkts1_len16(); 5093 } 5094 if (tx_len16_to_desc(txp->len16) > avail) 5095 goto cannot_coalesce; 5096 txp->npkt = 1; 5097 txp->plen = m->m_pkthdr.len; 5098 txp->mb[0] = m; 5099 } else { 5100 MPASS(tx_len16_to_desc(txp->len16) <= avail); 5101 MPASS(txp->npkt < txp->max_npkt); 5102 5103 if (m->m_pkthdr.len + txp->plen > 65535) { 5104 retry_after_send: 5105 *send = true; 5106 return (EAGAIN); 5107 } 5108 5109 MPASS(txp->wr_type == 0 || txp->wr_type == 1); 5110 if (txp->wr_type == 0) { 5111 if (tx_len16_to_desc(txp->len16 + 5112 txpkts0_len16(nsegs)) > min(avail, SGE_MAX_WR_NDESC)) 5113 goto retry_after_send; 5114 txp->len16 += txpkts0_len16(nsegs); 5115 } else { 5116 if (nsegs != 1) 5117 goto retry_after_send; 5118 if (tx_len16_to_desc(txp->len16 + txpkts1_len16()) > 5119 avail) 5120 goto retry_after_send; 5121 txp->len16 += txpkts1_len16(); 5122 } 5123 5124 txp->plen += m->m_pkthdr.len; 5125 txp->mb[txp->npkt++] = m; 5126 if (txp->npkt == txp->max_npkt) 5127 *send = true; 5128 } 5129 return (0); 5130 } 5131 5132 /* 5133 * Write a txpkts WR for the packets in txp to the hardware descriptors, update 5134 * the software descriptor, and advance the pidx. It is guaranteed that enough 5135 * descriptors are available. 5136 * 5137 * The return value is the # of hardware descriptors used. 5138 */ 5139 static u_int 5140 write_txpkts_wr(struct adapter *sc, struct sge_txq *txq) 5141 { 5142 const struct txpkts *txp = &txq->txp; 5143 struct sge_eq *eq = &txq->eq; 5144 struct fw_eth_tx_pkts_wr *wr; 5145 struct tx_sdesc *txsd; 5146 struct cpl_tx_pkt_core *cpl; 5147 uint64_t ctrl1; 5148 int ndesc, i, checkwrap; 5149 struct mbuf *m, *last; 5150 void *flitp; 5151 5152 TXQ_LOCK_ASSERT_OWNED(txq); 5153 MPASS(txp->npkt > 0); 5154 MPASS(txp->len16 <= howmany(SGE_MAX_WR_LEN, 16)); 5155 5156 wr = (void *)&eq->desc[eq->pidx]; 5157 wr->op_pkd = htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR)); 5158 wr->equiq_to_len16 = htobe32(V_FW_WR_LEN16(txp->len16)); 5159 wr->plen = htobe16(txp->plen); 5160 wr->npkt = txp->npkt; 5161 wr->r3 = 0; 5162 wr->type = txp->wr_type; 5163 flitp = wr + 1; 5164 5165 /* 5166 * At this point we are 16B into a hardware descriptor. If checkwrap is 5167 * set then we know the WR is going to wrap around somewhere. We'll 5168 * check for that at appropriate points. 5169 */ 5170 ndesc = tx_len16_to_desc(txp->len16); 5171 last = NULL; 5172 checkwrap = eq->sidx - ndesc < eq->pidx; 5173 for (i = 0; i < txp->npkt; i++) { 5174 m = txp->mb[i]; 5175 if (txp->wr_type == 0) { 5176 struct ulp_txpkt *ulpmc; 5177 struct ulptx_idata *ulpsc; 5178 5179 /* ULP master command */ 5180 ulpmc = flitp; 5181 ulpmc->cmd_dest = htobe32(V_ULPTX_CMD(ULP_TX_PKT) | 5182 V_ULP_TXPKT_DEST(0) | V_ULP_TXPKT_FID(eq->iqid)); 5183 ulpmc->len = htobe32(txpkts0_len16(mbuf_nsegs(m))); 5184 5185 /* ULP subcommand */ 5186 ulpsc = (void *)(ulpmc + 1); 5187 ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM) | 5188 F_ULP_TX_SC_MORE); 5189 ulpsc->len = htobe32(sizeof(struct cpl_tx_pkt_core)); 5190 5191 cpl = (void *)(ulpsc + 1); 5192 if (checkwrap && 5193 (uintptr_t)cpl == (uintptr_t)&eq->desc[eq->sidx]) 5194 cpl = (void *)&eq->desc[0]; 5195 } else { 5196 cpl = flitp; 5197 } 5198 5199 /* Checksum offload */ 5200 ctrl1 = csum_to_ctrl(sc, m); 5201 if (ctrl1 != (F_TXPKT_IPCSUM_DIS | F_TXPKT_L4CSUM_DIS)) 5202 txq->txcsum++; /* some hardware assistance provided */ 5203 5204 /* VLAN tag insertion */ 5205 if (needs_vlan_insertion(m)) { 5206 ctrl1 |= F_TXPKT_VLAN_VLD | 5207 V_TXPKT_VLAN(m->m_pkthdr.ether_vtag); 5208 txq->vlan_insertion++; 5209 } 5210 5211 /* CPL header */ 5212 cpl->ctrl0 = txq->cpl_ctrl0; 5213 cpl->pack = 0; 5214 cpl->len = htobe16(m->m_pkthdr.len); 5215 cpl->ctrl1 = htobe64(ctrl1); 5216 5217 flitp = cpl + 1; 5218 if (checkwrap && 5219 (uintptr_t)flitp == (uintptr_t)&eq->desc[eq->sidx]) 5220 flitp = (void *)&eq->desc[0]; 5221 5222 write_gl_to_txd(txq, m, (caddr_t *)(&flitp), checkwrap); 5223 5224 if (last != NULL) 5225 last->m_nextpkt = m; 5226 last = m; 5227 } 5228 5229 txq->sgl_wrs++; 5230 if (txp->wr_type == 0) { 5231 txq->txpkts0_pkts += txp->npkt; 5232 txq->txpkts0_wrs++; 5233 } else { 5234 txq->txpkts1_pkts += txp->npkt; 5235 txq->txpkts1_wrs++; 5236 } 5237 5238 txsd = &txq->sdesc[eq->pidx]; 5239 txsd->m = txp->mb[0]; 5240 txsd->desc_used = ndesc; 5241 5242 return (ndesc); 5243 } 5244 5245 static u_int 5246 write_txpkts_vm_wr(struct adapter *sc, struct sge_txq *txq) 5247 { 5248 const struct txpkts *txp = &txq->txp; 5249 struct sge_eq *eq = &txq->eq; 5250 struct fw_eth_tx_pkts_vm_wr *wr; 5251 struct tx_sdesc *txsd; 5252 struct cpl_tx_pkt_core *cpl; 5253 uint64_t ctrl1; 5254 int ndesc, i; 5255 struct mbuf *m, *last; 5256 void *flitp; 5257 5258 TXQ_LOCK_ASSERT_OWNED(txq); 5259 MPASS(txp->npkt > 0); 5260 MPASS(txp->wr_type == 1); /* VF supports type 1 only */ 5261 MPASS(txp->mb[0] != NULL); 5262 MPASS(txp->len16 <= howmany(SGE_MAX_WR_LEN, 16)); 5263 5264 wr = (void *)&eq->desc[eq->pidx]; 5265 wr->op_pkd = htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS_VM_WR)); 5266 wr->equiq_to_len16 = htobe32(V_FW_WR_LEN16(txp->len16)); 5267 wr->r3 = 0; 5268 wr->plen = htobe16(txp->plen); 5269 wr->npkt = txp->npkt; 5270 wr->r4 = 0; 5271 memcpy(&wr->ethmacdst[0], &txp->ethmacdst[0], 16); 5272 flitp = wr + 1; 5273 5274 /* 5275 * At this point we are 32B into a hardware descriptor. Each mbuf in 5276 * the WR will take 32B so we check for the end of the descriptor ring 5277 * before writing odd mbufs (mb[1], 3, 5, ..) 5278 */ 5279 ndesc = tx_len16_to_desc(txp->len16); 5280 last = NULL; 5281 for (i = 0; i < txp->npkt; i++) { 5282 m = txp->mb[i]; 5283 if (i & 1 && (uintptr_t)flitp == (uintptr_t)&eq->desc[eq->sidx]) 5284 flitp = &eq->desc[0]; 5285 cpl = flitp; 5286 5287 /* Checksum offload */ 5288 ctrl1 = csum_to_ctrl(sc, m); 5289 if (ctrl1 != (F_TXPKT_IPCSUM_DIS | F_TXPKT_L4CSUM_DIS)) 5290 txq->txcsum++; /* some hardware assistance provided */ 5291 5292 /* VLAN tag insertion */ 5293 if (needs_vlan_insertion(m)) { 5294 ctrl1 |= F_TXPKT_VLAN_VLD | 5295 V_TXPKT_VLAN(m->m_pkthdr.ether_vtag); 5296 txq->vlan_insertion++; 5297 } 5298 5299 /* CPL header */ 5300 cpl->ctrl0 = txq->cpl_ctrl0; 5301 cpl->pack = 0; 5302 cpl->len = htobe16(m->m_pkthdr.len); 5303 cpl->ctrl1 = htobe64(ctrl1); 5304 5305 flitp = cpl + 1; 5306 MPASS(mbuf_nsegs(m) == 1); 5307 write_gl_to_txd(txq, m, (caddr_t *)(&flitp), 0); 5308 5309 if (last != NULL) 5310 last->m_nextpkt = m; 5311 last = m; 5312 } 5313 5314 txq->sgl_wrs++; 5315 txq->txpkts1_pkts += txp->npkt; 5316 txq->txpkts1_wrs++; 5317 5318 txsd = &txq->sdesc[eq->pidx]; 5319 txsd->m = txp->mb[0]; 5320 txsd->desc_used = ndesc; 5321 5322 return (ndesc); 5323 } 5324 5325 /* 5326 * If the SGL ends on an address that is not 16 byte aligned, this function will 5327 * add a 0 filled flit at the end. 5328 */ 5329 static void 5330 write_gl_to_txd(struct sge_txq *txq, struct mbuf *m, caddr_t *to, int checkwrap) 5331 { 5332 struct sge_eq *eq = &txq->eq; 5333 struct sglist *gl = txq->gl; 5334 struct sglist_seg *seg; 5335 __be64 *flitp, *wrap; 5336 struct ulptx_sgl *usgl; 5337 int i, nflits, nsegs; 5338 5339 KASSERT(((uintptr_t)(*to) & 0xf) == 0, 5340 ("%s: SGL must start at a 16 byte boundary: %p", __func__, *to)); 5341 MPASS((uintptr_t)(*to) >= (uintptr_t)&eq->desc[0]); 5342 MPASS((uintptr_t)(*to) < (uintptr_t)&eq->desc[eq->sidx]); 5343 5344 get_pkt_gl(m, gl); 5345 nsegs = gl->sg_nseg; 5346 MPASS(nsegs > 0); 5347 5348 nflits = (3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1) + 2; 5349 flitp = (__be64 *)(*to); 5350 wrap = (__be64 *)(&eq->desc[eq->sidx]); 5351 seg = &gl->sg_segs[0]; 5352 usgl = (void *)flitp; 5353 5354 /* 5355 * We start at a 16 byte boundary somewhere inside the tx descriptor 5356 * ring, so we're at least 16 bytes away from the status page. There is 5357 * no chance of a wrap around in the middle of usgl (which is 16 bytes). 5358 */ 5359 5360 usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | 5361 V_ULPTX_NSGE(nsegs)); 5362 usgl->len0 = htobe32(seg->ss_len); 5363 usgl->addr0 = htobe64(seg->ss_paddr); 5364 seg++; 5365 5366 if (checkwrap == 0 || (uintptr_t)(flitp + nflits) <= (uintptr_t)wrap) { 5367 5368 /* Won't wrap around at all */ 5369 5370 for (i = 0; i < nsegs - 1; i++, seg++) { 5371 usgl->sge[i / 2].len[i & 1] = htobe32(seg->ss_len); 5372 usgl->sge[i / 2].addr[i & 1] = htobe64(seg->ss_paddr); 5373 } 5374 if (i & 1) 5375 usgl->sge[i / 2].len[1] = htobe32(0); 5376 flitp += nflits; 5377 } else { 5378 5379 /* Will wrap somewhere in the rest of the SGL */ 5380 5381 /* 2 flits already written, write the rest flit by flit */ 5382 flitp = (void *)(usgl + 1); 5383 for (i = 0; i < nflits - 2; i++) { 5384 if (flitp == wrap) 5385 flitp = (void *)eq->desc; 5386 *flitp++ = get_flit(seg, nsegs - 1, i); 5387 } 5388 } 5389 5390 if (nflits & 1) { 5391 MPASS(((uintptr_t)flitp) & 0xf); 5392 *flitp++ = 0; 5393 } 5394 5395 MPASS((((uintptr_t)flitp) & 0xf) == 0); 5396 if (__predict_false(flitp == wrap)) 5397 *to = (void *)eq->desc; 5398 else 5399 *to = (void *)flitp; 5400 } 5401 5402 static inline void 5403 copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, int len) 5404 { 5405 5406 MPASS((uintptr_t)(*to) >= (uintptr_t)&eq->desc[0]); 5407 MPASS((uintptr_t)(*to) < (uintptr_t)&eq->desc[eq->sidx]); 5408 5409 if (__predict_true((uintptr_t)(*to) + len <= 5410 (uintptr_t)&eq->desc[eq->sidx])) { 5411 bcopy(from, *to, len); 5412 (*to) += len; 5413 } else { 5414 int portion = (uintptr_t)&eq->desc[eq->sidx] - (uintptr_t)(*to); 5415 5416 bcopy(from, *to, portion); 5417 from += portion; 5418 portion = len - portion; /* remaining */ 5419 bcopy(from, (void *)eq->desc, portion); 5420 (*to) = (caddr_t)eq->desc + portion; 5421 } 5422 } 5423 5424 static inline void 5425 ring_eq_db(struct adapter *sc, struct sge_eq *eq, u_int n) 5426 { 5427 u_int db; 5428 5429 MPASS(n > 0); 5430 5431 db = eq->doorbells; 5432 if (n > 1) 5433 clrbit(&db, DOORBELL_WCWR); 5434 wmb(); 5435 5436 switch (ffs(db) - 1) { 5437 case DOORBELL_UDB: 5438 *eq->udb = htole32(V_QID(eq->udb_qid) | V_PIDX(n)); 5439 break; 5440 5441 case DOORBELL_WCWR: { 5442 volatile uint64_t *dst, *src; 5443 int i; 5444 5445 /* 5446 * Queues whose 128B doorbell segment fits in the page do not 5447 * use relative qid (udb_qid is always 0). Only queues with 5448 * doorbell segments can do WCWR. 5449 */ 5450 KASSERT(eq->udb_qid == 0 && n == 1, 5451 ("%s: inappropriate doorbell (0x%x, %d, %d) for eq %p", 5452 __func__, eq->doorbells, n, eq->dbidx, eq)); 5453 5454 dst = (volatile void *)((uintptr_t)eq->udb + UDBS_WR_OFFSET - 5455 UDBS_DB_OFFSET); 5456 i = eq->dbidx; 5457 src = (void *)&eq->desc[i]; 5458 while (src != (void *)&eq->desc[i + 1]) 5459 *dst++ = *src++; 5460 wmb(); 5461 break; 5462 } 5463 5464 case DOORBELL_UDBWC: 5465 *eq->udb = htole32(V_QID(eq->udb_qid) | V_PIDX(n)); 5466 wmb(); 5467 break; 5468 5469 case DOORBELL_KDB: 5470 t4_write_reg(sc, sc->sge_kdoorbell_reg, 5471 V_QID(eq->cntxt_id) | V_PIDX(n)); 5472 break; 5473 } 5474 5475 IDXINCR(eq->dbidx, n, eq->sidx); 5476 } 5477 5478 static inline u_int 5479 reclaimable_tx_desc(struct sge_eq *eq) 5480 { 5481 uint16_t hw_cidx; 5482 5483 hw_cidx = read_hw_cidx(eq); 5484 return (IDXDIFF(hw_cidx, eq->cidx, eq->sidx)); 5485 } 5486 5487 static inline u_int 5488 total_available_tx_desc(struct sge_eq *eq) 5489 { 5490 uint16_t hw_cidx, pidx; 5491 5492 hw_cidx = read_hw_cidx(eq); 5493 pidx = eq->pidx; 5494 5495 if (pidx == hw_cidx) 5496 return (eq->sidx - 1); 5497 else 5498 return (IDXDIFF(hw_cidx, pidx, eq->sidx) - 1); 5499 } 5500 5501 static inline uint16_t 5502 read_hw_cidx(struct sge_eq *eq) 5503 { 5504 struct sge_qstat *spg = (void *)&eq->desc[eq->sidx]; 5505 uint16_t cidx = spg->cidx; /* stable snapshot */ 5506 5507 return (be16toh(cidx)); 5508 } 5509 5510 /* 5511 * Reclaim 'n' descriptors approximately. 5512 */ 5513 static u_int 5514 reclaim_tx_descs(struct sge_txq *txq, u_int n) 5515 { 5516 struct tx_sdesc *txsd; 5517 struct sge_eq *eq = &txq->eq; 5518 u_int can_reclaim, reclaimed; 5519 5520 TXQ_LOCK_ASSERT_OWNED(txq); 5521 MPASS(n > 0); 5522 5523 reclaimed = 0; 5524 can_reclaim = reclaimable_tx_desc(eq); 5525 while (can_reclaim && reclaimed < n) { 5526 int ndesc; 5527 struct mbuf *m, *nextpkt; 5528 5529 txsd = &txq->sdesc[eq->cidx]; 5530 ndesc = txsd->desc_used; 5531 5532 /* Firmware doesn't return "partial" credits. */ 5533 KASSERT(can_reclaim >= ndesc, 5534 ("%s: unexpected number of credits: %d, %d", 5535 __func__, can_reclaim, ndesc)); 5536 KASSERT(ndesc != 0, 5537 ("%s: descriptor with no credits: cidx %d", 5538 __func__, eq->cidx)); 5539 5540 for (m = txsd->m; m != NULL; m = nextpkt) { 5541 nextpkt = m->m_nextpkt; 5542 m->m_nextpkt = NULL; 5543 m_freem(m); 5544 } 5545 reclaimed += ndesc; 5546 can_reclaim -= ndesc; 5547 IDXINCR(eq->cidx, ndesc, eq->sidx); 5548 } 5549 5550 return (reclaimed); 5551 } 5552 5553 static void 5554 tx_reclaim(void *arg, int n) 5555 { 5556 struct sge_txq *txq = arg; 5557 struct sge_eq *eq = &txq->eq; 5558 5559 do { 5560 if (TXQ_TRYLOCK(txq) == 0) 5561 break; 5562 n = reclaim_tx_descs(txq, 32); 5563 if (eq->cidx == eq->pidx) 5564 eq->equeqidx = eq->pidx; 5565 TXQ_UNLOCK(txq); 5566 } while (n > 0); 5567 } 5568 5569 static __be64 5570 get_flit(struct sglist_seg *segs, int nsegs, int idx) 5571 { 5572 int i = (idx / 3) * 2; 5573 5574 switch (idx % 3) { 5575 case 0: { 5576 uint64_t rc; 5577 5578 rc = (uint64_t)segs[i].ss_len << 32; 5579 if (i + 1 < nsegs) 5580 rc |= (uint64_t)(segs[i + 1].ss_len); 5581 5582 return (htobe64(rc)); 5583 } 5584 case 1: 5585 return (htobe64(segs[i].ss_paddr)); 5586 case 2: 5587 return (htobe64(segs[i + 1].ss_paddr)); 5588 } 5589 5590 return (0); 5591 } 5592 5593 static int 5594 find_refill_source(struct adapter *sc, int maxp, bool packing) 5595 { 5596 int i, zidx = -1; 5597 struct rx_buf_info *rxb = &sc->sge.rx_buf_info[0]; 5598 5599 if (packing) { 5600 for (i = 0; i < SW_ZONE_SIZES; i++, rxb++) { 5601 if (rxb->hwidx2 == -1) 5602 continue; 5603 if (rxb->size1 < PAGE_SIZE && 5604 rxb->size1 < largest_rx_cluster) 5605 continue; 5606 if (rxb->size1 > largest_rx_cluster) 5607 break; 5608 MPASS(rxb->size1 - rxb->size2 >= CL_METADATA_SIZE); 5609 if (rxb->size2 >= maxp) 5610 return (i); 5611 zidx = i; 5612 } 5613 } else { 5614 for (i = 0; i < SW_ZONE_SIZES; i++, rxb++) { 5615 if (rxb->hwidx1 == -1) 5616 continue; 5617 if (rxb->size1 > largest_rx_cluster) 5618 break; 5619 if (rxb->size1 >= maxp) 5620 return (i); 5621 zidx = i; 5622 } 5623 } 5624 5625 return (zidx); 5626 } 5627 5628 static void 5629 add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl) 5630 { 5631 mtx_lock(&sc->sfl_lock); 5632 FL_LOCK(fl); 5633 if ((fl->flags & FL_DOOMED) == 0) { 5634 fl->flags |= FL_STARVING; 5635 TAILQ_INSERT_TAIL(&sc->sfl, fl, link); 5636 callout_reset(&sc->sfl_callout, hz / 5, refill_sfl, sc); 5637 } 5638 FL_UNLOCK(fl); 5639 mtx_unlock(&sc->sfl_lock); 5640 } 5641 5642 static void 5643 handle_wrq_egr_update(struct adapter *sc, struct sge_eq *eq) 5644 { 5645 struct sge_wrq *wrq = (void *)eq; 5646 5647 atomic_readandclear_int(&eq->equiq); 5648 taskqueue_enqueue(sc->tq[eq->tx_chan], &wrq->wrq_tx_task); 5649 } 5650 5651 static void 5652 handle_eth_egr_update(struct adapter *sc, struct sge_eq *eq) 5653 { 5654 struct sge_txq *txq = (void *)eq; 5655 5656 MPASS((eq->flags & EQ_TYPEMASK) == EQ_ETH); 5657 5658 atomic_readandclear_int(&eq->equiq); 5659 if (mp_ring_is_idle(txq->r)) 5660 taskqueue_enqueue(sc->tq[eq->tx_chan], &txq->tx_reclaim_task); 5661 else 5662 mp_ring_check_drainage(txq->r, 64); 5663 } 5664 5665 static int 5666 handle_sge_egr_update(struct sge_iq *iq, const struct rss_header *rss, 5667 struct mbuf *m) 5668 { 5669 const struct cpl_sge_egr_update *cpl = (const void *)(rss + 1); 5670 unsigned int qid = G_EGR_QID(ntohl(cpl->opcode_qid)); 5671 struct adapter *sc = iq->adapter; 5672 struct sge *s = &sc->sge; 5673 struct sge_eq *eq; 5674 static void (*h[])(struct adapter *, struct sge_eq *) = {NULL, 5675 &handle_wrq_egr_update, &handle_eth_egr_update, 5676 &handle_wrq_egr_update}; 5677 5678 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 5679 rss->opcode)); 5680 5681 eq = s->eqmap[qid - s->eq_start - s->eq_base]; 5682 (*h[eq->flags & EQ_TYPEMASK])(sc, eq); 5683 5684 return (0); 5685 } 5686 5687 /* handle_fw_msg works for both fw4_msg and fw6_msg because this is valid */ 5688 CTASSERT(offsetof(struct cpl_fw4_msg, data) == \ 5689 offsetof(struct cpl_fw6_msg, data)); 5690 5691 static int 5692 handle_fw_msg(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 5693 { 5694 struct adapter *sc = iq->adapter; 5695 const struct cpl_fw6_msg *cpl = (const void *)(rss + 1); 5696 5697 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 5698 rss->opcode)); 5699 5700 if (cpl->type == FW_TYPE_RSSCPL || cpl->type == FW6_TYPE_RSSCPL) { 5701 const struct rss_header *rss2; 5702 5703 rss2 = (const struct rss_header *)&cpl->data[0]; 5704 return (t4_cpl_handler[rss2->opcode](iq, rss2, m)); 5705 } 5706 5707 return (t4_fw_msg_handler[cpl->type](sc, &cpl->data[0])); 5708 } 5709 5710 /** 5711 * t4_handle_wrerr_rpl - process a FW work request error message 5712 * @adap: the adapter 5713 * @rpl: start of the FW message 5714 */ 5715 static int 5716 t4_handle_wrerr_rpl(struct adapter *adap, const __be64 *rpl) 5717 { 5718 u8 opcode = *(const u8 *)rpl; 5719 const struct fw_error_cmd *e = (const void *)rpl; 5720 unsigned int i; 5721 5722 if (opcode != FW_ERROR_CMD) { 5723 log(LOG_ERR, 5724 "%s: Received WRERR_RPL message with opcode %#x\n", 5725 device_get_nameunit(adap->dev), opcode); 5726 return (EINVAL); 5727 } 5728 log(LOG_ERR, "%s: FW_ERROR (%s) ", device_get_nameunit(adap->dev), 5729 G_FW_ERROR_CMD_FATAL(be32toh(e->op_to_type)) ? "fatal" : 5730 "non-fatal"); 5731 switch (G_FW_ERROR_CMD_TYPE(be32toh(e->op_to_type))) { 5732 case FW_ERROR_TYPE_EXCEPTION: 5733 log(LOG_ERR, "exception info:\n"); 5734 for (i = 0; i < nitems(e->u.exception.info); i++) 5735 log(LOG_ERR, "%s%08x", i == 0 ? "\t" : " ", 5736 be32toh(e->u.exception.info[i])); 5737 log(LOG_ERR, "\n"); 5738 break; 5739 case FW_ERROR_TYPE_HWMODULE: 5740 log(LOG_ERR, "HW module regaddr %08x regval %08x\n", 5741 be32toh(e->u.hwmodule.regaddr), 5742 be32toh(e->u.hwmodule.regval)); 5743 break; 5744 case FW_ERROR_TYPE_WR: 5745 log(LOG_ERR, "WR cidx %d PF %d VF %d eqid %d hdr:\n", 5746 be16toh(e->u.wr.cidx), 5747 G_FW_ERROR_CMD_PFN(be16toh(e->u.wr.pfn_vfn)), 5748 G_FW_ERROR_CMD_VFN(be16toh(e->u.wr.pfn_vfn)), 5749 be32toh(e->u.wr.eqid)); 5750 for (i = 0; i < nitems(e->u.wr.wrhdr); i++) 5751 log(LOG_ERR, "%s%02x", i == 0 ? "\t" : " ", 5752 e->u.wr.wrhdr[i]); 5753 log(LOG_ERR, "\n"); 5754 break; 5755 case FW_ERROR_TYPE_ACL: 5756 log(LOG_ERR, "ACL cidx %d PF %d VF %d eqid %d %s", 5757 be16toh(e->u.acl.cidx), 5758 G_FW_ERROR_CMD_PFN(be16toh(e->u.acl.pfn_vfn)), 5759 G_FW_ERROR_CMD_VFN(be16toh(e->u.acl.pfn_vfn)), 5760 be32toh(e->u.acl.eqid), 5761 G_FW_ERROR_CMD_MV(be16toh(e->u.acl.mv_pkd)) ? "vlanid" : 5762 "MAC"); 5763 for (i = 0; i < nitems(e->u.acl.val); i++) 5764 log(LOG_ERR, " %02x", e->u.acl.val[i]); 5765 log(LOG_ERR, "\n"); 5766 break; 5767 default: 5768 log(LOG_ERR, "type %#x\n", 5769 G_FW_ERROR_CMD_TYPE(be32toh(e->op_to_type))); 5770 return (EINVAL); 5771 } 5772 return (0); 5773 } 5774 5775 static int 5776 sysctl_uint16(SYSCTL_HANDLER_ARGS) 5777 { 5778 uint16_t *id = arg1; 5779 int i = *id; 5780 5781 return sysctl_handle_int(oidp, &i, 0, req); 5782 } 5783 5784 static inline bool 5785 bufidx_used(struct adapter *sc, int idx) 5786 { 5787 struct rx_buf_info *rxb = &sc->sge.rx_buf_info[0]; 5788 int i; 5789 5790 for (i = 0; i < SW_ZONE_SIZES; i++, rxb++) { 5791 if (rxb->size1 > largest_rx_cluster) 5792 continue; 5793 if (rxb->hwidx1 == idx || rxb->hwidx2 == idx) 5794 return (true); 5795 } 5796 5797 return (false); 5798 } 5799 5800 static int 5801 sysctl_bufsizes(SYSCTL_HANDLER_ARGS) 5802 { 5803 struct adapter *sc = arg1; 5804 struct sge_params *sp = &sc->params.sge; 5805 int i, rc; 5806 struct sbuf sb; 5807 char c; 5808 5809 sbuf_new(&sb, NULL, 128, SBUF_AUTOEXTEND); 5810 for (i = 0; i < SGE_FLBUF_SIZES; i++) { 5811 if (bufidx_used(sc, i)) 5812 c = '*'; 5813 else 5814 c = '\0'; 5815 5816 sbuf_printf(&sb, "%u%c ", sp->sge_fl_buffer_size[i], c); 5817 } 5818 sbuf_trim(&sb); 5819 sbuf_finish(&sb); 5820 rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); 5821 sbuf_delete(&sb); 5822 return (rc); 5823 } 5824 5825 #ifdef RATELIMIT 5826 /* 5827 * len16 for a txpkt WR with a GL. Includes the firmware work request header. 5828 */ 5829 static inline u_int 5830 txpkt_eo_len16(u_int nsegs, u_int immhdrs, u_int tso) 5831 { 5832 u_int n; 5833 5834 MPASS(immhdrs > 0); 5835 5836 n = roundup2(sizeof(struct fw_eth_tx_eo_wr) + 5837 sizeof(struct cpl_tx_pkt_core) + immhdrs, 16); 5838 if (__predict_false(nsegs == 0)) 5839 goto done; 5840 5841 nsegs--; /* first segment is part of ulptx_sgl */ 5842 n += sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1)); 5843 if (tso) 5844 n += sizeof(struct cpl_tx_pkt_lso_core); 5845 5846 done: 5847 return (howmany(n, 16)); 5848 } 5849 5850 #define ETID_FLOWC_NPARAMS 6 5851 #define ETID_FLOWC_LEN (roundup2((sizeof(struct fw_flowc_wr) + \ 5852 ETID_FLOWC_NPARAMS * sizeof(struct fw_flowc_mnemval)), 16)) 5853 #define ETID_FLOWC_LEN16 (howmany(ETID_FLOWC_LEN, 16)) 5854 5855 static int 5856 send_etid_flowc_wr(struct cxgbe_rate_tag *cst, struct port_info *pi, 5857 struct vi_info *vi) 5858 { 5859 struct wrq_cookie cookie; 5860 u_int pfvf = pi->adapter->pf << S_FW_VIID_PFN; 5861 struct fw_flowc_wr *flowc; 5862 5863 mtx_assert(&cst->lock, MA_OWNED); 5864 MPASS((cst->flags & (EO_FLOWC_PENDING | EO_FLOWC_RPL_PENDING)) == 5865 EO_FLOWC_PENDING); 5866 5867 flowc = start_wrq_wr(cst->eo_txq, ETID_FLOWC_LEN16, &cookie); 5868 if (__predict_false(flowc == NULL)) 5869 return (ENOMEM); 5870 5871 bzero(flowc, ETID_FLOWC_LEN); 5872 flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) | 5873 V_FW_FLOWC_WR_NPARAMS(ETID_FLOWC_NPARAMS) | V_FW_WR_COMPL(0)); 5874 flowc->flowid_len16 = htonl(V_FW_WR_LEN16(ETID_FLOWC_LEN16) | 5875 V_FW_WR_FLOWID(cst->etid)); 5876 flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN; 5877 flowc->mnemval[0].val = htobe32(pfvf); 5878 flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH; 5879 flowc->mnemval[1].val = htobe32(pi->tx_chan); 5880 flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT; 5881 flowc->mnemval[2].val = htobe32(pi->tx_chan); 5882 flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID; 5883 flowc->mnemval[3].val = htobe32(cst->iqid); 5884 flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_EOSTATE; 5885 flowc->mnemval[4].val = htobe32(FW_FLOWC_MNEM_EOSTATE_ESTABLISHED); 5886 flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_SCHEDCLASS; 5887 flowc->mnemval[5].val = htobe32(cst->schedcl); 5888 5889 commit_wrq_wr(cst->eo_txq, flowc, &cookie); 5890 5891 cst->flags &= ~EO_FLOWC_PENDING; 5892 cst->flags |= EO_FLOWC_RPL_PENDING; 5893 MPASS(cst->tx_credits >= ETID_FLOWC_LEN16); /* flowc is first WR. */ 5894 cst->tx_credits -= ETID_FLOWC_LEN16; 5895 5896 return (0); 5897 } 5898 5899 #define ETID_FLUSH_LEN16 (howmany(sizeof (struct fw_flowc_wr), 16)) 5900 5901 void 5902 send_etid_flush_wr(struct cxgbe_rate_tag *cst) 5903 { 5904 struct fw_flowc_wr *flowc; 5905 struct wrq_cookie cookie; 5906 5907 mtx_assert(&cst->lock, MA_OWNED); 5908 5909 flowc = start_wrq_wr(cst->eo_txq, ETID_FLUSH_LEN16, &cookie); 5910 if (__predict_false(flowc == NULL)) 5911 CXGBE_UNIMPLEMENTED(__func__); 5912 5913 bzero(flowc, ETID_FLUSH_LEN16 * 16); 5914 flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) | 5915 V_FW_FLOWC_WR_NPARAMS(0) | F_FW_WR_COMPL); 5916 flowc->flowid_len16 = htobe32(V_FW_WR_LEN16(ETID_FLUSH_LEN16) | 5917 V_FW_WR_FLOWID(cst->etid)); 5918 5919 commit_wrq_wr(cst->eo_txq, flowc, &cookie); 5920 5921 cst->flags |= EO_FLUSH_RPL_PENDING; 5922 MPASS(cst->tx_credits >= ETID_FLUSH_LEN16); 5923 cst->tx_credits -= ETID_FLUSH_LEN16; 5924 cst->ncompl++; 5925 } 5926 5927 static void 5928 write_ethofld_wr(struct cxgbe_rate_tag *cst, struct fw_eth_tx_eo_wr *wr, 5929 struct mbuf *m0, int compl) 5930 { 5931 struct cpl_tx_pkt_core *cpl; 5932 uint64_t ctrl1; 5933 uint32_t ctrl; /* used in many unrelated places */ 5934 int len16, pktlen, nsegs, immhdrs; 5935 caddr_t dst; 5936 uintptr_t p; 5937 struct ulptx_sgl *usgl; 5938 struct sglist sg; 5939 struct sglist_seg segs[38]; /* XXX: find real limit. XXX: get off the stack */ 5940 5941 mtx_assert(&cst->lock, MA_OWNED); 5942 M_ASSERTPKTHDR(m0); 5943 KASSERT(m0->m_pkthdr.l2hlen > 0 && m0->m_pkthdr.l3hlen > 0 && 5944 m0->m_pkthdr.l4hlen > 0, 5945 ("%s: ethofld mbuf %p is missing header lengths", __func__, m0)); 5946 5947 len16 = mbuf_eo_len16(m0); 5948 nsegs = mbuf_eo_nsegs(m0); 5949 pktlen = m0->m_pkthdr.len; 5950 ctrl = sizeof(struct cpl_tx_pkt_core); 5951 if (needs_tso(m0)) 5952 ctrl += sizeof(struct cpl_tx_pkt_lso_core); 5953 immhdrs = m0->m_pkthdr.l2hlen + m0->m_pkthdr.l3hlen + m0->m_pkthdr.l4hlen; 5954 ctrl += immhdrs; 5955 5956 wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_EO_WR) | 5957 V_FW_ETH_TX_EO_WR_IMMDLEN(ctrl) | V_FW_WR_COMPL(!!compl)); 5958 wr->equiq_to_len16 = htobe32(V_FW_WR_LEN16(len16) | 5959 V_FW_WR_FLOWID(cst->etid)); 5960 wr->r3 = 0; 5961 if (needs_udp_csum(m0)) { 5962 wr->u.udpseg.type = FW_ETH_TX_EO_TYPE_UDPSEG; 5963 wr->u.udpseg.ethlen = m0->m_pkthdr.l2hlen; 5964 wr->u.udpseg.iplen = htobe16(m0->m_pkthdr.l3hlen); 5965 wr->u.udpseg.udplen = m0->m_pkthdr.l4hlen; 5966 wr->u.udpseg.rtplen = 0; 5967 wr->u.udpseg.r4 = 0; 5968 wr->u.udpseg.mss = htobe16(pktlen - immhdrs); 5969 wr->u.udpseg.schedpktsize = wr->u.udpseg.mss; 5970 wr->u.udpseg.plen = htobe32(pktlen - immhdrs); 5971 cpl = (void *)(wr + 1); 5972 } else { 5973 MPASS(needs_tcp_csum(m0)); 5974 wr->u.tcpseg.type = FW_ETH_TX_EO_TYPE_TCPSEG; 5975 wr->u.tcpseg.ethlen = m0->m_pkthdr.l2hlen; 5976 wr->u.tcpseg.iplen = htobe16(m0->m_pkthdr.l3hlen); 5977 wr->u.tcpseg.tcplen = m0->m_pkthdr.l4hlen; 5978 wr->u.tcpseg.tsclk_tsoff = mbuf_eo_tsclk_tsoff(m0); 5979 wr->u.tcpseg.r4 = 0; 5980 wr->u.tcpseg.r5 = 0; 5981 wr->u.tcpseg.plen = htobe32(pktlen - immhdrs); 5982 5983 if (needs_tso(m0)) { 5984 struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1); 5985 5986 wr->u.tcpseg.mss = htobe16(m0->m_pkthdr.tso_segsz); 5987 5988 ctrl = V_LSO_OPCODE(CPL_TX_PKT_LSO) | 5989 F_LSO_FIRST_SLICE | F_LSO_LAST_SLICE | 5990 V_LSO_ETHHDR_LEN((m0->m_pkthdr.l2hlen - 5991 ETHER_HDR_LEN) >> 2) | 5992 V_LSO_IPHDR_LEN(m0->m_pkthdr.l3hlen >> 2) | 5993 V_LSO_TCPHDR_LEN(m0->m_pkthdr.l4hlen >> 2); 5994 if (m0->m_pkthdr.l3hlen == sizeof(struct ip6_hdr)) 5995 ctrl |= F_LSO_IPV6; 5996 lso->lso_ctrl = htobe32(ctrl); 5997 lso->ipid_ofst = htobe16(0); 5998 lso->mss = htobe16(m0->m_pkthdr.tso_segsz); 5999 lso->seqno_offset = htobe32(0); 6000 lso->len = htobe32(pktlen); 6001 6002 cpl = (void *)(lso + 1); 6003 } else { 6004 wr->u.tcpseg.mss = htobe16(0xffff); 6005 cpl = (void *)(wr + 1); 6006 } 6007 } 6008 6009 /* Checksum offload must be requested for ethofld. */ 6010 MPASS(needs_l4_csum(m0)); 6011 ctrl1 = csum_to_ctrl(cst->adapter, m0); 6012 6013 /* VLAN tag insertion */ 6014 if (needs_vlan_insertion(m0)) { 6015 ctrl1 |= F_TXPKT_VLAN_VLD | 6016 V_TXPKT_VLAN(m0->m_pkthdr.ether_vtag); 6017 } 6018 6019 /* CPL header */ 6020 cpl->ctrl0 = cst->ctrl0; 6021 cpl->pack = 0; 6022 cpl->len = htobe16(pktlen); 6023 cpl->ctrl1 = htobe64(ctrl1); 6024 6025 /* Copy Ethernet, IP & TCP/UDP hdrs as immediate data */ 6026 p = (uintptr_t)(cpl + 1); 6027 m_copydata(m0, 0, immhdrs, (void *)p); 6028 6029 /* SGL */ 6030 dst = (void *)(cpl + 1); 6031 if (nsegs > 0) { 6032 int i, pad; 6033 6034 /* zero-pad upto next 16Byte boundary, if not 16Byte aligned */ 6035 p += immhdrs; 6036 pad = 16 - (immhdrs & 0xf); 6037 bzero((void *)p, pad); 6038 6039 usgl = (void *)(p + pad); 6040 usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | 6041 V_ULPTX_NSGE(nsegs)); 6042 6043 sglist_init(&sg, nitems(segs), segs); 6044 for (; m0 != NULL; m0 = m0->m_next) { 6045 if (__predict_false(m0->m_len == 0)) 6046 continue; 6047 if (immhdrs >= m0->m_len) { 6048 immhdrs -= m0->m_len; 6049 continue; 6050 } 6051 if (m0->m_flags & M_EXTPG) 6052 sglist_append_mbuf_epg(&sg, m0, 6053 mtod(m0, vm_offset_t), m0->m_len); 6054 else 6055 sglist_append(&sg, mtod(m0, char *) + immhdrs, 6056 m0->m_len - immhdrs); 6057 immhdrs = 0; 6058 } 6059 MPASS(sg.sg_nseg == nsegs); 6060 6061 /* 6062 * Zero pad last 8B in case the WR doesn't end on a 16B 6063 * boundary. 6064 */ 6065 *(uint64_t *)((char *)wr + len16 * 16 - 8) = 0; 6066 6067 usgl->len0 = htobe32(segs[0].ss_len); 6068 usgl->addr0 = htobe64(segs[0].ss_paddr); 6069 for (i = 0; i < nsegs - 1; i++) { 6070 usgl->sge[i / 2].len[i & 1] = htobe32(segs[i + 1].ss_len); 6071 usgl->sge[i / 2].addr[i & 1] = htobe64(segs[i + 1].ss_paddr); 6072 } 6073 if (i & 1) 6074 usgl->sge[i / 2].len[1] = htobe32(0); 6075 } 6076 6077 } 6078 6079 static void 6080 ethofld_tx(struct cxgbe_rate_tag *cst) 6081 { 6082 struct mbuf *m; 6083 struct wrq_cookie cookie; 6084 int next_credits, compl; 6085 struct fw_eth_tx_eo_wr *wr; 6086 6087 mtx_assert(&cst->lock, MA_OWNED); 6088 6089 while ((m = mbufq_first(&cst->pending_tx)) != NULL) { 6090 M_ASSERTPKTHDR(m); 6091 6092 /* How many len16 credits do we need to send this mbuf. */ 6093 next_credits = mbuf_eo_len16(m); 6094 MPASS(next_credits > 0); 6095 if (next_credits > cst->tx_credits) { 6096 /* 6097 * Tx will make progress eventually because there is at 6098 * least one outstanding fw4_ack that will return 6099 * credits and kick the tx. 6100 */ 6101 MPASS(cst->ncompl > 0); 6102 return; 6103 } 6104 wr = start_wrq_wr(cst->eo_txq, next_credits, &cookie); 6105 if (__predict_false(wr == NULL)) { 6106 /* XXX: wishful thinking, not a real assertion. */ 6107 MPASS(cst->ncompl > 0); 6108 return; 6109 } 6110 cst->tx_credits -= next_credits; 6111 cst->tx_nocompl += next_credits; 6112 compl = cst->ncompl == 0 || cst->tx_nocompl >= cst->tx_total / 2; 6113 ETHER_BPF_MTAP(cst->com.com.ifp, m); 6114 write_ethofld_wr(cst, wr, m, compl); 6115 commit_wrq_wr(cst->eo_txq, wr, &cookie); 6116 if (compl) { 6117 cst->ncompl++; 6118 cst->tx_nocompl = 0; 6119 } 6120 (void) mbufq_dequeue(&cst->pending_tx); 6121 6122 /* 6123 * Drop the mbuf's reference on the tag now rather 6124 * than waiting until m_freem(). This ensures that 6125 * cxgbe_rate_tag_free gets called when the inp drops 6126 * its reference on the tag and there are no more 6127 * mbufs in the pending_tx queue and can flush any 6128 * pending requests. Otherwise if the last mbuf 6129 * doesn't request a completion the etid will never be 6130 * released. 6131 */ 6132 m->m_pkthdr.snd_tag = NULL; 6133 m->m_pkthdr.csum_flags &= ~CSUM_SND_TAG; 6134 m_snd_tag_rele(&cst->com.com); 6135 6136 mbufq_enqueue(&cst->pending_fwack, m); 6137 } 6138 } 6139 6140 int 6141 ethofld_transmit(struct ifnet *ifp, struct mbuf *m0) 6142 { 6143 struct cxgbe_rate_tag *cst; 6144 int rc; 6145 6146 MPASS(m0->m_nextpkt == NULL); 6147 MPASS(m0->m_pkthdr.csum_flags & CSUM_SND_TAG); 6148 MPASS(m0->m_pkthdr.snd_tag != NULL); 6149 cst = mst_to_crt(m0->m_pkthdr.snd_tag); 6150 6151 mtx_lock(&cst->lock); 6152 MPASS(cst->flags & EO_SND_TAG_REF); 6153 6154 if (__predict_false(cst->flags & EO_FLOWC_PENDING)) { 6155 struct vi_info *vi = ifp->if_softc; 6156 struct port_info *pi = vi->pi; 6157 struct adapter *sc = pi->adapter; 6158 const uint32_t rss_mask = vi->rss_size - 1; 6159 uint32_t rss_hash; 6160 6161 cst->eo_txq = &sc->sge.ofld_txq[vi->first_ofld_txq]; 6162 if (M_HASHTYPE_ISHASH(m0)) 6163 rss_hash = m0->m_pkthdr.flowid; 6164 else 6165 rss_hash = arc4random(); 6166 /* We assume RSS hashing */ 6167 cst->iqid = vi->rss[rss_hash & rss_mask]; 6168 cst->eo_txq += rss_hash % vi->nofldtxq; 6169 rc = send_etid_flowc_wr(cst, pi, vi); 6170 if (rc != 0) 6171 goto done; 6172 } 6173 6174 if (__predict_false(cst->plen + m0->m_pkthdr.len > eo_max_backlog)) { 6175 rc = ENOBUFS; 6176 goto done; 6177 } 6178 6179 mbufq_enqueue(&cst->pending_tx, m0); 6180 cst->plen += m0->m_pkthdr.len; 6181 6182 /* 6183 * Hold an extra reference on the tag while generating work 6184 * requests to ensure that we don't try to free the tag during 6185 * ethofld_tx() in case we are sending the final mbuf after 6186 * the inp was freed. 6187 */ 6188 m_snd_tag_ref(&cst->com.com); 6189 ethofld_tx(cst); 6190 mtx_unlock(&cst->lock); 6191 m_snd_tag_rele(&cst->com.com); 6192 return (0); 6193 6194 done: 6195 mtx_unlock(&cst->lock); 6196 if (__predict_false(rc != 0)) 6197 m_freem(m0); 6198 return (rc); 6199 } 6200 6201 static int 6202 ethofld_fw4_ack(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m0) 6203 { 6204 struct adapter *sc = iq->adapter; 6205 const struct cpl_fw4_ack *cpl = (const void *)(rss + 1); 6206 struct mbuf *m; 6207 u_int etid = G_CPL_FW4_ACK_FLOWID(be32toh(OPCODE_TID(cpl))); 6208 struct cxgbe_rate_tag *cst; 6209 uint8_t credits = cpl->credits; 6210 6211 cst = lookup_etid(sc, etid); 6212 mtx_lock(&cst->lock); 6213 if (__predict_false(cst->flags & EO_FLOWC_RPL_PENDING)) { 6214 MPASS(credits >= ETID_FLOWC_LEN16); 6215 credits -= ETID_FLOWC_LEN16; 6216 cst->flags &= ~EO_FLOWC_RPL_PENDING; 6217 } 6218 6219 KASSERT(cst->ncompl > 0, 6220 ("%s: etid %u (%p) wasn't expecting completion.", 6221 __func__, etid, cst)); 6222 cst->ncompl--; 6223 6224 while (credits > 0) { 6225 m = mbufq_dequeue(&cst->pending_fwack); 6226 if (__predict_false(m == NULL)) { 6227 /* 6228 * The remaining credits are for the final flush that 6229 * was issued when the tag was freed by the kernel. 6230 */ 6231 MPASS((cst->flags & 6232 (EO_FLUSH_RPL_PENDING | EO_SND_TAG_REF)) == 6233 EO_FLUSH_RPL_PENDING); 6234 MPASS(credits == ETID_FLUSH_LEN16); 6235 MPASS(cst->tx_credits + cpl->credits == cst->tx_total); 6236 MPASS(cst->ncompl == 0); 6237 6238 cst->flags &= ~EO_FLUSH_RPL_PENDING; 6239 cst->tx_credits += cpl->credits; 6240 cxgbe_rate_tag_free_locked(cst); 6241 return (0); /* cst is gone. */ 6242 } 6243 KASSERT(m != NULL, 6244 ("%s: too many credits (%u, %u)", __func__, cpl->credits, 6245 credits)); 6246 KASSERT(credits >= mbuf_eo_len16(m), 6247 ("%s: too few credits (%u, %u, %u)", __func__, 6248 cpl->credits, credits, mbuf_eo_len16(m))); 6249 credits -= mbuf_eo_len16(m); 6250 cst->plen -= m->m_pkthdr.len; 6251 m_freem(m); 6252 } 6253 6254 cst->tx_credits += cpl->credits; 6255 MPASS(cst->tx_credits <= cst->tx_total); 6256 6257 if (cst->flags & EO_SND_TAG_REF) { 6258 /* 6259 * As with ethofld_transmit(), hold an extra reference 6260 * so that the tag is stable across ethold_tx(). 6261 */ 6262 m_snd_tag_ref(&cst->com.com); 6263 m = mbufq_first(&cst->pending_tx); 6264 if (m != NULL && cst->tx_credits >= mbuf_eo_len16(m)) 6265 ethofld_tx(cst); 6266 mtx_unlock(&cst->lock); 6267 m_snd_tag_rele(&cst->com.com); 6268 } else { 6269 /* 6270 * There shouldn't be any pending packets if the tag 6271 * was freed by the kernel since any pending packet 6272 * should hold a reference to the tag. 6273 */ 6274 MPASS(mbufq_first(&cst->pending_tx) == NULL); 6275 mtx_unlock(&cst->lock); 6276 } 6277 6278 return (0); 6279 } 6280 #endif 6281