1 /*- 2 * Copyright (c) 2015-2021 Mellanox Technologies. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 * 25 * $FreeBSD$ 26 */ 27 28 #include "opt_rss.h" 29 #include "opt_ratelimit.h" 30 31 #include <dev/mlx5/mlx5_en/en.h> 32 #include <machine/in_cksum.h> 33 34 static inline int 35 mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, 36 struct mlx5e_rx_wqe *wqe, u16 ix) 37 { 38 bus_dma_segment_t segs[MLX5E_MAX_BUSDMA_RX_SEGS]; 39 struct mbuf *mb; 40 int nsegs; 41 int err; 42 struct mbuf *mb_head; 43 int i; 44 45 if (rq->mbuf[ix].mbuf != NULL) 46 return (0); 47 48 mb_head = mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, 49 MLX5E_MAX_RX_BYTES); 50 if (unlikely(mb == NULL)) 51 return (-ENOMEM); 52 53 mb->m_len = MLX5E_MAX_RX_BYTES; 54 mb->m_pkthdr.len = MLX5E_MAX_RX_BYTES; 55 56 for (i = 1; i < rq->nsegs; i++) { 57 if (mb_head->m_pkthdr.len >= rq->wqe_sz) 58 break; 59 mb = mb->m_next = m_getjcl(M_NOWAIT, MT_DATA, 0, 60 MLX5E_MAX_RX_BYTES); 61 if (unlikely(mb == NULL)) { 62 m_freem(mb_head); 63 return (-ENOMEM); 64 } 65 mb->m_len = MLX5E_MAX_RX_BYTES; 66 mb_head->m_pkthdr.len += MLX5E_MAX_RX_BYTES; 67 } 68 /* rewind to first mbuf in chain */ 69 mb = mb_head; 70 71 /* get IP header aligned */ 72 m_adj(mb, MLX5E_NET_IP_ALIGN); 73 74 err = -bus_dmamap_load_mbuf_sg(rq->dma_tag, rq->mbuf[ix].dma_map, 75 mb, segs, &nsegs, BUS_DMA_NOWAIT); 76 if (err != 0) 77 goto err_free_mbuf; 78 if (unlikely(nsegs == 0)) { 79 bus_dmamap_unload(rq->dma_tag, rq->mbuf[ix].dma_map); 80 err = -ENOMEM; 81 goto err_free_mbuf; 82 } 83 wqe->data[0].addr = cpu_to_be64(segs[0].ds_addr); 84 wqe->data[0].byte_count = cpu_to_be32(segs[0].ds_len | 85 MLX5_HW_START_PADDING); 86 for (i = 1; i != nsegs; i++) { 87 wqe->data[i].addr = cpu_to_be64(segs[i].ds_addr); 88 wqe->data[i].byte_count = cpu_to_be32(segs[i].ds_len); 89 } 90 for (; i < rq->nsegs; i++) { 91 wqe->data[i].addr = 0; 92 wqe->data[i].byte_count = 0; 93 } 94 95 rq->mbuf[ix].mbuf = mb; 96 rq->mbuf[ix].data = mb->m_data; 97 98 bus_dmamap_sync(rq->dma_tag, rq->mbuf[ix].dma_map, 99 BUS_DMASYNC_PREREAD); 100 return (0); 101 102 err_free_mbuf: 103 m_freem(mb); 104 return (err); 105 } 106 107 static void 108 mlx5e_post_rx_wqes(struct mlx5e_rq *rq) 109 { 110 if (unlikely(rq->enabled == 0)) 111 return; 112 113 while (!mlx5_wq_ll_is_full(&rq->wq)) { 114 struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, rq->wq.head); 115 116 if (unlikely(mlx5e_alloc_rx_wqe(rq, wqe, rq->wq.head))) { 117 callout_reset_curcpu(&rq->watchdog, 1, (void *)&mlx5e_post_rx_wqes, rq); 118 break; 119 } 120 mlx5_wq_ll_push(&rq->wq, be16_to_cpu(wqe->next.next_wqe_index)); 121 } 122 123 /* ensure wqes are visible to device before updating doorbell record */ 124 atomic_thread_fence_rel(); 125 126 mlx5_wq_ll_update_db_record(&rq->wq); 127 } 128 129 static void 130 mlx5e_lro_update_hdr(struct mbuf *mb, struct mlx5_cqe64 *cqe) 131 { 132 /* TODO: consider vlans, ip options, ... */ 133 struct ether_header *eh; 134 uint16_t eh_type; 135 uint16_t tot_len; 136 struct ip6_hdr *ip6 = NULL; 137 struct ip *ip4 = NULL; 138 struct tcphdr *th; 139 uint32_t *ts_ptr; 140 uint8_t l4_hdr_type; 141 int tcp_ack; 142 143 eh = mtod(mb, struct ether_header *); 144 eh_type = ntohs(eh->ether_type); 145 146 l4_hdr_type = get_cqe_l4_hdr_type(cqe); 147 tcp_ack = ((CQE_L4_HDR_TYPE_TCP_ACK_NO_DATA == l4_hdr_type) || 148 (CQE_L4_HDR_TYPE_TCP_ACK_AND_DATA == l4_hdr_type)); 149 150 /* TODO: consider vlan */ 151 tot_len = be32_to_cpu(cqe->byte_cnt) - ETHER_HDR_LEN; 152 153 switch (eh_type) { 154 case ETHERTYPE_IP: 155 ip4 = (struct ip *)(eh + 1); 156 th = (struct tcphdr *)(ip4 + 1); 157 break; 158 case ETHERTYPE_IPV6: 159 ip6 = (struct ip6_hdr *)(eh + 1); 160 th = (struct tcphdr *)(ip6 + 1); 161 break; 162 default: 163 return; 164 } 165 166 ts_ptr = (uint32_t *)(th + 1); 167 168 if (get_cqe_lro_tcppsh(cqe)) 169 th->th_flags |= TH_PUSH; 170 171 if (tcp_ack) { 172 th->th_flags |= TH_ACK; 173 th->th_ack = cqe->lro_ack_seq_num; 174 th->th_win = cqe->lro_tcp_win; 175 176 /* 177 * FreeBSD handles only 32bit aligned timestamp right after 178 * the TCP hdr 179 * +--------+--------+--------+--------+ 180 * | NOP | NOP | TSopt | 10 | 181 * +--------+--------+--------+--------+ 182 * | TSval timestamp | 183 * +--------+--------+--------+--------+ 184 * | TSecr timestamp | 185 * +--------+--------+--------+--------+ 186 */ 187 if (get_cqe_lro_timestamp_valid(cqe) && 188 (__predict_true(*ts_ptr) == ntohl(TCPOPT_NOP << 24 | 189 TCPOPT_NOP << 16 | TCPOPT_TIMESTAMP << 8 | 190 TCPOLEN_TIMESTAMP))) { 191 /* 192 * cqe->timestamp is 64bit long. 193 * [0-31] - timestamp. 194 * [32-64] - timestamp echo replay. 195 */ 196 ts_ptr[1] = *(uint32_t *)&cqe->timestamp; 197 ts_ptr[2] = *((uint32_t *)&cqe->timestamp + 1); 198 } 199 } 200 if (ip4) { 201 ip4->ip_ttl = cqe->lro_min_ttl; 202 ip4->ip_len = cpu_to_be16(tot_len); 203 ip4->ip_sum = 0; 204 ip4->ip_sum = in_cksum(mb, ip4->ip_hl << 2); 205 } else { 206 ip6->ip6_hlim = cqe->lro_min_ttl; 207 ip6->ip6_plen = cpu_to_be16(tot_len - 208 sizeof(struct ip6_hdr)); 209 } 210 /* TODO: handle tcp checksum */ 211 } 212 213 static uint64_t 214 mlx5e_mbuf_tstmp(struct mlx5e_priv *priv, uint64_t hw_tstmp) 215 { 216 struct mlx5e_clbr_point *cp, dcp; 217 uint64_t tstmp_sec, tstmp_nsec; 218 uint64_t hw_clocks; 219 uint64_t rt_cur_to_prev, res_s, res_n, res_s_modulo, res; 220 uint64_t hw_clk_div; 221 u_int gen; 222 223 do { 224 cp = &priv->clbr_points[priv->clbr_curr]; 225 gen = atomic_load_acq_int(&cp->clbr_gen); 226 if (gen == 0) 227 return (0); 228 dcp = *cp; 229 atomic_thread_fence_acq(); 230 } while (gen != dcp.clbr_gen); 231 /* 232 * Our goal here is to have a result that is: 233 * 234 * ( (cur_time - prev_time) ) 235 * ((hw_tstmp - hw_prev) * ----------------------------- ) + prev_time 236 * ( (hw_cur - hw_prev) ) 237 * 238 * With the constraints that we cannot use float and we 239 * don't want to overflow the uint64_t numbers we are using. 240 * 241 * The plan is to take the clocking value of the hw timestamps 242 * and split them into seconds and nanosecond equivalent portions. 243 * Then we operate on the two portions seperately making sure to 244 * bring back the carry over from the seconds when we divide. 245 * 246 * First up lets get the two divided into separate entities 247 * i.e. the seconds. We use the clock frequency for this. 248 * Note that priv->cclk was setup with the clock frequency 249 * in hz so we are all set to go. 250 */ 251 hw_clocks = hw_tstmp - dcp.clbr_hw_prev; 252 tstmp_sec = hw_clocks / priv->cclk; 253 tstmp_nsec = hw_clocks % priv->cclk; 254 /* Now work with them separately */ 255 rt_cur_to_prev = (dcp.base_curr - dcp.base_prev); 256 res_s = tstmp_sec * rt_cur_to_prev; 257 res_n = tstmp_nsec * rt_cur_to_prev; 258 /* Now lets get our divider */ 259 hw_clk_div = dcp.clbr_hw_curr - dcp.clbr_hw_prev; 260 /* Make sure to save the remainder from the seconds divide */ 261 res_s_modulo = res_s % hw_clk_div; 262 res_s /= hw_clk_div; 263 /* scale the remainder to where it should be */ 264 res_s_modulo *= priv->cclk; 265 /* Now add in the remainder */ 266 res_n += res_s_modulo; 267 /* Now do the divide */ 268 res_n /= hw_clk_div; 269 res_s *= priv->cclk; 270 /* Recombine the two */ 271 res = res_s + res_n; 272 /* And now add in the base time to get to the real timestamp */ 273 res += dcp.base_prev; 274 return (res); 275 } 276 277 static inline void 278 mlx5e_build_rx_mbuf(struct mlx5_cqe64 *cqe, 279 struct mlx5e_rq *rq, struct mbuf *mb, 280 u32 cqe_bcnt) 281 { 282 if_t ifp = rq->ifp; 283 struct mlx5e_channel *c; 284 struct mbuf *mb_head; 285 int lro_num_seg; /* HW LRO session aggregated packets counter */ 286 uint64_t tstmp; 287 288 lro_num_seg = be32_to_cpu(cqe->srqn) >> 24; 289 if (lro_num_seg > 1) { 290 mlx5e_lro_update_hdr(mb, cqe); 291 rq->stats.lro_packets++; 292 rq->stats.lro_bytes += cqe_bcnt; 293 } 294 295 mb->m_pkthdr.len = cqe_bcnt; 296 for (mb_head = mb; mb != NULL; mb = mb->m_next) { 297 if (mb->m_len > cqe_bcnt) 298 mb->m_len = cqe_bcnt; 299 cqe_bcnt -= mb->m_len; 300 if (likely(cqe_bcnt == 0)) { 301 if (likely(mb->m_next != NULL)) { 302 /* trim off empty mbufs */ 303 m_freem(mb->m_next); 304 mb->m_next = NULL; 305 } 306 break; 307 } 308 } 309 /* rewind to first mbuf in chain */ 310 mb = mb_head; 311 312 /* check if a Toeplitz hash was computed */ 313 if (cqe->rss_hash_type != 0) { 314 mb->m_pkthdr.flowid = be32_to_cpu(cqe->rss_hash_result); 315 #ifdef RSS 316 /* decode the RSS hash type */ 317 switch (cqe->rss_hash_type & 318 (CQE_RSS_DST_HTYPE_L4 | CQE_RSS_DST_HTYPE_IP)) { 319 /* IPv4 */ 320 case (CQE_RSS_DST_HTYPE_TCP | CQE_RSS_DST_HTYPE_IPV4): 321 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_TCP_IPV4); 322 break; 323 case (CQE_RSS_DST_HTYPE_UDP | CQE_RSS_DST_HTYPE_IPV4): 324 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_UDP_IPV4); 325 break; 326 case CQE_RSS_DST_HTYPE_IPV4: 327 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_IPV4); 328 break; 329 /* IPv6 */ 330 case (CQE_RSS_DST_HTYPE_TCP | CQE_RSS_DST_HTYPE_IPV6): 331 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_TCP_IPV6); 332 break; 333 case (CQE_RSS_DST_HTYPE_UDP | CQE_RSS_DST_HTYPE_IPV6): 334 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_UDP_IPV6); 335 break; 336 case CQE_RSS_DST_HTYPE_IPV6: 337 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_IPV6); 338 break; 339 default: /* Other */ 340 M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE_HASH); 341 break; 342 } 343 #else 344 M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE_HASH); 345 #endif 346 #ifdef M_HASHTYPE_SETINNER 347 if (cqe_is_tunneled(cqe)) 348 M_HASHTYPE_SETINNER(mb); 349 #endif 350 } else { 351 mb->m_pkthdr.flowid = rq->ix; 352 M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE); 353 } 354 mb->m_pkthdr.rcvif = ifp; 355 mb->m_pkthdr.leaf_rcvif = ifp; 356 357 if (cqe_is_tunneled(cqe)) { 358 /* 359 * CQE can be tunneled only if TIR is configured to 360 * enable parsing of tunneled payload, so no need to 361 * check for capabilities. 362 */ 363 if (((cqe->hds_ip_ext & (CQE_L2_OK | CQE_L3_OK)) == 364 (CQE_L2_OK | CQE_L3_OK))) { 365 mb->m_pkthdr.csum_flags |= 366 CSUM_INNER_L3_CALC | CSUM_INNER_L3_VALID | 367 CSUM_IP_CHECKED | CSUM_IP_VALID | 368 CSUM_DATA_VALID | CSUM_PSEUDO_HDR; 369 mb->m_pkthdr.csum_data = htons(0xffff); 370 371 if (likely((cqe->hds_ip_ext & CQE_L4_OK) == CQE_L4_OK)) { 372 mb->m_pkthdr.csum_flags |= 373 CSUM_INNER_L4_CALC | CSUM_INNER_L4_VALID; 374 } 375 } else { 376 rq->stats.csum_none++; 377 } 378 } else if (likely((if_getcapenable(ifp) & (IFCAP_RXCSUM | 379 IFCAP_RXCSUM_IPV6)) != 0) && 380 ((cqe->hds_ip_ext & (CQE_L2_OK | CQE_L3_OK | CQE_L4_OK)) == 381 (CQE_L2_OK | CQE_L3_OK | CQE_L4_OK))) { 382 mb->m_pkthdr.csum_flags = 383 CSUM_IP_CHECKED | CSUM_IP_VALID | 384 CSUM_DATA_VALID | CSUM_PSEUDO_HDR; 385 mb->m_pkthdr.csum_data = htons(0xffff); 386 } else { 387 rq->stats.csum_none++; 388 } 389 390 if (cqe_has_vlan(cqe)) { 391 mb->m_pkthdr.ether_vtag = be16_to_cpu(cqe->vlan_info); 392 mb->m_flags |= M_VLANTAG; 393 } 394 395 c = container_of(rq, struct mlx5e_channel, rq); 396 if (c->priv->clbr_done >= 2) { 397 tstmp = mlx5e_mbuf_tstmp(c->priv, be64_to_cpu(cqe->timestamp)); 398 if ((tstmp & MLX5_CQE_TSTMP_PTP) != 0) { 399 /* 400 * Timestamp was taken on the packet entrance, 401 * instead of the cqe generation. 402 */ 403 tstmp &= ~MLX5_CQE_TSTMP_PTP; 404 mb->m_flags |= M_TSTMP_HPREC; 405 } 406 if (tstmp != 0) { 407 mb->m_pkthdr.rcv_tstmp = tstmp; 408 mb->m_flags |= M_TSTMP; 409 } 410 } 411 switch (get_cqe_tls_offload(cqe)) { 412 case CQE_TLS_OFFLOAD_DECRYPTED: 413 /* set proper checksum flag for decrypted packets */ 414 mb->m_pkthdr.csum_flags |= CSUM_TLS_DECRYPTED; 415 rq->stats.decrypted_ok_packets++; 416 break; 417 case CQE_TLS_OFFLOAD_ERROR: 418 rq->stats.decrypted_error_packets++; 419 break; 420 default: 421 break; 422 } 423 } 424 425 static inline void 426 mlx5e_read_cqe_slot(struct mlx5e_cq *cq, u32 cc, void *data) 427 { 428 memcpy(data, mlx5_cqwq_get_wqe(&cq->wq, (cc & cq->wq.sz_m1)), 429 sizeof(struct mlx5_cqe64)); 430 } 431 432 static inline void 433 mlx5e_write_cqe_slot(struct mlx5e_cq *cq, u32 cc, void *data) 434 { 435 memcpy(mlx5_cqwq_get_wqe(&cq->wq, cc & cq->wq.sz_m1), 436 data, sizeof(struct mlx5_cqe64)); 437 } 438 439 static inline void 440 mlx5e_decompress_cqe(struct mlx5e_cq *cq, struct mlx5_cqe64 *title, 441 struct mlx5_mini_cqe8 *mini, 442 u16 wqe_counter, int i) 443 { 444 /* 445 * NOTE: The fields which are not set here are copied from the 446 * initial and common title. See memcpy() in 447 * mlx5e_write_cqe_slot(). 448 */ 449 title->byte_cnt = mini->byte_cnt; 450 title->wqe_counter = cpu_to_be16((wqe_counter + i) & cq->wq.sz_m1); 451 title->rss_hash_result = mini->rx_hash_result; 452 /* 453 * Since we use MLX5_CQE_FORMAT_HASH when creating the RX CQ, 454 * the value of the checksum should be ignored. 455 */ 456 title->check_sum = 0; 457 title->op_own = (title->op_own & 0xf0) | 458 (((cq->wq.cc + i) >> cq->wq.log_sz) & 1); 459 } 460 461 #define MLX5E_MINI_ARRAY_SZ 8 462 /* Make sure structs are not packet differently */ 463 CTASSERT(sizeof(struct mlx5_cqe64) == 464 sizeof(struct mlx5_mini_cqe8) * MLX5E_MINI_ARRAY_SZ); 465 static void 466 mlx5e_decompress_cqes(struct mlx5e_cq *cq) 467 { 468 struct mlx5_mini_cqe8 mini_array[MLX5E_MINI_ARRAY_SZ]; 469 struct mlx5_cqe64 title; 470 u32 cqe_count; 471 u32 i = 0; 472 u16 title_wqe_counter; 473 474 mlx5e_read_cqe_slot(cq, cq->wq.cc, &title); 475 title_wqe_counter = be16_to_cpu(title.wqe_counter); 476 cqe_count = be32_to_cpu(title.byte_cnt); 477 478 /* Make sure we won't overflow */ 479 KASSERT(cqe_count <= cq->wq.sz_m1, 480 ("%s: cqe_count %u > cq->wq.sz_m1 %u", __func__, 481 cqe_count, cq->wq.sz_m1)); 482 483 mlx5e_read_cqe_slot(cq, cq->wq.cc + 1, mini_array); 484 while (true) { 485 mlx5e_decompress_cqe(cq, &title, 486 &mini_array[i % MLX5E_MINI_ARRAY_SZ], 487 title_wqe_counter, i); 488 mlx5e_write_cqe_slot(cq, cq->wq.cc + i, &title); 489 i++; 490 491 if (i == cqe_count) 492 break; 493 if (i % MLX5E_MINI_ARRAY_SZ == 0) 494 mlx5e_read_cqe_slot(cq, cq->wq.cc + i, mini_array); 495 } 496 } 497 498 static int 499 mlx5e_poll_rx_cq(struct mlx5e_rq *rq, int budget) 500 { 501 struct pfil_head *pfil; 502 int i, rv; 503 504 CURVNET_SET_QUIET(if_getvnet(rq->ifp)); 505 pfil = rq->channel->priv->pfil; 506 for (i = 0; i < budget; i++) { 507 struct mlx5e_rx_wqe *wqe; 508 struct mlx5_cqe64 *cqe; 509 struct mbuf *mb; 510 __be16 wqe_counter_be; 511 u16 wqe_counter; 512 u32 byte_cnt, seglen; 513 514 cqe = mlx5e_get_cqe(&rq->cq); 515 if (!cqe) 516 break; 517 518 if (mlx5_get_cqe_format(cqe) == MLX5_COMPRESSED) 519 mlx5e_decompress_cqes(&rq->cq); 520 521 mlx5_cqwq_pop(&rq->cq.wq); 522 523 wqe_counter_be = cqe->wqe_counter; 524 wqe_counter = be16_to_cpu(wqe_counter_be); 525 wqe = mlx5_wq_ll_get_wqe(&rq->wq, wqe_counter); 526 byte_cnt = be32_to_cpu(cqe->byte_cnt); 527 528 bus_dmamap_sync(rq->dma_tag, 529 rq->mbuf[wqe_counter].dma_map, 530 BUS_DMASYNC_POSTREAD); 531 532 if (unlikely((cqe->op_own >> 4) != MLX5_CQE_RESP_SEND)) { 533 mlx5e_dump_err_cqe(&rq->cq, rq->rqn, (const void *)cqe); 534 rq->stats.wqe_err++; 535 goto wq_ll_pop; 536 } 537 if (pfil != NULL && PFIL_HOOKED_IN(pfil)) { 538 seglen = MIN(byte_cnt, MLX5E_MAX_RX_BYTES); 539 rv = pfil_mem_in(rq->channel->priv->pfil, 540 rq->mbuf[wqe_counter].data, seglen, rq->ifp, &mb); 541 542 switch (rv) { 543 case PFIL_DROPPED: 544 case PFIL_CONSUMED: 545 /* 546 * Filter dropped or consumed it. In 547 * either case, we can just recycle 548 * buffer; there is no more work to do. 549 */ 550 rq->stats.packets++; 551 goto wq_ll_pop; 552 case PFIL_REALLOCED: 553 /* 554 * Filter copied it; recycle buffer 555 * and receive the new mbuf allocated 556 * by the Filter 557 */ 558 goto rx_common; 559 default: 560 /* 561 * The Filter said it was OK, so 562 * receive like normal. 563 */ 564 KASSERT(rv == PFIL_PASS, 565 ("Filter returned %d!\n", rv)); 566 } 567 } 568 if ((MHLEN - MLX5E_NET_IP_ALIGN) >= byte_cnt && 569 (mb = m_gethdr(M_NOWAIT, MT_DATA)) != NULL) { 570 /* set maximum mbuf length */ 571 mb->m_len = MHLEN - MLX5E_NET_IP_ALIGN; 572 /* get IP header aligned */ 573 mb->m_data += MLX5E_NET_IP_ALIGN; 574 575 bcopy(rq->mbuf[wqe_counter].data, mtod(mb, caddr_t), 576 byte_cnt); 577 } else { 578 mb = rq->mbuf[wqe_counter].mbuf; 579 rq->mbuf[wqe_counter].mbuf = NULL; /* safety clear */ 580 581 bus_dmamap_unload(rq->dma_tag, 582 rq->mbuf[wqe_counter].dma_map); 583 } 584 rx_common: 585 mlx5e_build_rx_mbuf(cqe, rq, mb, byte_cnt); 586 rq->stats.bytes += byte_cnt; 587 rq->stats.packets++; 588 #ifdef NUMA 589 mb->m_pkthdr.numa_domain = if_getnumadomain(rq->ifp); 590 #endif 591 592 #if !defined(HAVE_TCP_LRO_RX) 593 tcp_lro_queue_mbuf(&rq->lro, mb); 594 #else 595 if (mb->m_pkthdr.csum_flags == 0 || 596 (if_getcapenable(rq->ifp) & IFCAP_LRO) == 0 || 597 rq->lro.lro_cnt == 0 || 598 tcp_lro_rx(&rq->lro, mb, 0) != 0) { 599 if_input(rq->ifp, mb); 600 } 601 #endif 602 wq_ll_pop: 603 mlx5_wq_ll_pop(&rq->wq, wqe_counter_be, 604 &wqe->next.next_wqe_index); 605 } 606 CURVNET_RESTORE(); 607 608 mlx5_cqwq_update_db_record(&rq->cq.wq); 609 610 /* ensure cq space is freed before enabling more cqes */ 611 atomic_thread_fence_rel(); 612 return (i); 613 } 614 615 void 616 mlx5e_rx_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe __unused) 617 { 618 struct mlx5e_channel *c = container_of(mcq, struct mlx5e_channel, rq.cq.mcq); 619 struct mlx5e_rq *rq = container_of(mcq, struct mlx5e_rq, cq.mcq); 620 int i = 0; 621 622 #ifdef HAVE_PER_CQ_EVENT_PACKET 623 #if (MHLEN < 15) 624 #error "MHLEN is too small" 625 #endif 626 struct mbuf *mb = m_gethdr(M_NOWAIT, MT_DATA); 627 628 if (mb != NULL) { 629 /* this code is used for debugging purpose only */ 630 mb->m_pkthdr.len = mb->m_len = 15; 631 memset(mb->m_data, 255, 14); 632 mb->m_data[14] = rq->ix; 633 mb->m_pkthdr.rcvif = rq->ifp; 634 mb->m_pkthdr.leaf_rcvif = rq->ifp; 635 if_input(rq->ifp, mb); 636 } 637 #endif 638 for (int j = 0; j != MLX5E_MAX_TX_NUM_TC; j++) { 639 mtx_lock(&c->sq[j].lock); 640 c->sq[j].db_inhibit++; 641 mtx_unlock(&c->sq[j].lock); 642 } 643 644 mtx_lock(&c->iq.lock); 645 c->iq.db_inhibit++; 646 mtx_unlock(&c->iq.lock); 647 648 mtx_lock(&rq->mtx); 649 650 /* 651 * Polling the entire CQ without posting new WQEs results in 652 * lack of receive WQEs during heavy traffic scenarios. 653 */ 654 while (1) { 655 if (mlx5e_poll_rx_cq(rq, MLX5E_RX_BUDGET_MAX) != 656 MLX5E_RX_BUDGET_MAX) 657 break; 658 i += MLX5E_RX_BUDGET_MAX; 659 if (i >= MLX5E_BUDGET_MAX) 660 break; 661 mlx5e_post_rx_wqes(rq); 662 } 663 mlx5e_post_rx_wqes(rq); 664 /* check for dynamic interrupt moderation callback */ 665 if (rq->dim.mode != NET_DIM_CQ_PERIOD_MODE_DISABLED) 666 net_dim(&rq->dim, rq->stats.packets, rq->stats.bytes); 667 mlx5e_cq_arm(&rq->cq, MLX5_GET_DOORBELL_LOCK(&rq->channel->priv->doorbell_lock)); 668 tcp_lro_flush_all(&rq->lro); 669 mtx_unlock(&rq->mtx); 670 671 for (int j = 0; j != MLX5E_MAX_TX_NUM_TC; j++) { 672 mtx_lock(&c->sq[j].lock); 673 c->sq[j].db_inhibit--; 674 /* Update the doorbell record, if any. */ 675 mlx5e_tx_notify_hw(c->sq + j, true); 676 mtx_unlock(&c->sq[j].lock); 677 } 678 679 mtx_lock(&c->iq.lock); 680 c->iq.db_inhibit--; 681 mlx5e_iq_notify_hw(&c->iq); 682 mtx_unlock(&c->iq.lock); 683 } 684