1 /* 2 * Copyright (c) 2015-2016, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/tcp.h> 34 #include <linux/if_vlan.h> 35 #include <net/geneve.h> 36 #include <net/dsfield.h> 37 #include "en.h" 38 #include "en/txrx.h" 39 #include "ipoib/ipoib.h" 40 #include "en_accel/en_accel.h" 41 #include "en_accel/ipsec_rxtx.h" 42 #include "en_accel/macsec.h" 43 #include "en/ptp.h" 44 #include <net/ipv6.h> 45 46 static void mlx5e_dma_unmap_wqe_err(struct mlx5e_txqsq *sq, u8 num_dma) 47 { 48 int i; 49 50 for (i = 0; i < num_dma; i++) { 51 struct mlx5e_sq_dma *last_pushed_dma = 52 mlx5e_dma_get(sq, --sq->dma_fifo_pc); 53 54 mlx5e_tx_dma_unmap(sq->pdev, last_pushed_dma); 55 } 56 } 57 58 static inline int mlx5e_skb_l2_header_offset(struct sk_buff *skb) 59 { 60 #define MLX5E_MIN_INLINE (ETH_HLEN + VLAN_HLEN) 61 62 return max(skb_network_offset(skb), MLX5E_MIN_INLINE); 63 } 64 65 static inline int mlx5e_skb_l3_header_offset(struct sk_buff *skb) 66 { 67 if (skb_transport_header_was_set(skb)) 68 return skb_transport_offset(skb); 69 else 70 return mlx5e_skb_l2_header_offset(skb); 71 } 72 73 static inline u16 mlx5e_calc_min_inline(enum mlx5_inline_modes mode, 74 struct sk_buff *skb) 75 { 76 u16 hlen; 77 78 switch (mode) { 79 case MLX5_INLINE_MODE_NONE: 80 return 0; 81 case MLX5_INLINE_MODE_TCP_UDP: 82 hlen = eth_get_headlen(skb->dev, skb->data, skb_headlen(skb)); 83 if (hlen == ETH_HLEN && !skb_vlan_tag_present(skb)) 84 hlen += VLAN_HLEN; 85 break; 86 case MLX5_INLINE_MODE_IP: 87 hlen = mlx5e_skb_l3_header_offset(skb); 88 break; 89 case MLX5_INLINE_MODE_L2: 90 default: 91 hlen = mlx5e_skb_l2_header_offset(skb); 92 } 93 return min_t(u16, hlen, skb_headlen(skb)); 94 } 95 96 #define MLX5_UNSAFE_MEMCPY_DISCLAIMER \ 97 "This copy has been bounds-checked earlier in " \ 98 "mlx5i_sq_calc_wqe_attr() and intentionally " \ 99 "crosses a flex array boundary. Since it is " \ 100 "performance sensitive, splitting the copy is " \ 101 "undesirable." 102 103 static inline void mlx5e_insert_vlan(void *start, struct sk_buff *skb, u16 ihs) 104 { 105 struct vlan_ethhdr *vhdr = (struct vlan_ethhdr *)start; 106 int cpy1_sz = 2 * ETH_ALEN; 107 int cpy2_sz = ihs - cpy1_sz; 108 109 memcpy(&vhdr->addrs, skb->data, cpy1_sz); 110 vhdr->h_vlan_proto = skb->vlan_proto; 111 vhdr->h_vlan_TCI = cpu_to_be16(skb_vlan_tag_get(skb)); 112 unsafe_memcpy(&vhdr->h_vlan_encapsulated_proto, 113 skb->data + cpy1_sz, 114 cpy2_sz, 115 MLX5_UNSAFE_MEMCPY_DISCLAIMER); 116 } 117 118 static inline void 119 mlx5e_txwqe_build_eseg_csum(struct mlx5e_txqsq *sq, struct sk_buff *skb, 120 struct mlx5e_accel_tx_state *accel, 121 struct mlx5_wqe_eth_seg *eseg) 122 { 123 if (unlikely(mlx5e_ipsec_txwqe_build_eseg_csum(sq, skb, eseg))) 124 return; 125 126 if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) { 127 eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM; 128 if (skb->encapsulation) { 129 eseg->cs_flags |= MLX5_ETH_WQE_L3_INNER_CSUM | 130 MLX5_ETH_WQE_L4_INNER_CSUM; 131 sq->stats->csum_partial_inner++; 132 } else { 133 eseg->cs_flags |= MLX5_ETH_WQE_L4_CSUM; 134 sq->stats->csum_partial++; 135 } 136 #ifdef CONFIG_MLX5_EN_TLS 137 } else if (unlikely(accel && accel->tls.tls_tisn)) { 138 eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM | MLX5_ETH_WQE_L4_CSUM; 139 sq->stats->csum_partial++; 140 #endif 141 } else 142 sq->stats->csum_none++; 143 } 144 145 /* Returns the number of header bytes that we plan 146 * to inline later in the transmit descriptor 147 */ 148 static inline u16 149 mlx5e_tx_get_gso_ihs(struct mlx5e_txqsq *sq, struct sk_buff *skb, int *hopbyhop) 150 { 151 struct mlx5e_sq_stats *stats = sq->stats; 152 u16 ihs; 153 154 *hopbyhop = 0; 155 if (skb->encapsulation) { 156 ihs = skb_inner_tcp_all_headers(skb); 157 stats->tso_inner_packets++; 158 stats->tso_inner_bytes += skb->len - ihs; 159 } else { 160 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) { 161 ihs = skb_transport_offset(skb) + sizeof(struct udphdr); 162 } else { 163 ihs = skb_tcp_all_headers(skb); 164 if (ipv6_has_hopopt_jumbo(skb)) { 165 *hopbyhop = sizeof(struct hop_jumbo_hdr); 166 ihs -= sizeof(struct hop_jumbo_hdr); 167 } 168 } 169 stats->tso_packets++; 170 stats->tso_bytes += skb->len - ihs - *hopbyhop; 171 } 172 173 return ihs; 174 } 175 176 static inline int 177 mlx5e_txwqe_build_dsegs(struct mlx5e_txqsq *sq, struct sk_buff *skb, 178 unsigned char *skb_data, u16 headlen, 179 struct mlx5_wqe_data_seg *dseg) 180 { 181 dma_addr_t dma_addr = 0; 182 u8 num_dma = 0; 183 int i; 184 185 if (headlen) { 186 dma_addr = dma_map_single(sq->pdev, skb_data, headlen, 187 DMA_TO_DEVICE); 188 if (unlikely(dma_mapping_error(sq->pdev, dma_addr))) 189 goto dma_unmap_wqe_err; 190 191 dseg->addr = cpu_to_be64(dma_addr); 192 dseg->lkey = sq->mkey_be; 193 dseg->byte_count = cpu_to_be32(headlen); 194 195 mlx5e_dma_push(sq, dma_addr, headlen, MLX5E_DMA_MAP_SINGLE); 196 num_dma++; 197 dseg++; 198 } 199 200 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 201 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 202 int fsz = skb_frag_size(frag); 203 204 dma_addr = skb_frag_dma_map(sq->pdev, frag, 0, fsz, 205 DMA_TO_DEVICE); 206 if (unlikely(dma_mapping_error(sq->pdev, dma_addr))) 207 goto dma_unmap_wqe_err; 208 209 dseg->addr = cpu_to_be64(dma_addr); 210 dseg->lkey = sq->mkey_be; 211 dseg->byte_count = cpu_to_be32(fsz); 212 213 mlx5e_dma_push(sq, dma_addr, fsz, MLX5E_DMA_MAP_PAGE); 214 num_dma++; 215 dseg++; 216 } 217 218 return num_dma; 219 220 dma_unmap_wqe_err: 221 mlx5e_dma_unmap_wqe_err(sq, num_dma); 222 return -ENOMEM; 223 } 224 225 struct mlx5e_tx_attr { 226 u32 num_bytes; 227 u16 headlen; 228 u16 ihs; 229 __be16 mss; 230 u16 insz; 231 u8 opcode; 232 u8 hopbyhop; 233 }; 234 235 struct mlx5e_tx_wqe_attr { 236 u16 ds_cnt; 237 u16 ds_cnt_inl; 238 u16 ds_cnt_ids; 239 u8 num_wqebbs; 240 }; 241 242 static u8 243 mlx5e_tx_wqe_inline_mode(struct mlx5e_txqsq *sq, struct sk_buff *skb, 244 struct mlx5e_accel_tx_state *accel) 245 { 246 u8 mode; 247 248 #ifdef CONFIG_MLX5_EN_TLS 249 if (accel && accel->tls.tls_tisn) 250 return MLX5_INLINE_MODE_TCP_UDP; 251 #endif 252 253 mode = sq->min_inline_mode; 254 255 if (skb_vlan_tag_present(skb) && 256 test_bit(MLX5E_SQ_STATE_VLAN_NEED_L2_INLINE, &sq->state)) 257 mode = max_t(u8, MLX5_INLINE_MODE_L2, mode); 258 259 return mode; 260 } 261 262 static void mlx5e_sq_xmit_prepare(struct mlx5e_txqsq *sq, struct sk_buff *skb, 263 struct mlx5e_accel_tx_state *accel, 264 struct mlx5e_tx_attr *attr) 265 { 266 struct mlx5e_sq_stats *stats = sq->stats; 267 268 if (skb_is_gso(skb)) { 269 int hopbyhop; 270 u16 ihs = mlx5e_tx_get_gso_ihs(sq, skb, &hopbyhop); 271 272 *attr = (struct mlx5e_tx_attr) { 273 .opcode = MLX5_OPCODE_LSO, 274 .mss = cpu_to_be16(skb_shinfo(skb)->gso_size), 275 .ihs = ihs, 276 .num_bytes = skb->len + (skb_shinfo(skb)->gso_segs - 1) * ihs, 277 .headlen = skb_headlen(skb) - ihs - hopbyhop, 278 .hopbyhop = hopbyhop, 279 }; 280 281 stats->packets += skb_shinfo(skb)->gso_segs; 282 } else { 283 u8 mode = mlx5e_tx_wqe_inline_mode(sq, skb, accel); 284 u16 ihs = mlx5e_calc_min_inline(mode, skb); 285 286 *attr = (struct mlx5e_tx_attr) { 287 .opcode = MLX5_OPCODE_SEND, 288 .mss = cpu_to_be16(0), 289 .ihs = ihs, 290 .num_bytes = max_t(unsigned int, skb->len, ETH_ZLEN), 291 .headlen = skb_headlen(skb) - ihs, 292 }; 293 294 stats->packets++; 295 } 296 297 attr->insz = mlx5e_accel_tx_ids_len(sq, accel); 298 stats->bytes += attr->num_bytes; 299 } 300 301 static void mlx5e_sq_calc_wqe_attr(struct sk_buff *skb, const struct mlx5e_tx_attr *attr, 302 struct mlx5e_tx_wqe_attr *wqe_attr) 303 { 304 u16 ds_cnt = MLX5E_TX_WQE_EMPTY_DS_COUNT; 305 u16 ds_cnt_inl = 0; 306 u16 ds_cnt_ids = 0; 307 308 /* Sync the calculation with MLX5E_MAX_TX_WQEBBS. */ 309 310 if (attr->insz) 311 ds_cnt_ids = DIV_ROUND_UP(sizeof(struct mlx5_wqe_inline_seg) + attr->insz, 312 MLX5_SEND_WQE_DS); 313 314 ds_cnt += !!attr->headlen + skb_shinfo(skb)->nr_frags + ds_cnt_ids; 315 if (attr->ihs) { 316 u16 inl = attr->ihs - INL_HDR_START_SZ; 317 318 if (skb_vlan_tag_present(skb)) 319 inl += VLAN_HLEN; 320 321 ds_cnt_inl = DIV_ROUND_UP(inl, MLX5_SEND_WQE_DS); 322 if (WARN_ON_ONCE(ds_cnt_inl > MLX5E_MAX_TX_INLINE_DS)) 323 netdev_warn(skb->dev, "ds_cnt_inl = %u > max %u\n", ds_cnt_inl, 324 (u16)MLX5E_MAX_TX_INLINE_DS); 325 ds_cnt += ds_cnt_inl; 326 } 327 328 *wqe_attr = (struct mlx5e_tx_wqe_attr) { 329 .ds_cnt = ds_cnt, 330 .ds_cnt_inl = ds_cnt_inl, 331 .ds_cnt_ids = ds_cnt_ids, 332 .num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS), 333 }; 334 } 335 336 static void mlx5e_tx_skb_update_hwts_flags(struct sk_buff *skb) 337 { 338 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) 339 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 340 } 341 342 static void mlx5e_tx_check_stop(struct mlx5e_txqsq *sq) 343 { 344 if (unlikely(!mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, sq->stop_room))) { 345 netif_tx_stop_queue(sq->txq); 346 sq->stats->stopped++; 347 } 348 } 349 350 static void mlx5e_tx_flush(struct mlx5e_txqsq *sq) 351 { 352 struct mlx5e_tx_wqe_info *wi; 353 struct mlx5e_tx_wqe *wqe; 354 u16 pi; 355 356 /* Must not be called when a MPWQE session is active but empty. */ 357 mlx5e_tx_mpwqe_ensure_complete(sq); 358 359 pi = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->pc); 360 wi = &sq->db.wqe_info[pi]; 361 362 *wi = (struct mlx5e_tx_wqe_info) { 363 .num_wqebbs = 1, 364 }; 365 366 wqe = mlx5e_post_nop(&sq->wq, sq->sqn, &sq->pc); 367 mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, &wqe->ctrl); 368 } 369 370 static inline void 371 mlx5e_txwqe_complete(struct mlx5e_txqsq *sq, struct sk_buff *skb, 372 const struct mlx5e_tx_attr *attr, 373 const struct mlx5e_tx_wqe_attr *wqe_attr, u8 num_dma, 374 struct mlx5e_tx_wqe_info *wi, struct mlx5_wqe_ctrl_seg *cseg, 375 bool xmit_more) 376 { 377 struct mlx5_wq_cyc *wq = &sq->wq; 378 bool send_doorbell; 379 380 *wi = (struct mlx5e_tx_wqe_info) { 381 .skb = skb, 382 .num_bytes = attr->num_bytes, 383 .num_dma = num_dma, 384 .num_wqebbs = wqe_attr->num_wqebbs, 385 .num_fifo_pkts = 0, 386 }; 387 388 cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | attr->opcode); 389 cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | wqe_attr->ds_cnt); 390 391 mlx5e_tx_skb_update_hwts_flags(skb); 392 393 sq->pc += wi->num_wqebbs; 394 395 mlx5e_tx_check_stop(sq); 396 397 if (unlikely(sq->ptpsq)) { 398 mlx5e_skb_cb_hwtstamp_init(skb); 399 mlx5e_skb_fifo_push(&sq->ptpsq->skb_fifo, skb); 400 if (!netif_tx_queue_stopped(sq->txq) && 401 !mlx5e_skb_fifo_has_room(&sq->ptpsq->skb_fifo)) { 402 netif_tx_stop_queue(sq->txq); 403 sq->stats->stopped++; 404 } 405 skb_get(skb); 406 } 407 408 send_doorbell = __netdev_tx_sent_queue(sq->txq, attr->num_bytes, xmit_more); 409 if (send_doorbell) 410 mlx5e_notify_hw(wq, sq->pc, sq->uar_map, cseg); 411 } 412 413 static void 414 mlx5e_sq_xmit_wqe(struct mlx5e_txqsq *sq, struct sk_buff *skb, 415 const struct mlx5e_tx_attr *attr, const struct mlx5e_tx_wqe_attr *wqe_attr, 416 struct mlx5e_tx_wqe *wqe, u16 pi, bool xmit_more) 417 { 418 struct mlx5_wqe_ctrl_seg *cseg; 419 struct mlx5_wqe_eth_seg *eseg; 420 struct mlx5_wqe_data_seg *dseg; 421 struct mlx5e_tx_wqe_info *wi; 422 u16 ihs = attr->ihs; 423 struct ipv6hdr *h6; 424 struct mlx5e_sq_stats *stats = sq->stats; 425 int num_dma; 426 427 stats->xmit_more += xmit_more; 428 429 /* fill wqe */ 430 wi = &sq->db.wqe_info[pi]; 431 cseg = &wqe->ctrl; 432 eseg = &wqe->eth; 433 dseg = wqe->data; 434 435 eseg->mss = attr->mss; 436 437 if (ihs) { 438 u8 *start = eseg->inline_hdr.start; 439 440 if (unlikely(attr->hopbyhop)) { 441 /* remove the HBH header. 442 * Layout: [Ethernet header][IPv6 header][HBH][TCP header] 443 */ 444 if (skb_vlan_tag_present(skb)) { 445 mlx5e_insert_vlan(start, skb, ETH_HLEN + sizeof(*h6)); 446 ihs += VLAN_HLEN; 447 h6 = (struct ipv6hdr *)(start + sizeof(struct vlan_ethhdr)); 448 } else { 449 unsafe_memcpy(start, skb->data, 450 ETH_HLEN + sizeof(*h6), 451 MLX5_UNSAFE_MEMCPY_DISCLAIMER); 452 h6 = (struct ipv6hdr *)(start + ETH_HLEN); 453 } 454 h6->nexthdr = IPPROTO_TCP; 455 /* Copy the TCP header after the IPv6 one */ 456 memcpy(h6 + 1, 457 skb->data + ETH_HLEN + sizeof(*h6) + 458 sizeof(struct hop_jumbo_hdr), 459 tcp_hdrlen(skb)); 460 /* Leave ipv6 payload_len set to 0, as LSO v2 specs request. */ 461 } else if (skb_vlan_tag_present(skb)) { 462 mlx5e_insert_vlan(start, skb, ihs); 463 ihs += VLAN_HLEN; 464 stats->added_vlan_packets++; 465 } else { 466 unsafe_memcpy(eseg->inline_hdr.start, skb->data, 467 attr->ihs, 468 MLX5_UNSAFE_MEMCPY_DISCLAIMER); 469 } 470 eseg->inline_hdr.sz |= cpu_to_be16(ihs); 471 dseg += wqe_attr->ds_cnt_inl; 472 } else if (skb_vlan_tag_present(skb)) { 473 eseg->insert.type = cpu_to_be16(MLX5_ETH_WQE_INSERT_VLAN); 474 if (skb->vlan_proto == cpu_to_be16(ETH_P_8021AD)) 475 eseg->insert.type |= cpu_to_be16(MLX5_ETH_WQE_SVLAN); 476 eseg->insert.vlan_tci = cpu_to_be16(skb_vlan_tag_get(skb)); 477 stats->added_vlan_packets++; 478 } 479 480 dseg += wqe_attr->ds_cnt_ids; 481 num_dma = mlx5e_txwqe_build_dsegs(sq, skb, skb->data + attr->ihs + attr->hopbyhop, 482 attr->headlen, dseg); 483 if (unlikely(num_dma < 0)) 484 goto err_drop; 485 486 mlx5e_txwqe_complete(sq, skb, attr, wqe_attr, num_dma, wi, cseg, xmit_more); 487 488 return; 489 490 err_drop: 491 stats->dropped++; 492 dev_kfree_skb_any(skb); 493 mlx5e_tx_flush(sq); 494 } 495 496 static bool mlx5e_tx_skb_supports_mpwqe(struct sk_buff *skb, struct mlx5e_tx_attr *attr) 497 { 498 return !skb_is_nonlinear(skb) && !skb_vlan_tag_present(skb) && !attr->ihs && 499 !attr->insz && !mlx5e_macsec_skb_is_offload(skb); 500 } 501 502 static bool mlx5e_tx_mpwqe_same_eseg(struct mlx5e_txqsq *sq, struct mlx5_wqe_eth_seg *eseg) 503 { 504 struct mlx5e_tx_mpwqe *session = &sq->mpwqe; 505 506 /* Assumes the session is already running and has at least one packet. */ 507 return !memcmp(&session->wqe->eth, eseg, MLX5E_ACCEL_ESEG_LEN); 508 } 509 510 static void mlx5e_tx_mpwqe_session_start(struct mlx5e_txqsq *sq, 511 struct mlx5_wqe_eth_seg *eseg) 512 { 513 struct mlx5e_tx_mpwqe *session = &sq->mpwqe; 514 struct mlx5e_tx_wqe *wqe; 515 u16 pi; 516 517 pi = mlx5e_txqsq_get_next_pi(sq, sq->max_sq_mpw_wqebbs); 518 wqe = MLX5E_TX_FETCH_WQE(sq, pi); 519 net_prefetchw(wqe->data); 520 521 *session = (struct mlx5e_tx_mpwqe) { 522 .wqe = wqe, 523 .bytes_count = 0, 524 .ds_count = MLX5E_TX_WQE_EMPTY_DS_COUNT, 525 .pkt_count = 0, 526 .inline_on = 0, 527 }; 528 529 memcpy(&session->wqe->eth, eseg, MLX5E_ACCEL_ESEG_LEN); 530 531 sq->stats->mpwqe_blks++; 532 } 533 534 static bool mlx5e_tx_mpwqe_session_is_active(struct mlx5e_txqsq *sq) 535 { 536 return sq->mpwqe.wqe; 537 } 538 539 static void mlx5e_tx_mpwqe_add_dseg(struct mlx5e_txqsq *sq, struct mlx5e_xmit_data *txd) 540 { 541 struct mlx5e_tx_mpwqe *session = &sq->mpwqe; 542 struct mlx5_wqe_data_seg *dseg; 543 544 dseg = (struct mlx5_wqe_data_seg *)session->wqe + session->ds_count; 545 546 session->pkt_count++; 547 session->bytes_count += txd->len; 548 549 dseg->addr = cpu_to_be64(txd->dma_addr); 550 dseg->byte_count = cpu_to_be32(txd->len); 551 dseg->lkey = sq->mkey_be; 552 session->ds_count++; 553 554 sq->stats->mpwqe_pkts++; 555 } 556 557 static struct mlx5_wqe_ctrl_seg *mlx5e_tx_mpwqe_session_complete(struct mlx5e_txqsq *sq) 558 { 559 struct mlx5e_tx_mpwqe *session = &sq->mpwqe; 560 u8 ds_count = session->ds_count; 561 struct mlx5_wqe_ctrl_seg *cseg; 562 struct mlx5e_tx_wqe_info *wi; 563 u16 pi; 564 565 cseg = &session->wqe->ctrl; 566 cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_ENHANCED_MPSW); 567 cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_count); 568 569 pi = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->pc); 570 wi = &sq->db.wqe_info[pi]; 571 *wi = (struct mlx5e_tx_wqe_info) { 572 .skb = NULL, 573 .num_bytes = session->bytes_count, 574 .num_wqebbs = DIV_ROUND_UP(ds_count, MLX5_SEND_WQEBB_NUM_DS), 575 .num_dma = session->pkt_count, 576 .num_fifo_pkts = session->pkt_count, 577 }; 578 579 sq->pc += wi->num_wqebbs; 580 581 session->wqe = NULL; 582 583 mlx5e_tx_check_stop(sq); 584 585 return cseg; 586 } 587 588 static void 589 mlx5e_sq_xmit_mpwqe(struct mlx5e_txqsq *sq, struct sk_buff *skb, 590 struct mlx5_wqe_eth_seg *eseg, bool xmit_more) 591 { 592 struct mlx5_wqe_ctrl_seg *cseg; 593 struct mlx5e_xmit_data txd; 594 595 txd.data = skb->data; 596 txd.len = skb->len; 597 598 txd.dma_addr = dma_map_single(sq->pdev, txd.data, txd.len, DMA_TO_DEVICE); 599 if (unlikely(dma_mapping_error(sq->pdev, txd.dma_addr))) 600 goto err_unmap; 601 602 if (!mlx5e_tx_mpwqe_session_is_active(sq)) { 603 mlx5e_tx_mpwqe_session_start(sq, eseg); 604 } else if (!mlx5e_tx_mpwqe_same_eseg(sq, eseg)) { 605 mlx5e_tx_mpwqe_session_complete(sq); 606 mlx5e_tx_mpwqe_session_start(sq, eseg); 607 } 608 609 sq->stats->xmit_more += xmit_more; 610 611 mlx5e_dma_push(sq, txd.dma_addr, txd.len, MLX5E_DMA_MAP_SINGLE); 612 mlx5e_skb_fifo_push(&sq->db.skb_fifo, skb); 613 mlx5e_tx_mpwqe_add_dseg(sq, &txd); 614 mlx5e_tx_skb_update_hwts_flags(skb); 615 616 if (unlikely(mlx5e_tx_mpwqe_is_full(&sq->mpwqe, sq->max_sq_mpw_wqebbs))) { 617 /* Might stop the queue and affect the retval of __netdev_tx_sent_queue. */ 618 cseg = mlx5e_tx_mpwqe_session_complete(sq); 619 620 if (__netdev_tx_sent_queue(sq->txq, txd.len, xmit_more)) 621 mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, cseg); 622 } else if (__netdev_tx_sent_queue(sq->txq, txd.len, xmit_more)) { 623 /* Might stop the queue, but we were asked to ring the doorbell anyway. */ 624 cseg = mlx5e_tx_mpwqe_session_complete(sq); 625 626 mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, cseg); 627 } 628 629 return; 630 631 err_unmap: 632 mlx5e_dma_unmap_wqe_err(sq, 1); 633 sq->stats->dropped++; 634 dev_kfree_skb_any(skb); 635 mlx5e_tx_flush(sq); 636 } 637 638 void mlx5e_tx_mpwqe_ensure_complete(struct mlx5e_txqsq *sq) 639 { 640 /* Unlikely in non-MPWQE workloads; not important in MPWQE workloads. */ 641 if (unlikely(mlx5e_tx_mpwqe_session_is_active(sq))) 642 mlx5e_tx_mpwqe_session_complete(sq); 643 } 644 645 static void mlx5e_cqe_ts_id_eseg(struct mlx5e_ptpsq *ptpsq, struct sk_buff *skb, 646 struct mlx5_wqe_eth_seg *eseg) 647 { 648 if (ptpsq->ts_cqe_ctr_mask && unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) 649 eseg->flow_table_metadata = cpu_to_be32(ptpsq->skb_fifo_pc & 650 ptpsq->ts_cqe_ctr_mask); 651 } 652 653 static void mlx5e_txwqe_build_eseg(struct mlx5e_priv *priv, struct mlx5e_txqsq *sq, 654 struct sk_buff *skb, struct mlx5e_accel_tx_state *accel, 655 struct mlx5_wqe_eth_seg *eseg, u16 ihs) 656 { 657 mlx5e_accel_tx_eseg(priv, skb, eseg, ihs); 658 mlx5e_txwqe_build_eseg_csum(sq, skb, accel, eseg); 659 if (unlikely(sq->ptpsq)) 660 mlx5e_cqe_ts_id_eseg(sq->ptpsq, skb, eseg); 661 } 662 663 netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev) 664 { 665 struct mlx5e_priv *priv = netdev_priv(dev); 666 struct mlx5e_accel_tx_state accel = {}; 667 struct mlx5e_tx_wqe_attr wqe_attr; 668 struct mlx5e_tx_attr attr; 669 struct mlx5e_tx_wqe *wqe; 670 struct mlx5e_txqsq *sq; 671 u16 pi; 672 673 /* All changes to txq2sq are performed in sync with mlx5e_xmit, when the 674 * queue being changed is disabled, and smp_wmb guarantees that the 675 * changes are visible before mlx5e_xmit tries to read from txq2sq. It 676 * guarantees that the value of txq2sq[qid] doesn't change while 677 * mlx5e_xmit is running on queue number qid. smb_wmb is paired with 678 * HARD_TX_LOCK around ndo_start_xmit, which serves as an ACQUIRE. 679 */ 680 sq = priv->txq2sq[skb_get_queue_mapping(skb)]; 681 if (unlikely(!sq)) { 682 /* Two cases when sq can be NULL: 683 * 1. The HTB node is registered, and mlx5e_select_queue 684 * selected its queue ID, but the SQ itself is not yet created. 685 * 2. HTB SQ creation failed. Similar to the previous case, but 686 * the SQ won't be created. 687 */ 688 dev_kfree_skb_any(skb); 689 return NETDEV_TX_OK; 690 } 691 692 /* May send SKBs and WQEs. */ 693 if (unlikely(!mlx5e_accel_tx_begin(dev, sq, skb, &accel))) 694 return NETDEV_TX_OK; 695 696 mlx5e_sq_xmit_prepare(sq, skb, &accel, &attr); 697 698 if (test_bit(MLX5E_SQ_STATE_MPWQE, &sq->state)) { 699 if (mlx5e_tx_skb_supports_mpwqe(skb, &attr)) { 700 struct mlx5_wqe_eth_seg eseg = {}; 701 702 mlx5e_txwqe_build_eseg(priv, sq, skb, &accel, &eseg, attr.ihs); 703 mlx5e_sq_xmit_mpwqe(sq, skb, &eseg, netdev_xmit_more()); 704 return NETDEV_TX_OK; 705 } 706 707 mlx5e_tx_mpwqe_ensure_complete(sq); 708 } 709 710 mlx5e_sq_calc_wqe_attr(skb, &attr, &wqe_attr); 711 pi = mlx5e_txqsq_get_next_pi(sq, wqe_attr.num_wqebbs); 712 wqe = MLX5E_TX_FETCH_WQE(sq, pi); 713 714 /* May update the WQE, but may not post other WQEs. */ 715 mlx5e_accel_tx_finish(sq, wqe, &accel, 716 (struct mlx5_wqe_inline_seg *)(wqe->data + wqe_attr.ds_cnt_inl)); 717 mlx5e_txwqe_build_eseg(priv, sq, skb, &accel, &wqe->eth, attr.ihs); 718 mlx5e_sq_xmit_wqe(sq, skb, &attr, &wqe_attr, wqe, pi, netdev_xmit_more()); 719 720 return NETDEV_TX_OK; 721 } 722 723 void mlx5e_sq_xmit_simple(struct mlx5e_txqsq *sq, struct sk_buff *skb, bool xmit_more) 724 { 725 struct mlx5e_tx_wqe_attr wqe_attr; 726 struct mlx5e_tx_attr attr; 727 struct mlx5e_tx_wqe *wqe; 728 u16 pi; 729 730 mlx5e_sq_xmit_prepare(sq, skb, NULL, &attr); 731 mlx5e_sq_calc_wqe_attr(skb, &attr, &wqe_attr); 732 pi = mlx5e_txqsq_get_next_pi(sq, wqe_attr.num_wqebbs); 733 wqe = MLX5E_TX_FETCH_WQE(sq, pi); 734 mlx5e_txwqe_build_eseg_csum(sq, skb, NULL, &wqe->eth); 735 mlx5e_sq_xmit_wqe(sq, skb, &attr, &wqe_attr, wqe, pi, xmit_more); 736 } 737 738 static void mlx5e_tx_wi_dma_unmap(struct mlx5e_txqsq *sq, struct mlx5e_tx_wqe_info *wi, 739 u32 *dma_fifo_cc) 740 { 741 int i; 742 743 for (i = 0; i < wi->num_dma; i++) { 744 struct mlx5e_sq_dma *dma = mlx5e_dma_get(sq, (*dma_fifo_cc)++); 745 746 mlx5e_tx_dma_unmap(sq->pdev, dma); 747 } 748 } 749 750 static void mlx5e_consume_skb(struct mlx5e_txqsq *sq, struct sk_buff *skb, 751 struct mlx5_cqe64 *cqe, int napi_budget) 752 { 753 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { 754 struct skb_shared_hwtstamps hwts = {}; 755 u64 ts = get_cqe_ts(cqe); 756 757 hwts.hwtstamp = mlx5e_cqe_ts_to_ns(sq->ptp_cyc2time, sq->clock, ts); 758 if (sq->ptpsq) 759 mlx5e_skb_cb_hwtstamp_handler(skb, MLX5E_SKB_CB_CQE_HWTSTAMP, 760 hwts.hwtstamp, sq->ptpsq->cq_stats); 761 else 762 skb_tstamp_tx(skb, &hwts); 763 } 764 765 napi_consume_skb(skb, napi_budget); 766 } 767 768 static void mlx5e_tx_wi_consume_fifo_skbs(struct mlx5e_txqsq *sq, struct mlx5e_tx_wqe_info *wi, 769 struct mlx5_cqe64 *cqe, int napi_budget) 770 { 771 int i; 772 773 for (i = 0; i < wi->num_fifo_pkts; i++) { 774 struct sk_buff *skb = mlx5e_skb_fifo_pop(&sq->db.skb_fifo); 775 776 mlx5e_consume_skb(sq, skb, cqe, napi_budget); 777 } 778 } 779 780 bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget) 781 { 782 struct mlx5e_sq_stats *stats; 783 struct mlx5e_txqsq *sq; 784 struct mlx5_cqe64 *cqe; 785 u32 dma_fifo_cc; 786 u32 nbytes; 787 u16 npkts; 788 u16 sqcc; 789 int i; 790 791 sq = container_of(cq, struct mlx5e_txqsq, cq); 792 793 if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state))) 794 return false; 795 796 cqe = mlx5_cqwq_get_cqe(&cq->wq); 797 if (!cqe) 798 return false; 799 800 stats = sq->stats; 801 802 npkts = 0; 803 nbytes = 0; 804 805 /* sq->cc must be updated only after mlx5_cqwq_update_db_record(), 806 * otherwise a cq overrun may occur 807 */ 808 sqcc = sq->cc; 809 810 /* avoid dirtying sq cache line every cqe */ 811 dma_fifo_cc = sq->dma_fifo_cc; 812 813 i = 0; 814 do { 815 struct mlx5e_tx_wqe_info *wi; 816 u16 wqe_counter; 817 bool last_wqe; 818 u16 ci; 819 820 mlx5_cqwq_pop(&cq->wq); 821 822 wqe_counter = be16_to_cpu(cqe->wqe_counter); 823 824 do { 825 last_wqe = (sqcc == wqe_counter); 826 827 ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sqcc); 828 wi = &sq->db.wqe_info[ci]; 829 830 sqcc += wi->num_wqebbs; 831 832 if (likely(wi->skb)) { 833 mlx5e_tx_wi_dma_unmap(sq, wi, &dma_fifo_cc); 834 mlx5e_consume_skb(sq, wi->skb, cqe, napi_budget); 835 836 npkts++; 837 nbytes += wi->num_bytes; 838 continue; 839 } 840 841 if (unlikely(mlx5e_ktls_tx_try_handle_resync_dump_comp(sq, wi, 842 &dma_fifo_cc))) 843 continue; 844 845 if (wi->num_fifo_pkts) { 846 mlx5e_tx_wi_dma_unmap(sq, wi, &dma_fifo_cc); 847 mlx5e_tx_wi_consume_fifo_skbs(sq, wi, cqe, napi_budget); 848 849 npkts += wi->num_fifo_pkts; 850 nbytes += wi->num_bytes; 851 } 852 } while (!last_wqe); 853 854 if (unlikely(get_cqe_opcode(cqe) == MLX5_CQE_REQ_ERR)) { 855 if (!test_and_set_bit(MLX5E_SQ_STATE_RECOVERING, 856 &sq->state)) { 857 mlx5e_dump_error_cqe(&sq->cq, sq->sqn, 858 (struct mlx5_err_cqe *)cqe); 859 mlx5_wq_cyc_wqe_dump(&sq->wq, ci, wi->num_wqebbs); 860 queue_work(cq->priv->wq, &sq->recover_work); 861 } 862 stats->cqe_err++; 863 } 864 865 } while ((++i < MLX5E_TX_CQ_POLL_BUDGET) && (cqe = mlx5_cqwq_get_cqe(&cq->wq))); 866 867 stats->cqes += i; 868 869 mlx5_cqwq_update_db_record(&cq->wq); 870 871 /* ensure cq space is freed before enabling more cqes */ 872 wmb(); 873 874 sq->dma_fifo_cc = dma_fifo_cc; 875 sq->cc = sqcc; 876 877 netdev_tx_completed_queue(sq->txq, npkts, nbytes); 878 879 if (netif_tx_queue_stopped(sq->txq) && 880 mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, sq->stop_room) && 881 mlx5e_ptpsq_fifo_has_room(sq) && 882 !test_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state)) { 883 netif_tx_wake_queue(sq->txq); 884 stats->wake++; 885 } 886 887 return (i == MLX5E_TX_CQ_POLL_BUDGET); 888 } 889 890 static void mlx5e_tx_wi_kfree_fifo_skbs(struct mlx5e_txqsq *sq, struct mlx5e_tx_wqe_info *wi) 891 { 892 int i; 893 894 for (i = 0; i < wi->num_fifo_pkts; i++) 895 dev_kfree_skb_any(mlx5e_skb_fifo_pop(&sq->db.skb_fifo)); 896 } 897 898 void mlx5e_free_txqsq_descs(struct mlx5e_txqsq *sq) 899 { 900 struct mlx5e_tx_wqe_info *wi; 901 u32 dma_fifo_cc, nbytes = 0; 902 u16 ci, sqcc, npkts = 0; 903 904 sqcc = sq->cc; 905 dma_fifo_cc = sq->dma_fifo_cc; 906 907 while (sqcc != sq->pc) { 908 ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sqcc); 909 wi = &sq->db.wqe_info[ci]; 910 911 sqcc += wi->num_wqebbs; 912 913 if (likely(wi->skb)) { 914 mlx5e_tx_wi_dma_unmap(sq, wi, &dma_fifo_cc); 915 dev_kfree_skb_any(wi->skb); 916 917 npkts++; 918 nbytes += wi->num_bytes; 919 continue; 920 } 921 922 if (unlikely(mlx5e_ktls_tx_try_handle_resync_dump_comp(sq, wi, &dma_fifo_cc))) 923 continue; 924 925 if (wi->num_fifo_pkts) { 926 mlx5e_tx_wi_dma_unmap(sq, wi, &dma_fifo_cc); 927 mlx5e_tx_wi_kfree_fifo_skbs(sq, wi); 928 929 npkts += wi->num_fifo_pkts; 930 nbytes += wi->num_bytes; 931 } 932 } 933 934 sq->dma_fifo_cc = dma_fifo_cc; 935 sq->cc = sqcc; 936 937 netdev_tx_completed_queue(sq->txq, npkts, nbytes); 938 } 939 940 #ifdef CONFIG_MLX5_CORE_IPOIB 941 static inline void 942 mlx5i_txwqe_build_datagram(struct mlx5_av *av, u32 dqpn, u32 dqkey, 943 struct mlx5_wqe_datagram_seg *dseg) 944 { 945 memcpy(&dseg->av, av, sizeof(struct mlx5_av)); 946 dseg->av.dqp_dct = cpu_to_be32(dqpn | MLX5_EXTENDED_UD_AV); 947 dseg->av.key.qkey.qkey = cpu_to_be32(dqkey); 948 } 949 950 static void mlx5i_sq_calc_wqe_attr(struct sk_buff *skb, 951 const struct mlx5e_tx_attr *attr, 952 struct mlx5e_tx_wqe_attr *wqe_attr) 953 { 954 u16 ds_cnt = sizeof(struct mlx5i_tx_wqe) / MLX5_SEND_WQE_DS; 955 u16 ds_cnt_inl = 0; 956 957 ds_cnt += !!attr->headlen + skb_shinfo(skb)->nr_frags; 958 959 if (attr->ihs) { 960 u16 inl = attr->ihs - INL_HDR_START_SZ; 961 962 ds_cnt_inl = DIV_ROUND_UP(inl, MLX5_SEND_WQE_DS); 963 ds_cnt += ds_cnt_inl; 964 } 965 966 *wqe_attr = (struct mlx5e_tx_wqe_attr) { 967 .ds_cnt = ds_cnt, 968 .ds_cnt_inl = ds_cnt_inl, 969 .num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS), 970 }; 971 } 972 973 void mlx5i_sq_xmit(struct mlx5e_txqsq *sq, struct sk_buff *skb, 974 struct mlx5_av *av, u32 dqpn, u32 dqkey, bool xmit_more) 975 { 976 struct mlx5e_tx_wqe_attr wqe_attr; 977 struct mlx5e_tx_attr attr; 978 struct mlx5i_tx_wqe *wqe; 979 980 struct mlx5_wqe_datagram_seg *datagram; 981 struct mlx5_wqe_ctrl_seg *cseg; 982 struct mlx5_wqe_eth_seg *eseg; 983 struct mlx5_wqe_data_seg *dseg; 984 struct mlx5e_tx_wqe_info *wi; 985 986 struct mlx5e_sq_stats *stats = sq->stats; 987 int num_dma; 988 u16 pi; 989 990 mlx5e_sq_xmit_prepare(sq, skb, NULL, &attr); 991 mlx5i_sq_calc_wqe_attr(skb, &attr, &wqe_attr); 992 993 pi = mlx5e_txqsq_get_next_pi(sq, wqe_attr.num_wqebbs); 994 wqe = MLX5I_SQ_FETCH_WQE(sq, pi); 995 996 stats->xmit_more += xmit_more; 997 998 /* fill wqe */ 999 wi = &sq->db.wqe_info[pi]; 1000 cseg = &wqe->ctrl; 1001 datagram = &wqe->datagram; 1002 eseg = &wqe->eth; 1003 dseg = wqe->data; 1004 1005 mlx5i_txwqe_build_datagram(av, dqpn, dqkey, datagram); 1006 1007 mlx5e_txwqe_build_eseg_csum(sq, skb, NULL, eseg); 1008 1009 eseg->mss = attr.mss; 1010 1011 if (attr.ihs) { 1012 if (unlikely(attr.hopbyhop)) { 1013 struct ipv6hdr *h6; 1014 1015 /* remove the HBH header. 1016 * Layout: [Ethernet header][IPv6 header][HBH][TCP header] 1017 */ 1018 unsafe_memcpy(eseg->inline_hdr.start, skb->data, 1019 ETH_HLEN + sizeof(*h6), 1020 MLX5_UNSAFE_MEMCPY_DISCLAIMER); 1021 h6 = (struct ipv6hdr *)((char *)eseg->inline_hdr.start + ETH_HLEN); 1022 h6->nexthdr = IPPROTO_TCP; 1023 /* Copy the TCP header after the IPv6 one */ 1024 unsafe_memcpy(h6 + 1, 1025 skb->data + ETH_HLEN + sizeof(*h6) + 1026 sizeof(struct hop_jumbo_hdr), 1027 tcp_hdrlen(skb), 1028 MLX5_UNSAFE_MEMCPY_DISCLAIMER); 1029 /* Leave ipv6 payload_len set to 0, as LSO v2 specs request. */ 1030 } else { 1031 unsafe_memcpy(eseg->inline_hdr.start, skb->data, 1032 attr.ihs, 1033 MLX5_UNSAFE_MEMCPY_DISCLAIMER); 1034 } 1035 eseg->inline_hdr.sz = cpu_to_be16(attr.ihs); 1036 dseg += wqe_attr.ds_cnt_inl; 1037 } 1038 1039 num_dma = mlx5e_txwqe_build_dsegs(sq, skb, skb->data + attr.ihs + attr.hopbyhop, 1040 attr.headlen, dseg); 1041 if (unlikely(num_dma < 0)) 1042 goto err_drop; 1043 1044 mlx5e_txwqe_complete(sq, skb, &attr, &wqe_attr, num_dma, wi, cseg, xmit_more); 1045 1046 return; 1047 1048 err_drop: 1049 stats->dropped++; 1050 dev_kfree_skb_any(skb); 1051 mlx5e_tx_flush(sq); 1052 } 1053 #endif 1054