1 /* 2 * Copyright (c) 2015-2016, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/tcp.h> 34 #include <linux/if_vlan.h> 35 #include <net/geneve.h> 36 #include <net/dsfield.h> 37 #include "en.h" 38 #include "en/txrx.h" 39 #include "ipoib/ipoib.h" 40 #include "en_accel/en_accel.h" 41 #include "en_accel/ipsec_rxtx.h" 42 #include "en_accel/macsec.h" 43 #include "en/ptp.h" 44 #include <net/ipv6.h> 45 46 static void mlx5e_dma_unmap_wqe_err(struct mlx5e_txqsq *sq, u8 num_dma) 47 { 48 int i; 49 50 for (i = 0; i < num_dma; i++) { 51 struct mlx5e_sq_dma *last_pushed_dma = 52 mlx5e_dma_get(sq, --sq->dma_fifo_pc); 53 54 mlx5e_tx_dma_unmap(sq->pdev, last_pushed_dma); 55 } 56 } 57 58 static inline int mlx5e_skb_l2_header_offset(struct sk_buff *skb) 59 { 60 #define MLX5E_MIN_INLINE (ETH_HLEN + VLAN_HLEN) 61 62 return max(skb_network_offset(skb), MLX5E_MIN_INLINE); 63 } 64 65 static inline int mlx5e_skb_l3_header_offset(struct sk_buff *skb) 66 { 67 if (skb_transport_header_was_set(skb)) 68 return skb_transport_offset(skb); 69 else 70 return mlx5e_skb_l2_header_offset(skb); 71 } 72 73 static inline u16 mlx5e_calc_min_inline(enum mlx5_inline_modes mode, 74 struct sk_buff *skb) 75 { 76 u16 hlen; 77 78 switch (mode) { 79 case MLX5_INLINE_MODE_NONE: 80 return 0; 81 case MLX5_INLINE_MODE_TCP_UDP: 82 hlen = eth_get_headlen(skb->dev, skb->data, skb_headlen(skb)); 83 if (hlen == ETH_HLEN && !skb_vlan_tag_present(skb)) 84 hlen += VLAN_HLEN; 85 break; 86 case MLX5_INLINE_MODE_IP: 87 hlen = mlx5e_skb_l3_header_offset(skb); 88 break; 89 case MLX5_INLINE_MODE_L2: 90 default: 91 hlen = mlx5e_skb_l2_header_offset(skb); 92 } 93 return min_t(u16, hlen, skb_headlen(skb)); 94 } 95 96 #define MLX5_UNSAFE_MEMCPY_DISCLAIMER \ 97 "This copy has been bounds-checked earlier in " \ 98 "mlx5i_sq_calc_wqe_attr() and intentionally " \ 99 "crosses a flex array boundary. Since it is " \ 100 "performance sensitive, splitting the copy is " \ 101 "undesirable." 102 103 static inline void mlx5e_insert_vlan(void *start, struct sk_buff *skb, u16 ihs) 104 { 105 struct vlan_ethhdr *vhdr = (struct vlan_ethhdr *)start; 106 int cpy1_sz = 2 * ETH_ALEN; 107 int cpy2_sz = ihs - cpy1_sz; 108 109 memcpy(&vhdr->addrs, skb->data, cpy1_sz); 110 vhdr->h_vlan_proto = skb->vlan_proto; 111 vhdr->h_vlan_TCI = cpu_to_be16(skb_vlan_tag_get(skb)); 112 unsafe_memcpy(&vhdr->h_vlan_encapsulated_proto, 113 skb->data + cpy1_sz, 114 cpy2_sz, 115 MLX5_UNSAFE_MEMCPY_DISCLAIMER); 116 } 117 118 static inline void 119 mlx5e_txwqe_build_eseg_csum(struct mlx5e_txqsq *sq, struct sk_buff *skb, 120 struct mlx5e_accel_tx_state *accel, 121 struct mlx5_wqe_eth_seg *eseg) 122 { 123 if (unlikely(mlx5e_ipsec_txwqe_build_eseg_csum(sq, skb, eseg))) 124 return; 125 126 if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) { 127 eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM; 128 if (skb->encapsulation) { 129 eseg->cs_flags |= MLX5_ETH_WQE_L3_INNER_CSUM | 130 MLX5_ETH_WQE_L4_INNER_CSUM; 131 sq->stats->csum_partial_inner++; 132 } else { 133 eseg->cs_flags |= MLX5_ETH_WQE_L4_CSUM; 134 sq->stats->csum_partial++; 135 } 136 #ifdef CONFIG_MLX5_EN_TLS 137 } else if (unlikely(accel && accel->tls.tls_tisn)) { 138 eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM | MLX5_ETH_WQE_L4_CSUM; 139 sq->stats->csum_partial++; 140 #endif 141 } else 142 sq->stats->csum_none++; 143 } 144 145 /* Returns the number of header bytes that we plan 146 * to inline later in the transmit descriptor 147 */ 148 static inline u16 149 mlx5e_tx_get_gso_ihs(struct mlx5e_txqsq *sq, struct sk_buff *skb, int *hopbyhop) 150 { 151 struct mlx5e_sq_stats *stats = sq->stats; 152 u16 ihs; 153 154 *hopbyhop = 0; 155 if (skb->encapsulation) { 156 ihs = skb_inner_tcp_all_headers(skb); 157 stats->tso_inner_packets++; 158 stats->tso_inner_bytes += skb->len - ihs; 159 } else { 160 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) { 161 ihs = skb_transport_offset(skb) + sizeof(struct udphdr); 162 } else { 163 ihs = skb_tcp_all_headers(skb); 164 if (ipv6_has_hopopt_jumbo(skb)) { 165 *hopbyhop = sizeof(struct hop_jumbo_hdr); 166 ihs -= sizeof(struct hop_jumbo_hdr); 167 } 168 } 169 stats->tso_packets++; 170 stats->tso_bytes += skb->len - ihs - *hopbyhop; 171 } 172 173 return ihs; 174 } 175 176 static inline int 177 mlx5e_txwqe_build_dsegs(struct mlx5e_txqsq *sq, struct sk_buff *skb, 178 unsigned char *skb_data, u16 headlen, 179 struct mlx5_wqe_data_seg *dseg) 180 { 181 dma_addr_t dma_addr = 0; 182 u8 num_dma = 0; 183 int i; 184 185 if (headlen) { 186 dma_addr = dma_map_single(sq->pdev, skb_data, headlen, 187 DMA_TO_DEVICE); 188 if (unlikely(dma_mapping_error(sq->pdev, dma_addr))) 189 goto dma_unmap_wqe_err; 190 191 dseg->addr = cpu_to_be64(dma_addr); 192 dseg->lkey = sq->mkey_be; 193 dseg->byte_count = cpu_to_be32(headlen); 194 195 mlx5e_dma_push(sq, dma_addr, headlen, MLX5E_DMA_MAP_SINGLE); 196 num_dma++; 197 dseg++; 198 } 199 200 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 201 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 202 int fsz = skb_frag_size(frag); 203 204 dma_addr = skb_frag_dma_map(sq->pdev, frag, 0, fsz, 205 DMA_TO_DEVICE); 206 if (unlikely(dma_mapping_error(sq->pdev, dma_addr))) 207 goto dma_unmap_wqe_err; 208 209 dseg->addr = cpu_to_be64(dma_addr); 210 dseg->lkey = sq->mkey_be; 211 dseg->byte_count = cpu_to_be32(fsz); 212 213 mlx5e_dma_push(sq, dma_addr, fsz, MLX5E_DMA_MAP_PAGE); 214 num_dma++; 215 dseg++; 216 } 217 218 return num_dma; 219 220 dma_unmap_wqe_err: 221 mlx5e_dma_unmap_wqe_err(sq, num_dma); 222 return -ENOMEM; 223 } 224 225 struct mlx5e_tx_attr { 226 u32 num_bytes; 227 u16 headlen; 228 u16 ihs; 229 __be16 mss; 230 u16 insz; 231 u8 opcode; 232 u8 hopbyhop; 233 }; 234 235 struct mlx5e_tx_wqe_attr { 236 u16 ds_cnt; 237 u16 ds_cnt_inl; 238 u16 ds_cnt_ids; 239 u8 num_wqebbs; 240 }; 241 242 static u8 243 mlx5e_tx_wqe_inline_mode(struct mlx5e_txqsq *sq, struct sk_buff *skb, 244 struct mlx5e_accel_tx_state *accel) 245 { 246 u8 mode; 247 248 #ifdef CONFIG_MLX5_EN_TLS 249 if (accel && accel->tls.tls_tisn) 250 return MLX5_INLINE_MODE_TCP_UDP; 251 #endif 252 253 mode = sq->min_inline_mode; 254 255 if (skb_vlan_tag_present(skb) && 256 test_bit(MLX5E_SQ_STATE_VLAN_NEED_L2_INLINE, &sq->state)) 257 mode = max_t(u8, MLX5_INLINE_MODE_L2, mode); 258 259 return mode; 260 } 261 262 static void mlx5e_sq_xmit_prepare(struct mlx5e_txqsq *sq, struct sk_buff *skb, 263 struct mlx5e_accel_tx_state *accel, 264 struct mlx5e_tx_attr *attr) 265 { 266 struct mlx5e_sq_stats *stats = sq->stats; 267 268 if (skb_is_gso(skb)) { 269 int hopbyhop; 270 u16 ihs = mlx5e_tx_get_gso_ihs(sq, skb, &hopbyhop); 271 272 *attr = (struct mlx5e_tx_attr) { 273 .opcode = MLX5_OPCODE_LSO, 274 .mss = cpu_to_be16(skb_shinfo(skb)->gso_size), 275 .ihs = ihs, 276 .num_bytes = skb->len + (skb_shinfo(skb)->gso_segs - 1) * ihs, 277 .headlen = skb_headlen(skb) - ihs - hopbyhop, 278 .hopbyhop = hopbyhop, 279 }; 280 281 stats->packets += skb_shinfo(skb)->gso_segs; 282 } else { 283 u8 mode = mlx5e_tx_wqe_inline_mode(sq, skb, accel); 284 u16 ihs = mlx5e_calc_min_inline(mode, skb); 285 286 *attr = (struct mlx5e_tx_attr) { 287 .opcode = MLX5_OPCODE_SEND, 288 .mss = cpu_to_be16(0), 289 .ihs = ihs, 290 .num_bytes = max_t(unsigned int, skb->len, ETH_ZLEN), 291 .headlen = skb_headlen(skb) - ihs, 292 }; 293 294 stats->packets++; 295 } 296 297 attr->insz = mlx5e_accel_tx_ids_len(sq, accel); 298 stats->bytes += attr->num_bytes; 299 } 300 301 static void mlx5e_sq_calc_wqe_attr(struct sk_buff *skb, const struct mlx5e_tx_attr *attr, 302 struct mlx5e_tx_wqe_attr *wqe_attr) 303 { 304 u16 ds_cnt = MLX5E_TX_WQE_EMPTY_DS_COUNT; 305 u16 ds_cnt_inl = 0; 306 u16 ds_cnt_ids = 0; 307 308 /* Sync the calculation with MLX5E_MAX_TX_WQEBBS. */ 309 310 if (attr->insz) 311 ds_cnt_ids = DIV_ROUND_UP(sizeof(struct mlx5_wqe_inline_seg) + attr->insz, 312 MLX5_SEND_WQE_DS); 313 314 ds_cnt += !!attr->headlen + skb_shinfo(skb)->nr_frags + ds_cnt_ids; 315 if (attr->ihs) { 316 u16 inl = attr->ihs - INL_HDR_START_SZ; 317 318 if (skb_vlan_tag_present(skb)) 319 inl += VLAN_HLEN; 320 321 ds_cnt_inl = DIV_ROUND_UP(inl, MLX5_SEND_WQE_DS); 322 if (WARN_ON_ONCE(ds_cnt_inl > MLX5E_MAX_TX_INLINE_DS)) 323 netdev_warn(skb->dev, "ds_cnt_inl = %u > max %u\n", ds_cnt_inl, 324 (u16)MLX5E_MAX_TX_INLINE_DS); 325 ds_cnt += ds_cnt_inl; 326 } 327 328 *wqe_attr = (struct mlx5e_tx_wqe_attr) { 329 .ds_cnt = ds_cnt, 330 .ds_cnt_inl = ds_cnt_inl, 331 .ds_cnt_ids = ds_cnt_ids, 332 .num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS), 333 }; 334 } 335 336 static void mlx5e_tx_skb_update_hwts_flags(struct sk_buff *skb) 337 { 338 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) 339 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 340 } 341 342 static void mlx5e_tx_check_stop(struct mlx5e_txqsq *sq) 343 { 344 if (unlikely(!mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, sq->stop_room))) { 345 netif_tx_stop_queue(sq->txq); 346 sq->stats->stopped++; 347 } 348 } 349 350 static void mlx5e_tx_flush(struct mlx5e_txqsq *sq) 351 { 352 struct mlx5e_tx_wqe_info *wi; 353 struct mlx5e_tx_wqe *wqe; 354 u16 pi; 355 356 /* Must not be called when a MPWQE session is active but empty. */ 357 mlx5e_tx_mpwqe_ensure_complete(sq); 358 359 pi = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->pc); 360 wi = &sq->db.wqe_info[pi]; 361 362 *wi = (struct mlx5e_tx_wqe_info) { 363 .num_wqebbs = 1, 364 }; 365 366 wqe = mlx5e_post_nop(&sq->wq, sq->sqn, &sq->pc); 367 mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, &wqe->ctrl); 368 } 369 370 static inline void 371 mlx5e_txwqe_complete(struct mlx5e_txqsq *sq, struct sk_buff *skb, 372 const struct mlx5e_tx_attr *attr, 373 const struct mlx5e_tx_wqe_attr *wqe_attr, u8 num_dma, 374 struct mlx5e_tx_wqe_info *wi, struct mlx5_wqe_ctrl_seg *cseg, 375 struct mlx5_wqe_eth_seg *eseg, bool xmit_more) 376 { 377 struct mlx5_wq_cyc *wq = &sq->wq; 378 bool send_doorbell; 379 380 *wi = (struct mlx5e_tx_wqe_info) { 381 .skb = skb, 382 .num_bytes = attr->num_bytes, 383 .num_dma = num_dma, 384 .num_wqebbs = wqe_attr->num_wqebbs, 385 .num_fifo_pkts = 0, 386 }; 387 388 cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | attr->opcode); 389 cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | wqe_attr->ds_cnt); 390 391 mlx5e_tx_skb_update_hwts_flags(skb); 392 393 sq->pc += wi->num_wqebbs; 394 395 mlx5e_tx_check_stop(sq); 396 397 if (unlikely(sq->ptpsq && 398 (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))) { 399 u8 metadata_index = be32_to_cpu(eseg->flow_table_metadata); 400 401 mlx5e_ptp_metadata_fifo_pop(&sq->ptpsq->metadata_freelist); 402 403 mlx5e_skb_cb_hwtstamp_init(skb); 404 mlx5e_ptp_metadata_map_put(&sq->ptpsq->metadata_map, skb, 405 metadata_index); 406 /* ensure skb is put on metadata_map before tracking the index */ 407 wmb(); 408 mlx5e_ptpsq_track_metadata(sq->ptpsq, metadata_index); 409 if (!netif_tx_queue_stopped(sq->txq) && 410 mlx5e_ptpsq_metadata_freelist_empty(sq->ptpsq)) { 411 netif_tx_stop_queue(sq->txq); 412 sq->stats->stopped++; 413 } 414 skb_get(skb); 415 } 416 417 send_doorbell = __netdev_tx_sent_queue(sq->txq, attr->num_bytes, xmit_more); 418 if (send_doorbell) 419 mlx5e_notify_hw(wq, sq->pc, sq->uar_map, cseg); 420 } 421 422 static void 423 mlx5e_sq_xmit_wqe(struct mlx5e_txqsq *sq, struct sk_buff *skb, 424 const struct mlx5e_tx_attr *attr, const struct mlx5e_tx_wqe_attr *wqe_attr, 425 struct mlx5e_tx_wqe *wqe, u16 pi, bool xmit_more) 426 { 427 struct mlx5_wqe_ctrl_seg *cseg; 428 struct mlx5_wqe_eth_seg *eseg; 429 struct mlx5_wqe_data_seg *dseg; 430 struct mlx5e_tx_wqe_info *wi; 431 u16 ihs = attr->ihs; 432 struct ipv6hdr *h6; 433 struct mlx5e_sq_stats *stats = sq->stats; 434 int num_dma; 435 436 stats->xmit_more += xmit_more; 437 438 /* fill wqe */ 439 wi = &sq->db.wqe_info[pi]; 440 cseg = &wqe->ctrl; 441 eseg = &wqe->eth; 442 dseg = wqe->data; 443 444 eseg->mss = attr->mss; 445 446 if (ihs) { 447 u8 *start = eseg->inline_hdr.start; 448 449 if (unlikely(attr->hopbyhop)) { 450 /* remove the HBH header. 451 * Layout: [Ethernet header][IPv6 header][HBH][TCP header] 452 */ 453 if (skb_vlan_tag_present(skb)) { 454 mlx5e_insert_vlan(start, skb, ETH_HLEN + sizeof(*h6)); 455 ihs += VLAN_HLEN; 456 h6 = (struct ipv6hdr *)(start + sizeof(struct vlan_ethhdr)); 457 } else { 458 unsafe_memcpy(start, skb->data, 459 ETH_HLEN + sizeof(*h6), 460 MLX5_UNSAFE_MEMCPY_DISCLAIMER); 461 h6 = (struct ipv6hdr *)(start + ETH_HLEN); 462 } 463 h6->nexthdr = IPPROTO_TCP; 464 /* Copy the TCP header after the IPv6 one */ 465 memcpy(h6 + 1, 466 skb->data + ETH_HLEN + sizeof(*h6) + 467 sizeof(struct hop_jumbo_hdr), 468 tcp_hdrlen(skb)); 469 /* Leave ipv6 payload_len set to 0, as LSO v2 specs request. */ 470 } else if (skb_vlan_tag_present(skb)) { 471 mlx5e_insert_vlan(start, skb, ihs); 472 ihs += VLAN_HLEN; 473 stats->added_vlan_packets++; 474 } else { 475 unsafe_memcpy(eseg->inline_hdr.start, skb->data, 476 attr->ihs, 477 MLX5_UNSAFE_MEMCPY_DISCLAIMER); 478 } 479 eseg->inline_hdr.sz |= cpu_to_be16(ihs); 480 dseg += wqe_attr->ds_cnt_inl; 481 } else if (skb_vlan_tag_present(skb)) { 482 eseg->insert.type = cpu_to_be16(MLX5_ETH_WQE_INSERT_VLAN); 483 if (skb->vlan_proto == cpu_to_be16(ETH_P_8021AD)) 484 eseg->insert.type |= cpu_to_be16(MLX5_ETH_WQE_SVLAN); 485 eseg->insert.vlan_tci = cpu_to_be16(skb_vlan_tag_get(skb)); 486 stats->added_vlan_packets++; 487 } 488 489 dseg += wqe_attr->ds_cnt_ids; 490 num_dma = mlx5e_txwqe_build_dsegs(sq, skb, skb->data + attr->ihs + attr->hopbyhop, 491 attr->headlen, dseg); 492 if (unlikely(num_dma < 0)) 493 goto err_drop; 494 495 mlx5e_txwqe_complete(sq, skb, attr, wqe_attr, num_dma, wi, cseg, eseg, xmit_more); 496 497 return; 498 499 err_drop: 500 stats->dropped++; 501 dev_kfree_skb_any(skb); 502 mlx5e_tx_flush(sq); 503 } 504 505 static bool mlx5e_tx_skb_supports_mpwqe(struct sk_buff *skb, struct mlx5e_tx_attr *attr) 506 { 507 return !skb_is_nonlinear(skb) && !skb_vlan_tag_present(skb) && !attr->ihs && 508 !attr->insz && !mlx5e_macsec_skb_is_offload(skb); 509 } 510 511 static bool mlx5e_tx_mpwqe_same_eseg(struct mlx5e_txqsq *sq, struct mlx5_wqe_eth_seg *eseg) 512 { 513 struct mlx5e_tx_mpwqe *session = &sq->mpwqe; 514 515 /* Assumes the session is already running and has at least one packet. */ 516 return !memcmp(&session->wqe->eth, eseg, MLX5E_ACCEL_ESEG_LEN); 517 } 518 519 static void mlx5e_tx_mpwqe_session_start(struct mlx5e_txqsq *sq, 520 struct mlx5_wqe_eth_seg *eseg) 521 { 522 struct mlx5e_tx_mpwqe *session = &sq->mpwqe; 523 struct mlx5e_tx_wqe *wqe; 524 u16 pi; 525 526 pi = mlx5e_txqsq_get_next_pi(sq, sq->max_sq_mpw_wqebbs); 527 wqe = MLX5E_TX_FETCH_WQE(sq, pi); 528 net_prefetchw(wqe->data); 529 530 *session = (struct mlx5e_tx_mpwqe) { 531 .wqe = wqe, 532 .bytes_count = 0, 533 .ds_count = MLX5E_TX_WQE_EMPTY_DS_COUNT, 534 .pkt_count = 0, 535 .inline_on = 0, 536 }; 537 538 memcpy(&session->wqe->eth, eseg, MLX5E_ACCEL_ESEG_LEN); 539 540 sq->stats->mpwqe_blks++; 541 } 542 543 static bool mlx5e_tx_mpwqe_session_is_active(struct mlx5e_txqsq *sq) 544 { 545 return sq->mpwqe.wqe; 546 } 547 548 static void mlx5e_tx_mpwqe_add_dseg(struct mlx5e_txqsq *sq, struct mlx5e_xmit_data *txd) 549 { 550 struct mlx5e_tx_mpwqe *session = &sq->mpwqe; 551 struct mlx5_wqe_data_seg *dseg; 552 553 dseg = (struct mlx5_wqe_data_seg *)session->wqe + session->ds_count; 554 555 session->pkt_count++; 556 session->bytes_count += txd->len; 557 558 dseg->addr = cpu_to_be64(txd->dma_addr); 559 dseg->byte_count = cpu_to_be32(txd->len); 560 dseg->lkey = sq->mkey_be; 561 session->ds_count++; 562 563 sq->stats->mpwqe_pkts++; 564 } 565 566 static struct mlx5_wqe_ctrl_seg *mlx5e_tx_mpwqe_session_complete(struct mlx5e_txqsq *sq) 567 { 568 struct mlx5e_tx_mpwqe *session = &sq->mpwqe; 569 u8 ds_count = session->ds_count; 570 struct mlx5_wqe_ctrl_seg *cseg; 571 struct mlx5e_tx_wqe_info *wi; 572 u16 pi; 573 574 cseg = &session->wqe->ctrl; 575 cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_ENHANCED_MPSW); 576 cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_count); 577 578 pi = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->pc); 579 wi = &sq->db.wqe_info[pi]; 580 *wi = (struct mlx5e_tx_wqe_info) { 581 .skb = NULL, 582 .num_bytes = session->bytes_count, 583 .num_wqebbs = DIV_ROUND_UP(ds_count, MLX5_SEND_WQEBB_NUM_DS), 584 .num_dma = session->pkt_count, 585 .num_fifo_pkts = session->pkt_count, 586 }; 587 588 sq->pc += wi->num_wqebbs; 589 590 session->wqe = NULL; 591 592 mlx5e_tx_check_stop(sq); 593 594 return cseg; 595 } 596 597 static void 598 mlx5e_sq_xmit_mpwqe(struct mlx5e_txqsq *sq, struct sk_buff *skb, 599 struct mlx5_wqe_eth_seg *eseg, bool xmit_more) 600 { 601 struct mlx5_wqe_ctrl_seg *cseg; 602 struct mlx5e_xmit_data txd; 603 604 txd.data = skb->data; 605 txd.len = skb->len; 606 607 txd.dma_addr = dma_map_single(sq->pdev, txd.data, txd.len, DMA_TO_DEVICE); 608 if (unlikely(dma_mapping_error(sq->pdev, txd.dma_addr))) 609 goto err_unmap; 610 611 if (!mlx5e_tx_mpwqe_session_is_active(sq)) { 612 mlx5e_tx_mpwqe_session_start(sq, eseg); 613 } else if (!mlx5e_tx_mpwqe_same_eseg(sq, eseg)) { 614 mlx5e_tx_mpwqe_session_complete(sq); 615 mlx5e_tx_mpwqe_session_start(sq, eseg); 616 } 617 618 sq->stats->xmit_more += xmit_more; 619 620 mlx5e_dma_push(sq, txd.dma_addr, txd.len, MLX5E_DMA_MAP_SINGLE); 621 mlx5e_skb_fifo_push(&sq->db.skb_fifo, skb); 622 mlx5e_tx_mpwqe_add_dseg(sq, &txd); 623 mlx5e_tx_skb_update_hwts_flags(skb); 624 625 if (unlikely(mlx5e_tx_mpwqe_is_full(&sq->mpwqe, sq->max_sq_mpw_wqebbs))) { 626 /* Might stop the queue and affect the retval of __netdev_tx_sent_queue. */ 627 cseg = mlx5e_tx_mpwqe_session_complete(sq); 628 629 if (__netdev_tx_sent_queue(sq->txq, txd.len, xmit_more)) 630 mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, cseg); 631 } else if (__netdev_tx_sent_queue(sq->txq, txd.len, xmit_more)) { 632 /* Might stop the queue, but we were asked to ring the doorbell anyway. */ 633 cseg = mlx5e_tx_mpwqe_session_complete(sq); 634 635 mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, cseg); 636 } 637 638 return; 639 640 err_unmap: 641 mlx5e_dma_unmap_wqe_err(sq, 1); 642 sq->stats->dropped++; 643 dev_kfree_skb_any(skb); 644 mlx5e_tx_flush(sq); 645 } 646 647 void mlx5e_tx_mpwqe_ensure_complete(struct mlx5e_txqsq *sq) 648 { 649 /* Unlikely in non-MPWQE workloads; not important in MPWQE workloads. */ 650 if (unlikely(mlx5e_tx_mpwqe_session_is_active(sq))) 651 mlx5e_tx_mpwqe_session_complete(sq); 652 } 653 654 static void mlx5e_cqe_ts_id_eseg(struct mlx5e_ptpsq *ptpsq, struct sk_buff *skb, 655 struct mlx5_wqe_eth_seg *eseg) 656 { 657 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) 658 eseg->flow_table_metadata = 659 cpu_to_be32(mlx5e_ptp_metadata_fifo_peek(&ptpsq->metadata_freelist)); 660 } 661 662 static void mlx5e_txwqe_build_eseg(struct mlx5e_priv *priv, struct mlx5e_txqsq *sq, 663 struct sk_buff *skb, struct mlx5e_accel_tx_state *accel, 664 struct mlx5_wqe_eth_seg *eseg, u16 ihs) 665 { 666 mlx5e_accel_tx_eseg(priv, skb, eseg, ihs); 667 mlx5e_txwqe_build_eseg_csum(sq, skb, accel, eseg); 668 if (unlikely(sq->ptpsq)) 669 mlx5e_cqe_ts_id_eseg(sq->ptpsq, skb, eseg); 670 } 671 672 netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev) 673 { 674 struct mlx5e_priv *priv = netdev_priv(dev); 675 struct mlx5e_accel_tx_state accel = {}; 676 struct mlx5e_tx_wqe_attr wqe_attr; 677 struct mlx5e_tx_attr attr; 678 struct mlx5e_tx_wqe *wqe; 679 struct mlx5e_txqsq *sq; 680 u16 pi; 681 682 /* All changes to txq2sq are performed in sync with mlx5e_xmit, when the 683 * queue being changed is disabled, and smp_wmb guarantees that the 684 * changes are visible before mlx5e_xmit tries to read from txq2sq. It 685 * guarantees that the value of txq2sq[qid] doesn't change while 686 * mlx5e_xmit is running on queue number qid. smb_wmb is paired with 687 * HARD_TX_LOCK around ndo_start_xmit, which serves as an ACQUIRE. 688 */ 689 sq = priv->txq2sq[skb_get_queue_mapping(skb)]; 690 if (unlikely(!sq)) { 691 /* Two cases when sq can be NULL: 692 * 1. The HTB node is registered, and mlx5e_select_queue 693 * selected its queue ID, but the SQ itself is not yet created. 694 * 2. HTB SQ creation failed. Similar to the previous case, but 695 * the SQ won't be created. 696 */ 697 dev_kfree_skb_any(skb); 698 return NETDEV_TX_OK; 699 } 700 701 /* May send SKBs and WQEs. */ 702 if (unlikely(!mlx5e_accel_tx_begin(dev, sq, skb, &accel))) 703 return NETDEV_TX_OK; 704 705 mlx5e_sq_xmit_prepare(sq, skb, &accel, &attr); 706 707 if (test_bit(MLX5E_SQ_STATE_MPWQE, &sq->state)) { 708 if (mlx5e_tx_skb_supports_mpwqe(skb, &attr)) { 709 struct mlx5_wqe_eth_seg eseg = {}; 710 711 mlx5e_txwqe_build_eseg(priv, sq, skb, &accel, &eseg, attr.ihs); 712 mlx5e_sq_xmit_mpwqe(sq, skb, &eseg, netdev_xmit_more()); 713 return NETDEV_TX_OK; 714 } 715 716 mlx5e_tx_mpwqe_ensure_complete(sq); 717 } 718 719 mlx5e_sq_calc_wqe_attr(skb, &attr, &wqe_attr); 720 pi = mlx5e_txqsq_get_next_pi(sq, wqe_attr.num_wqebbs); 721 wqe = MLX5E_TX_FETCH_WQE(sq, pi); 722 723 /* May update the WQE, but may not post other WQEs. */ 724 mlx5e_accel_tx_finish(sq, wqe, &accel, 725 (struct mlx5_wqe_inline_seg *)(wqe->data + wqe_attr.ds_cnt_inl)); 726 mlx5e_txwqe_build_eseg(priv, sq, skb, &accel, &wqe->eth, attr.ihs); 727 mlx5e_sq_xmit_wqe(sq, skb, &attr, &wqe_attr, wqe, pi, netdev_xmit_more()); 728 729 return NETDEV_TX_OK; 730 } 731 732 static void mlx5e_tx_wi_dma_unmap(struct mlx5e_txqsq *sq, struct mlx5e_tx_wqe_info *wi, 733 u32 *dma_fifo_cc) 734 { 735 int i; 736 737 for (i = 0; i < wi->num_dma; i++) { 738 struct mlx5e_sq_dma *dma = mlx5e_dma_get(sq, (*dma_fifo_cc)++); 739 740 mlx5e_tx_dma_unmap(sq->pdev, dma); 741 } 742 } 743 744 static void mlx5e_consume_skb(struct mlx5e_txqsq *sq, struct sk_buff *skb, 745 struct mlx5_cqe64 *cqe, int napi_budget) 746 { 747 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { 748 struct skb_shared_hwtstamps hwts = {}; 749 u64 ts = get_cqe_ts(cqe); 750 751 hwts.hwtstamp = mlx5e_cqe_ts_to_ns(sq->ptp_cyc2time, sq->clock, ts); 752 if (sq->ptpsq) 753 mlx5e_skb_cb_hwtstamp_handler(skb, MLX5E_SKB_CB_CQE_HWTSTAMP, 754 hwts.hwtstamp, sq->ptpsq->cq_stats); 755 else 756 skb_tstamp_tx(skb, &hwts); 757 } 758 759 napi_consume_skb(skb, napi_budget); 760 } 761 762 static void mlx5e_tx_wi_consume_fifo_skbs(struct mlx5e_txqsq *sq, struct mlx5e_tx_wqe_info *wi, 763 struct mlx5_cqe64 *cqe, int napi_budget) 764 { 765 int i; 766 767 for (i = 0; i < wi->num_fifo_pkts; i++) { 768 struct sk_buff *skb = mlx5e_skb_fifo_pop(&sq->db.skb_fifo); 769 770 mlx5e_consume_skb(sq, skb, cqe, napi_budget); 771 } 772 } 773 774 void mlx5e_txqsq_wake(struct mlx5e_txqsq *sq) 775 { 776 if (netif_tx_queue_stopped(sq->txq) && 777 mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, sq->stop_room) && 778 !mlx5e_ptpsq_metadata_freelist_empty(sq->ptpsq) && 779 !test_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state)) { 780 netif_tx_wake_queue(sq->txq); 781 sq->stats->wake++; 782 } 783 } 784 785 bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget) 786 { 787 struct mlx5e_sq_stats *stats; 788 struct mlx5e_txqsq *sq; 789 struct mlx5_cqe64 *cqe; 790 u32 dma_fifo_cc; 791 u32 nbytes; 792 u16 npkts; 793 u16 sqcc; 794 int i; 795 796 sq = container_of(cq, struct mlx5e_txqsq, cq); 797 798 if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state))) 799 return false; 800 801 cqe = mlx5_cqwq_get_cqe(&cq->wq); 802 if (!cqe) 803 return false; 804 805 stats = sq->stats; 806 807 npkts = 0; 808 nbytes = 0; 809 810 /* sq->cc must be updated only after mlx5_cqwq_update_db_record(), 811 * otherwise a cq overrun may occur 812 */ 813 sqcc = sq->cc; 814 815 /* avoid dirtying sq cache line every cqe */ 816 dma_fifo_cc = sq->dma_fifo_cc; 817 818 i = 0; 819 do { 820 struct mlx5e_tx_wqe_info *wi; 821 u16 wqe_counter; 822 bool last_wqe; 823 u16 ci; 824 825 mlx5_cqwq_pop(&cq->wq); 826 827 wqe_counter = be16_to_cpu(cqe->wqe_counter); 828 829 do { 830 last_wqe = (sqcc == wqe_counter); 831 832 ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sqcc); 833 wi = &sq->db.wqe_info[ci]; 834 835 sqcc += wi->num_wqebbs; 836 837 if (likely(wi->skb)) { 838 mlx5e_tx_wi_dma_unmap(sq, wi, &dma_fifo_cc); 839 mlx5e_consume_skb(sq, wi->skb, cqe, napi_budget); 840 841 npkts++; 842 nbytes += wi->num_bytes; 843 continue; 844 } 845 846 if (unlikely(mlx5e_ktls_tx_try_handle_resync_dump_comp(sq, wi, 847 &dma_fifo_cc))) 848 continue; 849 850 if (wi->num_fifo_pkts) { 851 mlx5e_tx_wi_dma_unmap(sq, wi, &dma_fifo_cc); 852 mlx5e_tx_wi_consume_fifo_skbs(sq, wi, cqe, napi_budget); 853 854 npkts += wi->num_fifo_pkts; 855 nbytes += wi->num_bytes; 856 } 857 } while (!last_wqe); 858 859 if (unlikely(get_cqe_opcode(cqe) == MLX5_CQE_REQ_ERR)) { 860 if (!test_and_set_bit(MLX5E_SQ_STATE_RECOVERING, 861 &sq->state)) { 862 mlx5e_dump_error_cqe(&sq->cq, sq->sqn, 863 (struct mlx5_err_cqe *)cqe); 864 mlx5_wq_cyc_wqe_dump(&sq->wq, ci, wi->num_wqebbs); 865 queue_work(cq->workqueue, &sq->recover_work); 866 } 867 stats->cqe_err++; 868 } 869 870 } while ((++i < MLX5E_TX_CQ_POLL_BUDGET) && (cqe = mlx5_cqwq_get_cqe(&cq->wq))); 871 872 stats->cqes += i; 873 874 mlx5_cqwq_update_db_record(&cq->wq); 875 876 /* ensure cq space is freed before enabling more cqes */ 877 wmb(); 878 879 sq->dma_fifo_cc = dma_fifo_cc; 880 sq->cc = sqcc; 881 882 netdev_tx_completed_queue(sq->txq, npkts, nbytes); 883 884 mlx5e_txqsq_wake(sq); 885 886 return (i == MLX5E_TX_CQ_POLL_BUDGET); 887 } 888 889 static void mlx5e_tx_wi_kfree_fifo_skbs(struct mlx5e_txqsq *sq, struct mlx5e_tx_wqe_info *wi) 890 { 891 int i; 892 893 for (i = 0; i < wi->num_fifo_pkts; i++) 894 dev_kfree_skb_any(mlx5e_skb_fifo_pop(&sq->db.skb_fifo)); 895 } 896 897 void mlx5e_free_txqsq_descs(struct mlx5e_txqsq *sq) 898 { 899 struct mlx5e_tx_wqe_info *wi; 900 u32 dma_fifo_cc, nbytes = 0; 901 u16 ci, sqcc, npkts = 0; 902 903 sqcc = sq->cc; 904 dma_fifo_cc = sq->dma_fifo_cc; 905 906 while (sqcc != sq->pc) { 907 ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sqcc); 908 wi = &sq->db.wqe_info[ci]; 909 910 sqcc += wi->num_wqebbs; 911 912 if (likely(wi->skb)) { 913 mlx5e_tx_wi_dma_unmap(sq, wi, &dma_fifo_cc); 914 dev_kfree_skb_any(wi->skb); 915 916 npkts++; 917 nbytes += wi->num_bytes; 918 continue; 919 } 920 921 if (unlikely(mlx5e_ktls_tx_try_handle_resync_dump_comp(sq, wi, &dma_fifo_cc))) 922 continue; 923 924 if (wi->num_fifo_pkts) { 925 mlx5e_tx_wi_dma_unmap(sq, wi, &dma_fifo_cc); 926 mlx5e_tx_wi_kfree_fifo_skbs(sq, wi); 927 928 npkts += wi->num_fifo_pkts; 929 nbytes += wi->num_bytes; 930 } 931 } 932 933 sq->dma_fifo_cc = dma_fifo_cc; 934 sq->cc = sqcc; 935 936 netdev_tx_completed_queue(sq->txq, npkts, nbytes); 937 } 938 939 #ifdef CONFIG_MLX5_CORE_IPOIB 940 static inline void 941 mlx5i_txwqe_build_datagram(struct mlx5_av *av, u32 dqpn, u32 dqkey, 942 struct mlx5_wqe_datagram_seg *dseg) 943 { 944 memcpy(&dseg->av, av, sizeof(struct mlx5_av)); 945 dseg->av.dqp_dct = cpu_to_be32(dqpn | MLX5_EXTENDED_UD_AV); 946 dseg->av.key.qkey.qkey = cpu_to_be32(dqkey); 947 } 948 949 static void mlx5i_sq_calc_wqe_attr(struct sk_buff *skb, 950 const struct mlx5e_tx_attr *attr, 951 struct mlx5e_tx_wqe_attr *wqe_attr) 952 { 953 u16 ds_cnt = sizeof(struct mlx5i_tx_wqe) / MLX5_SEND_WQE_DS; 954 u16 ds_cnt_inl = 0; 955 956 ds_cnt += !!attr->headlen + skb_shinfo(skb)->nr_frags; 957 958 if (attr->ihs) { 959 u16 inl = attr->ihs - INL_HDR_START_SZ; 960 961 ds_cnt_inl = DIV_ROUND_UP(inl, MLX5_SEND_WQE_DS); 962 ds_cnt += ds_cnt_inl; 963 } 964 965 *wqe_attr = (struct mlx5e_tx_wqe_attr) { 966 .ds_cnt = ds_cnt, 967 .ds_cnt_inl = ds_cnt_inl, 968 .num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS), 969 }; 970 } 971 972 void mlx5i_sq_xmit(struct mlx5e_txqsq *sq, struct sk_buff *skb, 973 struct mlx5_av *av, u32 dqpn, u32 dqkey, bool xmit_more) 974 { 975 struct mlx5e_tx_wqe_attr wqe_attr; 976 struct mlx5e_tx_attr attr; 977 struct mlx5i_tx_wqe *wqe; 978 979 struct mlx5_wqe_datagram_seg *datagram; 980 struct mlx5_wqe_ctrl_seg *cseg; 981 struct mlx5_wqe_eth_seg *eseg; 982 struct mlx5_wqe_data_seg *dseg; 983 struct mlx5e_tx_wqe_info *wi; 984 985 struct mlx5e_sq_stats *stats = sq->stats; 986 int num_dma; 987 u16 pi; 988 989 mlx5e_sq_xmit_prepare(sq, skb, NULL, &attr); 990 mlx5i_sq_calc_wqe_attr(skb, &attr, &wqe_attr); 991 992 pi = mlx5e_txqsq_get_next_pi(sq, wqe_attr.num_wqebbs); 993 wqe = MLX5I_SQ_FETCH_WQE(sq, pi); 994 995 stats->xmit_more += xmit_more; 996 997 /* fill wqe */ 998 wi = &sq->db.wqe_info[pi]; 999 cseg = &wqe->ctrl; 1000 datagram = &wqe->datagram; 1001 eseg = &wqe->eth; 1002 dseg = wqe->data; 1003 1004 mlx5i_txwqe_build_datagram(av, dqpn, dqkey, datagram); 1005 1006 mlx5e_txwqe_build_eseg_csum(sq, skb, NULL, eseg); 1007 1008 eseg->mss = attr.mss; 1009 1010 if (attr.ihs) { 1011 if (unlikely(attr.hopbyhop)) { 1012 struct ipv6hdr *h6; 1013 1014 /* remove the HBH header. 1015 * Layout: [Ethernet header][IPv6 header][HBH][TCP header] 1016 */ 1017 unsafe_memcpy(eseg->inline_hdr.start, skb->data, 1018 ETH_HLEN + sizeof(*h6), 1019 MLX5_UNSAFE_MEMCPY_DISCLAIMER); 1020 h6 = (struct ipv6hdr *)((char *)eseg->inline_hdr.start + ETH_HLEN); 1021 h6->nexthdr = IPPROTO_TCP; 1022 /* Copy the TCP header after the IPv6 one */ 1023 unsafe_memcpy(h6 + 1, 1024 skb->data + ETH_HLEN + sizeof(*h6) + 1025 sizeof(struct hop_jumbo_hdr), 1026 tcp_hdrlen(skb), 1027 MLX5_UNSAFE_MEMCPY_DISCLAIMER); 1028 /* Leave ipv6 payload_len set to 0, as LSO v2 specs request. */ 1029 } else { 1030 unsafe_memcpy(eseg->inline_hdr.start, skb->data, 1031 attr.ihs, 1032 MLX5_UNSAFE_MEMCPY_DISCLAIMER); 1033 } 1034 eseg->inline_hdr.sz = cpu_to_be16(attr.ihs); 1035 dseg += wqe_attr.ds_cnt_inl; 1036 } 1037 1038 num_dma = mlx5e_txwqe_build_dsegs(sq, skb, skb->data + attr.ihs + attr.hopbyhop, 1039 attr.headlen, dseg); 1040 if (unlikely(num_dma < 0)) 1041 goto err_drop; 1042 1043 mlx5e_txwqe_complete(sq, skb, &attr, &wqe_attr, num_dma, wi, cseg, eseg, xmit_more); 1044 1045 return; 1046 1047 err_drop: 1048 stats->dropped++; 1049 dev_kfree_skb_any(skb); 1050 mlx5e_tx_flush(sq); 1051 } 1052 #endif 1053