1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2019 Mellanox Technologies. */ 3 4 #include "rx.h" 5 #include "en/xdp.h" 6 #include <net/xdp_sock.h> 7 8 /* RX data path */ 9 10 bool mlx5e_xsk_pages_enough_umem(struct mlx5e_rq *rq, int count) 11 { 12 /* Check in advance that we have enough frames, instead of allocating 13 * one-by-one, failing and moving frames to the Reuse Ring. 14 */ 15 return xsk_umem_has_addrs_rq(rq->umem, count); 16 } 17 18 int mlx5e_xsk_page_alloc_umem(struct mlx5e_rq *rq, 19 struct mlx5e_dma_info *dma_info) 20 { 21 struct xdp_umem *umem = rq->umem; 22 u64 handle; 23 24 if (!xsk_umem_peek_addr_rq(umem, &handle)) 25 return -ENOMEM; 26 27 dma_info->xsk.handle = xsk_umem_adjust_offset(umem, handle, 28 rq->buff.umem_headroom); 29 dma_info->xsk.data = xdp_umem_get_data(umem, dma_info->xsk.handle); 30 31 /* No need to add headroom to the DMA address. In striding RQ case, we 32 * just provide pages for UMR, and headroom is counted at the setup 33 * stage when creating a WQE. In non-striding RQ case, headroom is 34 * accounted in mlx5e_alloc_rx_wqe. 35 */ 36 dma_info->addr = xdp_umem_get_dma(umem, handle); 37 38 xsk_umem_discard_addr_rq(umem); 39 40 dma_sync_single_for_device(rq->pdev, dma_info->addr, PAGE_SIZE, 41 DMA_BIDIRECTIONAL); 42 43 return 0; 44 } 45 46 static inline void mlx5e_xsk_recycle_frame(struct mlx5e_rq *rq, u64 handle) 47 { 48 xsk_umem_fq_reuse(rq->umem, handle & rq->umem->chunk_mask); 49 } 50 51 /* XSKRQ uses pages from UMEM, they must not be released. They are returned to 52 * the userspace if possible, and if not, this function is called to reuse them 53 * in the driver. 54 */ 55 void mlx5e_xsk_page_release(struct mlx5e_rq *rq, 56 struct mlx5e_dma_info *dma_info) 57 { 58 mlx5e_xsk_recycle_frame(rq, dma_info->xsk.handle); 59 } 60 61 /* Return a frame back to the hardware to fill in again. It is used by XDP when 62 * the XDP program returns XDP_TX or XDP_REDIRECT not to an XSKMAP. 63 */ 64 void mlx5e_xsk_zca_free(struct zero_copy_allocator *zca, unsigned long handle) 65 { 66 struct mlx5e_rq *rq = container_of(zca, struct mlx5e_rq, zca); 67 68 mlx5e_xsk_recycle_frame(rq, handle); 69 } 70 71 static struct sk_buff *mlx5e_xsk_construct_skb(struct mlx5e_rq *rq, void *data, 72 u32 cqe_bcnt) 73 { 74 struct sk_buff *skb; 75 76 skb = napi_alloc_skb(rq->cq.napi, cqe_bcnt); 77 if (unlikely(!skb)) { 78 rq->stats->buff_alloc_err++; 79 return NULL; 80 } 81 82 skb_put_data(skb, data, cqe_bcnt); 83 84 return skb; 85 } 86 87 struct sk_buff *mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, 88 struct mlx5e_mpw_info *wi, 89 u16 cqe_bcnt, 90 u32 head_offset, 91 u32 page_idx) 92 { 93 struct mlx5e_dma_info *di = &wi->umr.dma_info[page_idx]; 94 u16 rx_headroom = rq->buff.headroom - rq->buff.umem_headroom; 95 u32 cqe_bcnt32 = cqe_bcnt; 96 void *va, *data; 97 u32 frag_size; 98 bool consumed; 99 100 /* Check packet size. Note LRO doesn't use linear SKB */ 101 if (unlikely(cqe_bcnt > rq->hw_mtu)) { 102 rq->stats->oversize_pkts_sw_drop++; 103 return NULL; 104 } 105 106 /* head_offset is not used in this function, because di->xsk.data and 107 * di->addr point directly to the necessary place. Furthermore, in the 108 * current implementation, UMR pages are mapped to XSK frames, so 109 * head_offset should always be 0. 110 */ 111 WARN_ON_ONCE(head_offset); 112 113 va = di->xsk.data; 114 data = va + rx_headroom; 115 frag_size = rq->buff.headroom + cqe_bcnt32; 116 117 dma_sync_single_for_cpu(rq->pdev, di->addr, frag_size, DMA_BIDIRECTIONAL); 118 prefetch(data); 119 120 rcu_read_lock(); 121 consumed = mlx5e_xdp_handle(rq, di, va, &rx_headroom, &cqe_bcnt32, true); 122 rcu_read_unlock(); 123 124 /* Possible flows: 125 * - XDP_REDIRECT to XSKMAP: 126 * The page is owned by the userspace from now. 127 * - XDP_TX and other XDP_REDIRECTs: 128 * The page was returned by ZCA and recycled. 129 * - XDP_DROP: 130 * Recycle the page. 131 * - XDP_PASS: 132 * Allocate an SKB, copy the data and recycle the page. 133 * 134 * Pages to be recycled go to the Reuse Ring on MPWQE deallocation. Its 135 * size is the same as the Driver RX Ring's size, and pages for WQEs are 136 * allocated first from the Reuse Ring, so it has enough space. 137 */ 138 139 if (likely(consumed)) { 140 if (likely(__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags))) 141 __set_bit(page_idx, wi->xdp_xmit_bitmap); /* non-atomic */ 142 return NULL; /* page/packet was consumed by XDP */ 143 } 144 145 /* XDP_PASS: copy the data from the UMEM to a new SKB and reuse the 146 * frame. On SKB allocation failure, NULL is returned. 147 */ 148 return mlx5e_xsk_construct_skb(rq, data, cqe_bcnt32); 149 } 150 151 struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq, 152 struct mlx5_cqe64 *cqe, 153 struct mlx5e_wqe_frag_info *wi, 154 u32 cqe_bcnt) 155 { 156 struct mlx5e_dma_info *di = wi->di; 157 u16 rx_headroom = rq->buff.headroom - rq->buff.umem_headroom; 158 void *va, *data; 159 bool consumed; 160 u32 frag_size; 161 162 /* wi->offset is not used in this function, because di->xsk.data and 163 * di->addr point directly to the necessary place. Furthermore, in the 164 * current implementation, one page = one packet = one frame, so 165 * wi->offset should always be 0. 166 */ 167 WARN_ON_ONCE(wi->offset); 168 169 va = di->xsk.data; 170 data = va + rx_headroom; 171 frag_size = rq->buff.headroom + cqe_bcnt; 172 173 dma_sync_single_for_cpu(rq->pdev, di->addr, frag_size, DMA_BIDIRECTIONAL); 174 prefetch(data); 175 176 if (unlikely(get_cqe_opcode(cqe) != MLX5_CQE_RESP_SEND)) { 177 rq->stats->wqe_err++; 178 return NULL; 179 } 180 181 rcu_read_lock(); 182 consumed = mlx5e_xdp_handle(rq, di, va, &rx_headroom, &cqe_bcnt, true); 183 rcu_read_unlock(); 184 185 if (likely(consumed)) 186 return NULL; /* page/packet was consumed by XDP */ 187 188 /* XDP_PASS: copy the data from the UMEM to a new SKB. The frame reuse 189 * will be handled by mlx5e_put_rx_frag. 190 * On SKB allocation failure, NULL is returned. 191 */ 192 return mlx5e_xsk_construct_skb(rq, data, cqe_bcnt); 193 } 194