1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2019 Mellanox Technologies. */
3 
4 #include "rx.h"
5 #include "en/xdp.h"
6 #include <net/xdp_sock_drv.h>
7 #include <linux/filter.h>
8 
9 /* RX data path */
10 
11 static struct mlx5e_xdp_buff *xsk_buff_to_mxbuf(struct xdp_buff *xdp)
12 {
13 	/* mlx5e_xdp_buff shares its layout with xdp_buff_xsk
14 	 * and private mlx5e_xdp_buff fields fall into xdp_buff_xsk->cb
15 	 */
16 	return (struct mlx5e_xdp_buff *)xdp;
17 }
18 
19 int mlx5e_xsk_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
20 {
21 	struct mlx5e_mpw_info *wi = mlx5e_get_mpw_info(rq, ix);
22 	struct mlx5e_icosq *icosq = rq->icosq;
23 	struct mlx5_wq_cyc *wq = &icosq->wq;
24 	struct mlx5e_umr_wqe *umr_wqe;
25 	struct xdp_buff **xsk_buffs;
26 	int batch, i;
27 	u32 offset; /* 17-bit value with MTT. */
28 	u16 pi;
29 
30 	if (unlikely(!xsk_buff_can_alloc(rq->xsk_pool, rq->mpwqe.pages_per_wqe)))
31 		goto err;
32 
33 	XSK_CHECK_PRIV_TYPE(struct mlx5e_xdp_buff);
34 	xsk_buffs = (struct xdp_buff **)wi->alloc_units.xsk_buffs;
35 	batch = xsk_buff_alloc_batch(rq->xsk_pool, xsk_buffs,
36 				     rq->mpwqe.pages_per_wqe);
37 
38 	/* If batch < pages_per_wqe, either:
39 	 * 1. Some (or all) descriptors were invalid.
40 	 * 2. dma_need_sync is true, and it fell back to allocating one frame.
41 	 * In either case, try to continue allocating frames one by one, until
42 	 * the first error, which will mean there are no more valid descriptors.
43 	 */
44 	for (; batch < rq->mpwqe.pages_per_wqe; batch++) {
45 		xsk_buffs[batch] = xsk_buff_alloc(rq->xsk_pool);
46 		if (unlikely(!xsk_buffs[batch]))
47 			goto err_reuse_batch;
48 	}
49 
50 	pi = mlx5e_icosq_get_next_pi(icosq, rq->mpwqe.umr_wqebbs);
51 	umr_wqe = mlx5_wq_cyc_get_wqe(wq, pi);
52 	memcpy(umr_wqe, &rq->mpwqe.umr_wqe, sizeof(struct mlx5e_umr_wqe));
53 
54 	if (likely(rq->mpwqe.umr_mode == MLX5E_MPWRQ_UMR_MODE_ALIGNED)) {
55 		for (i = 0; i < batch; i++) {
56 			struct mlx5e_xdp_buff *mxbuf = xsk_buff_to_mxbuf(xsk_buffs[i]);
57 			dma_addr_t addr = xsk_buff_xdp_get_frame_dma(xsk_buffs[i]);
58 
59 			umr_wqe->inline_mtts[i] = (struct mlx5_mtt) {
60 				.ptag = cpu_to_be64(addr | MLX5_EN_WR),
61 			};
62 			mxbuf->rq = rq;
63 		}
64 	} else if (unlikely(rq->mpwqe.umr_mode == MLX5E_MPWRQ_UMR_MODE_UNALIGNED)) {
65 		for (i = 0; i < batch; i++) {
66 			struct mlx5e_xdp_buff *mxbuf = xsk_buff_to_mxbuf(xsk_buffs[i]);
67 			dma_addr_t addr = xsk_buff_xdp_get_frame_dma(xsk_buffs[i]);
68 
69 			umr_wqe->inline_ksms[i] = (struct mlx5_ksm) {
70 				.key = rq->mkey_be,
71 				.va = cpu_to_be64(addr),
72 			};
73 			mxbuf->rq = rq;
74 		}
75 	} else if (likely(rq->mpwqe.umr_mode == MLX5E_MPWRQ_UMR_MODE_TRIPLE)) {
76 		u32 mapping_size = 1 << (rq->mpwqe.page_shift - 2);
77 
78 		for (i = 0; i < batch; i++) {
79 			struct mlx5e_xdp_buff *mxbuf = xsk_buff_to_mxbuf(xsk_buffs[i]);
80 			dma_addr_t addr = xsk_buff_xdp_get_frame_dma(xsk_buffs[i]);
81 
82 			umr_wqe->inline_ksms[i << 2] = (struct mlx5_ksm) {
83 				.key = rq->mkey_be,
84 				.va = cpu_to_be64(addr),
85 			};
86 			umr_wqe->inline_ksms[(i << 2) + 1] = (struct mlx5_ksm) {
87 				.key = rq->mkey_be,
88 				.va = cpu_to_be64(addr + mapping_size),
89 			};
90 			umr_wqe->inline_ksms[(i << 2) + 2] = (struct mlx5_ksm) {
91 				.key = rq->mkey_be,
92 				.va = cpu_to_be64(addr + mapping_size * 2),
93 			};
94 			umr_wqe->inline_ksms[(i << 2) + 3] = (struct mlx5_ksm) {
95 				.key = rq->mkey_be,
96 				.va = cpu_to_be64(rq->wqe_overflow.addr),
97 			};
98 			mxbuf->rq = rq;
99 		}
100 	} else {
101 		__be32 pad_size = cpu_to_be32((1 << rq->mpwqe.page_shift) -
102 					      rq->xsk_pool->chunk_size);
103 		__be32 frame_size = cpu_to_be32(rq->xsk_pool->chunk_size);
104 
105 		for (i = 0; i < batch; i++) {
106 			struct mlx5e_xdp_buff *mxbuf = xsk_buff_to_mxbuf(xsk_buffs[i]);
107 			dma_addr_t addr = xsk_buff_xdp_get_frame_dma(xsk_buffs[i]);
108 
109 			umr_wqe->inline_klms[i << 1] = (struct mlx5_klm) {
110 				.key = rq->mkey_be,
111 				.va = cpu_to_be64(addr),
112 				.bcount = frame_size,
113 			};
114 			umr_wqe->inline_klms[(i << 1) + 1] = (struct mlx5_klm) {
115 				.key = rq->mkey_be,
116 				.va = cpu_to_be64(rq->wqe_overflow.addr),
117 				.bcount = pad_size,
118 			};
119 			mxbuf->rq = rq;
120 		}
121 	}
122 
123 	bitmap_zero(wi->skip_release_bitmap, rq->mpwqe.pages_per_wqe);
124 	wi->consumed_strides = 0;
125 
126 	umr_wqe->ctrl.opmod_idx_opcode =
127 		cpu_to_be32((icosq->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) | MLX5_OPCODE_UMR);
128 
129 	/* Optimized for speed: keep in sync with mlx5e_mpwrq_umr_entry_size. */
130 	offset = ix * rq->mpwqe.mtts_per_wqe;
131 	if (likely(rq->mpwqe.umr_mode == MLX5E_MPWRQ_UMR_MODE_ALIGNED))
132 		offset = offset * sizeof(struct mlx5_mtt) / MLX5_OCTWORD;
133 	else if (unlikely(rq->mpwqe.umr_mode == MLX5E_MPWRQ_UMR_MODE_OVERSIZED))
134 		offset = offset * sizeof(struct mlx5_klm) * 2 / MLX5_OCTWORD;
135 	else if (unlikely(rq->mpwqe.umr_mode == MLX5E_MPWRQ_UMR_MODE_TRIPLE))
136 		offset = offset * sizeof(struct mlx5_ksm) * 4 / MLX5_OCTWORD;
137 	umr_wqe->uctrl.xlt_offset = cpu_to_be16(offset);
138 
139 	icosq->db.wqe_info[pi] = (struct mlx5e_icosq_wqe_info) {
140 		.wqe_type = MLX5E_ICOSQ_WQE_UMR_RX,
141 		.num_wqebbs = rq->mpwqe.umr_wqebbs,
142 		.umr.rq = rq,
143 	};
144 
145 	icosq->pc += rq->mpwqe.umr_wqebbs;
146 
147 	icosq->doorbell_cseg = &umr_wqe->ctrl;
148 
149 	return 0;
150 
151 err_reuse_batch:
152 	while (--batch >= 0)
153 		xsk_buff_free(xsk_buffs[batch]);
154 
155 err:
156 	rq->stats->buff_alloc_err++;
157 	return -ENOMEM;
158 }
159 
160 int mlx5e_xsk_alloc_rx_wqes_batched(struct mlx5e_rq *rq, u16 ix, int wqe_bulk)
161 {
162 	struct mlx5_wq_cyc *wq = &rq->wqe.wq;
163 	struct xdp_buff **buffs;
164 	u32 contig, alloc;
165 	int i;
166 
167 	/* Each rq->wqe.frags->xskp is 1:1 mapped to an element inside the
168 	 * rq->wqe.alloc_units->xsk_buffs array allocated here.
169 	 */
170 	buffs = rq->wqe.alloc_units->xsk_buffs;
171 	contig = mlx5_wq_cyc_get_size(wq) - ix;
172 	if (wqe_bulk <= contig) {
173 		alloc = xsk_buff_alloc_batch(rq->xsk_pool, buffs + ix, wqe_bulk);
174 	} else {
175 		alloc = xsk_buff_alloc_batch(rq->xsk_pool, buffs + ix, contig);
176 		if (likely(alloc == contig))
177 			alloc += xsk_buff_alloc_batch(rq->xsk_pool, buffs, wqe_bulk - contig);
178 	}
179 
180 	for (i = 0; i < alloc; i++) {
181 		int j = mlx5_wq_cyc_ctr2ix(wq, ix + i);
182 		struct mlx5e_wqe_frag_info *frag;
183 		struct mlx5e_rx_wqe_cyc *wqe;
184 		dma_addr_t addr;
185 
186 		wqe = mlx5_wq_cyc_get_wqe(wq, j);
187 		/* Assumes log_num_frags == 0. */
188 		frag = &rq->wqe.frags[j];
189 
190 		addr = xsk_buff_xdp_get_frame_dma(*frag->xskp);
191 		wqe->data[0].addr = cpu_to_be64(addr + rq->buff.headroom);
192 		frag->flags &= ~BIT(MLX5E_WQE_FRAG_SKIP_RELEASE);
193 	}
194 
195 	return alloc;
196 }
197 
198 int mlx5e_xsk_alloc_rx_wqes(struct mlx5e_rq *rq, u16 ix, int wqe_bulk)
199 {
200 	struct mlx5_wq_cyc *wq = &rq->wqe.wq;
201 	int i;
202 
203 	for (i = 0; i < wqe_bulk; i++) {
204 		int j = mlx5_wq_cyc_ctr2ix(wq, ix + i);
205 		struct mlx5e_wqe_frag_info *frag;
206 		struct mlx5e_rx_wqe_cyc *wqe;
207 		dma_addr_t addr;
208 
209 		wqe = mlx5_wq_cyc_get_wqe(wq, j);
210 		/* Assumes log_num_frags == 0. */
211 		frag = &rq->wqe.frags[j];
212 
213 		*frag->xskp = xsk_buff_alloc(rq->xsk_pool);
214 		if (unlikely(!*frag->xskp))
215 			return i;
216 
217 		addr = xsk_buff_xdp_get_frame_dma(*frag->xskp);
218 		wqe->data[0].addr = cpu_to_be64(addr + rq->buff.headroom);
219 		frag->flags &= ~BIT(MLX5E_WQE_FRAG_SKIP_RELEASE);
220 	}
221 
222 	return wqe_bulk;
223 }
224 
225 static struct sk_buff *mlx5e_xsk_construct_skb(struct mlx5e_rq *rq, struct xdp_buff *xdp)
226 {
227 	u32 totallen = xdp->data_end - xdp->data_meta;
228 	u32 metalen = xdp->data - xdp->data_meta;
229 	struct sk_buff *skb;
230 
231 	skb = napi_alloc_skb(rq->cq.napi, totallen);
232 	if (unlikely(!skb)) {
233 		rq->stats->buff_alloc_err++;
234 		return NULL;
235 	}
236 
237 	skb_put_data(skb, xdp->data_meta, totallen);
238 
239 	if (metalen) {
240 		skb_metadata_set(skb, metalen);
241 		__skb_pull(skb, metalen);
242 	}
243 
244 	return skb;
245 }
246 
247 struct sk_buff *mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq,
248 						    struct mlx5e_mpw_info *wi,
249 						    struct mlx5_cqe64 *cqe,
250 						    u16 cqe_bcnt,
251 						    u32 head_offset,
252 						    u32 page_idx)
253 {
254 	struct mlx5e_xdp_buff *mxbuf = xsk_buff_to_mxbuf(wi->alloc_units.xsk_buffs[page_idx]);
255 	struct bpf_prog *prog;
256 
257 	/* Check packet size. Note LRO doesn't use linear SKB */
258 	if (unlikely(cqe_bcnt > rq->hw_mtu)) {
259 		rq->stats->oversize_pkts_sw_drop++;
260 		return NULL;
261 	}
262 
263 	/* head_offset is not used in this function, because xdp->data and the
264 	 * DMA address point directly to the necessary place. Furthermore, in
265 	 * the current implementation, UMR pages are mapped to XSK frames, so
266 	 * head_offset should always be 0.
267 	 */
268 	WARN_ON_ONCE(head_offset);
269 
270 	/* mxbuf->rq is set on allocation, but cqe is per-packet so set it here */
271 	mxbuf->cqe = cqe;
272 	xsk_buff_set_size(&mxbuf->xdp, cqe_bcnt);
273 	xsk_buff_dma_sync_for_cpu(&mxbuf->xdp, rq->xsk_pool);
274 	net_prefetch(mxbuf->xdp.data);
275 
276 	/* Possible flows:
277 	 * - XDP_REDIRECT to XSKMAP:
278 	 *   The page is owned by the userspace from now.
279 	 * - XDP_TX and other XDP_REDIRECTs:
280 	 *   The page was returned by ZCA and recycled.
281 	 * - XDP_DROP:
282 	 *   Recycle the page.
283 	 * - XDP_PASS:
284 	 *   Allocate an SKB, copy the data and recycle the page.
285 	 *
286 	 * Pages to be recycled go to the Reuse Ring on MPWQE deallocation. Its
287 	 * size is the same as the Driver RX Ring's size, and pages for WQEs are
288 	 * allocated first from the Reuse Ring, so it has enough space.
289 	 */
290 
291 	prog = rcu_dereference(rq->xdp_prog);
292 	if (likely(prog && mlx5e_xdp_handle(rq, prog, mxbuf))) {
293 		if (likely(__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)))
294 			__set_bit(page_idx, wi->skip_release_bitmap); /* non-atomic */
295 		return NULL; /* page/packet was consumed by XDP */
296 	}
297 
298 	/* XDP_PASS: copy the data from the UMEM to a new SKB and reuse the
299 	 * frame. On SKB allocation failure, NULL is returned.
300 	 */
301 	return mlx5e_xsk_construct_skb(rq, &mxbuf->xdp);
302 }
303 
304 struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq,
305 					      struct mlx5e_wqe_frag_info *wi,
306 					      struct mlx5_cqe64 *cqe,
307 					      u32 cqe_bcnt)
308 {
309 	struct mlx5e_xdp_buff *mxbuf = xsk_buff_to_mxbuf(*wi->xskp);
310 	struct bpf_prog *prog;
311 
312 	/* wi->offset is not used in this function, because xdp->data and the
313 	 * DMA address point directly to the necessary place. Furthermore, the
314 	 * XSK allocator allocates frames per packet, instead of pages, so
315 	 * wi->offset should always be 0.
316 	 */
317 	WARN_ON_ONCE(wi->offset);
318 
319 	/* mxbuf->rq is set on allocation, but cqe is per-packet so set it here */
320 	mxbuf->cqe = cqe;
321 	xsk_buff_set_size(&mxbuf->xdp, cqe_bcnt);
322 	xsk_buff_dma_sync_for_cpu(&mxbuf->xdp, rq->xsk_pool);
323 	net_prefetch(mxbuf->xdp.data);
324 
325 	prog = rcu_dereference(rq->xdp_prog);
326 	if (likely(prog && mlx5e_xdp_handle(rq, prog, mxbuf)))
327 		return NULL; /* page/packet was consumed by XDP */
328 
329 	/* XDP_PASS: copy the data from the UMEM to a new SKB. The frame reuse
330 	 * will be handled by mlx5e_free_rx_wqe.
331 	 * On SKB allocation failure, NULL is returned.
332 	 */
333 	return mlx5e_xsk_construct_skb(rq, &mxbuf->xdp);
334 }
335