136 	struct xdp_test_data *xdp = arg;  in xdp_test_run_init_page()  local
141 	orig_ctx = xdp->orig_ctx;  in xdp_test_run_init_page()
151 	xdp_init_buff(new_ctx, TEST_XDP_FRAME_SIZE, &xdp->rxq);  in xdp_test_run_init_page()
161 static int xdp_test_run_setup(struct xdp_test_data *xdp, struct xdp_buff *orig_ctx)  in xdp_test_run_setup()  argument
168 		.pool_size = xdp->batch_size,  in xdp_test_run_setup()
171 		.init_arg = xdp,  in xdp_test_run_setup()
174 	xdp->frames = kvmalloc_array(xdp->batch_size, sizeof(void *), GFP_KERNEL);  in xdp_test_run_setup()
175 	if (!xdp->frames)  in xdp_test_run_setup()
178 	xdp->skbs = kvmalloc_array(xdp->batch_size, sizeof(void *), GFP_KERNEL);  in xdp_test_run_setup()
179 	if (!xdp->skbs)  in xdp_test_run_setup()
189 	err = xdp_reg_mem_model(&xdp->mem, MEM_TYPE_PAGE_POOL, pp);  in xdp_test_run_setup()
193 	xdp->pp = pp;  in xdp_test_run_setup()
198 	xdp_rxq_info_reg(&xdp->rxq, orig_ctx->rxq->dev, 0, 0);  in xdp_test_run_setup()
199 	xdp->rxq.mem.type = MEM_TYPE_PAGE_POOL;  in xdp_test_run_setup()
200 	xdp->rxq.mem.id = pp->xdp_mem_id;  in xdp_test_run_setup()
201 	xdp->dev = orig_ctx->rxq->dev;  in xdp_test_run_setup()
202 	xdp->orig_ctx = orig_ctx;  in xdp_test_run_setup()
209 	kvfree(xdp->skbs);  in xdp_test_run_setup()
211 	kvfree(xdp->frames);  in xdp_test_run_setup()
215 static void xdp_test_run_teardown(struct xdp_test_data *xdp)  in xdp_test_run_teardown()  argument
217 	xdp_unreg_mem_model(&xdp->mem);  in xdp_test_run_teardown()
218 	page_pool_destroy(xdp->pp);  in xdp_test_run_teardown()
219 	kfree(xdp->frames);  in xdp_test_run_teardown()
220 	kfree(xdp->skbs);  in xdp_test_run_teardown()
285 static int xdp_test_run_batch(struct xdp_test_data *xdp, struct bpf_prog *prog,  in xdp_test_run_batch()  argument
290 	struct xdp_frame **frames = xdp->frames;  in xdp_test_run_batch()
298 	batch_sz = min_t(u32, repeat, xdp->batch_size);  in xdp_test_run_batch()
306 		page = page_pool_dev_alloc_pages(xdp->pp);  in xdp_test_run_batch()
316 		xdp->frame_cnt++;  in xdp_test_run_batch()
335 			ri->tgt_index = xdp->dev->ifindex;  in xdp_test_run_batch()
341 			ret = xdp_do_redirect_frame(xdp->dev, ctx, frm, prog);  in xdp_test_run_batch()
361 		ret = xdp_recv_frames(frames, nframes, xdp->skbs, xdp->dev);  in xdp_test_run_batch()
376 	struct xdp_test_data xdp = { .batch_size = batch_size };  in bpf_test_run_xdp_live()  local
383 	ret = xdp_test_run_setup(&xdp, ctx);  in bpf_test_run_xdp_live()
389 		xdp.frame_cnt = 0;  in bpf_test_run_xdp_live()
390 		ret = xdp_test_run_batch(&xdp, prog, repeat - t.i);  in bpf_test_run_xdp_live()
393 	} while (bpf_test_timer_continue(&t, xdp.frame_cnt, repeat, &ret, time));  in bpf_test_run_xdp_live()
396 	xdp_test_run_teardown(&xdp);  in bpf_test_run_xdp_live()
401 			u32 *retval, u32 *time, bool xdp)  in bpf_test_run()  argument
431 		if (xdp)  in bpf_test_run()
1143 static int xdp_convert_md_to_buff(struct xdp_md *xdp_md, struct xdp_buff *xdp)  in xdp_convert_md_to_buff()  argument
1175 		xdp->rxq = &rxqueue->xdp_rxq;  in xdp_convert_md_to_buff()
1181 	xdp->data = xdp->data_meta + xdp_md->data;  in xdp_convert_md_to_buff()
1189 static void xdp_convert_buff_to_md(struct xdp_buff *xdp, struct xdp_md *xdp_md)  in xdp_convert_buff_to_md()  argument
1194 	xdp_md->data = xdp->data - xdp->data_meta;  in xdp_convert_buff_to_md()
1195 	xdp_md->data_end = xdp->data_end - xdp->data_meta;  in xdp_convert_buff_to_md()
1198 		dev_put(xdp->rxq->dev);  in xdp_convert_buff_to_md()
1213 	struct xdp_buff xdp = {};  in bpf_prog_test_run_xdp()  local
1270 	xdp_init_buff(&xdp, rxqueue->xdp_rxq.frag_size, &rxqueue->xdp_rxq);  in bpf_prog_test_run_xdp()
1271 	xdp_prepare_buff(&xdp, data, headroom, size, true);  in bpf_prog_test_run_xdp()
1272 	sinfo = xdp_get_shared_info_from_buff(&xdp);  in bpf_prog_test_run_xdp()
1274 	ret = xdp_convert_md_to_buff(ctx, &xdp);  in bpf_prog_test_run_xdp()
1311 		xdp_buff_set_frags_flag(&xdp);  in bpf_prog_test_run_xdp()
1318 		ret = bpf_test_run_xdp_live(prog, &xdp, repeat, batch_size, &duration);  in bpf_prog_test_run_xdp()
1320 		ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration, true);  in bpf_prog_test_run_xdp()
1325 	xdp_convert_buff_to_md(&xdp, ctx);  in bpf_prog_test_run_xdp()
1329 	size = xdp.data_end - xdp.data_meta + sinfo->xdp_frags_size;  in bpf_prog_test_run_xdp()
1330 	ret = bpf_test_finish(kattr, uattr, xdp.data_meta, sinfo, size,  in bpf_prog_test_run_xdp()