/linux/net/core/ |
H A D | xdp.c | 422 __xdp_return(xdpf->data, &xdpf->mem, false, NULL); in xdp_return_frame() 441 __xdp_return(xdpf->data, &xdpf->mem, true, NULL); in xdp_return_frame_rx_napi() 561 xdpf = addr; in xdp_convert_zc_to_xdp_frame() 562 memset(xdpf, 0, sizeof(*xdpf)); in xdp_convert_zc_to_xdp_frame() 564 addr += sizeof(*xdpf); in xdp_convert_zc_to_xdp_frame() 570 xdpf->headroom = 0; in xdp_convert_zc_to_xdp_frame() 576 return xdpf; in xdp_convert_zc_to_xdp_frame() 611 headroom = sizeof(*xdpf) + xdpf->headroom; in __xdp_build_skb_from_frame() 625 if (xdpf->metasize) in __xdp_build_skb_from_frame() 647 xdp_scrub_frame(xdpf); in __xdp_build_skb_from_frame() [all …]
|
/linux/kernel/bpf/ |
H A D | cpumap.c | 192 rxq.dev = xdpf->dev_rx; in cpu_map_bpf_prog_run_xdp() 193 rxq.mem = xdpf->mem; in cpu_map_bpf_prog_run_xdp() 203 xdp_return_frame(xdpf); in cpu_map_bpf_prog_run_xdp() 206 frames[nframes++] = xdpf; in cpu_map_bpf_prog_run_xdp() 214 xdp_return_frame(xdpf); in cpu_map_bpf_prog_run_xdp() 224 xdp_return_frame(xdpf); in cpu_map_bpf_prog_run_xdp() 349 xdpf->dev_rx); in cpu_map_kthread_run() 351 xdp_return_frame(xdpf); in cpu_map_kthread_run() 724 bq->q[bq->count++] = xdpf; in bq_enqueue() 737 xdpf->dev_rx = dev_rx; in cpu_map_enqueue() [all …]
|
H A D | devmap.c | 355 xdp_return_frame_rx_napi(xdpf); in dev_map_bpf_prog_run() 357 frames[nframes++] = xdpf; in dev_map_bpf_prog_run() 366 xdp_return_frame_rx_napi(xdpf); in dev_map_bpf_prog_run() 387 prefetch(xdpf); in bq_xmit_all() 476 bq->q[bq->count++] = xdpf; in bq_enqueue() 489 xdp_frame_has_frags(xdpf))) in __xdp_enqueue() 554 xdp_frame_has_frags(xdpf))) in is_valid_dst() 565 struct xdp_frame *xdpf) in dev_map_enqueue_clone() argument 569 nxdpf = xdpf_clone(xdpf); in dev_map_enqueue_clone() 623 if (!is_valid_dst(dst, xdpf)) in dev_map_enqueue_multi() [all …]
|
/linux/drivers/net/ethernet/amazon/ena/ |
H A D | ena_xdp.c | 13 if (likely(tx_info->xdpf)) in validate_xdp_req_id() 21 struct xdp_frame *xdpf, in ena_xdp_tx_map_frame() argument 31 tx_info->xdpf = xdpf; in ena_xdp_tx_map_frame() 32 data = tx_info->xdpf->data; in ena_xdp_tx_map_frame() 33 size = tx_info->xdpf->len; in ena_xdp_tx_map_frame() 101 xdpf->len); in ena_xdp_xmit_frame() 116 tx_info->xdpf = NULL; in ena_xdp_xmit_frame() 381 struct xdp_frame *xdpf; in ena_clean_xdp_irq() local 400 xdpf = tx_info->xdpf; in ena_clean_xdp_irq() 401 tx_info->xdpf = NULL; in ena_clean_xdp_irq() [all …]
|
H A D | ena_xdp.h | 40 struct xdp_frame *xdpf, 87 struct xdp_frame *xdpf; in ena_xdp_execute() local 96 xdpf = xdp_convert_buff_to_frame(xdp); in ena_xdp_execute() 97 if (unlikely(!xdpf)) { in ena_xdp_execute() 110 if (ena_xdp_xmit_frame(xdp_ring, rx_ring->adapter, xdpf, in ena_xdp_execute() 112 xdp_return_frame(xdpf); in ena_xdp_execute()
|
/linux/drivers/net/vmxnet3/ |
H A D | vmxnet3_xdp.c | 116 struct xdp_frame *xdpf, in vmxnet3_xdp_xmit_frame() argument 128 dw2 |= xdpf->len; in vmxnet3_xdp_xmit_frame() 132 buf_size = xdpf->len; in vmxnet3_xdp_xmit_frame() 143 xdpf->data, buf_size, in vmxnet3_xdp_xmit_frame() 149 page = virt_to_page(xdpf->data); in vmxnet3_xdp_xmit_frame() 156 tbi->xdpf = xdpf; in vmxnet3_xdp_xmit_frame() 202 struct xdp_frame *xdpf) in vmxnet3_xdp_xmit_back() argument 254 struct xdp_frame *xdpf; in vmxnet3_run_xdp() local 276 xdpf = xdp_convert_buff_to_frame(xdp); in vmxnet3_run_xdp() 277 if (unlikely(!xdpf || in vmxnet3_run_xdp() [all …]
|
/linux/drivers/net/ethernet/mellanox/mlx5/core/en/ |
H A D | xdp.c | 67 struct xdp_frame *xdpf; in mlx5e_xmit_xdp_buff() local 72 if (unlikely(!xdpf)) in mlx5e_xmit_xdp_buff() 76 xdptxd->data = xdpf->data; in mlx5e_xmit_xdp_buff() 77 xdptxd->len = xdpf->len; in mlx5e_xmit_xdp_buff() 99 xdp_return_frame(xdpf); in mlx5e_xmit_xdp_buff() 113 (union mlx5e_xdp_info) { .frame.xdpf = xdpf }); in mlx5e_xmit_xdp_buff() 125 dma_addr = page_pool_get_dma_addr(page) + (xdpf->data - (void *)xdpf); in mlx5e_xmit_xdp_buff() 675 struct xdp_frame *xdpf; in mlx5e_free_xdpsq_desc() local 679 xdpf = xdpi.frame.xdpf; in mlx5e_free_xdpsq_desc() 879 xdptxd->len = xdpf->len; in mlx5e_xdp_xmit() [all …]
|
/linux/include/net/ |
H A D | xdp.h | 243 struct sk_buff *__xdp_build_skb_from_frame(struct xdp_frame *xdpf, 246 struct sk_buff *xdp_build_skb_from_frame(struct xdp_frame *xdpf, 249 struct xdp_frame *xdpf_clone(struct xdp_frame *xdpf); 313 void xdp_return_frame(struct xdp_frame *xdpf); 314 void xdp_return_frame_rx_napi(struct xdp_frame *xdpf); 317 void xdp_return_frame_bulk(struct xdp_frame *xdpf, 320 static __always_inline unsigned int xdp_get_frame_len(struct xdp_frame *xdpf) in xdp_get_frame_len() argument 323 unsigned int len = xdpf->len; in xdp_get_frame_len() 325 if (likely(!xdp_frame_has_frags(xdpf))) in xdp_get_frame_len() 328 sinfo = xdp_get_shared_info_from_frame(xdpf); in xdp_get_frame_len()
|
/linux/drivers/net/ethernet/intel/ixgbe/ |
H A D | ixgbe_xsk.c | 104 struct xdp_frame *xdpf; in ixgbe_run_xdp_zc() local 125 xdpf = xdp_convert_buff_to_frame(xdp); in ixgbe_run_xdp_zc() 126 if (unlikely(!xdpf)) in ixgbe_run_xdp_zc() 131 result = ixgbe_xmit_xdp_ring(ring, xdpf); in ixgbe_run_xdp_zc() 418 tx_bi->xdpf = NULL; in ixgbe_xmit_zc() 449 xdp_return_frame(tx_bi->xdpf); in ixgbe_clean_xdp_tx_buffer() 476 if (tx_bi->xdpf) in ixgbe_clean_xdp_tx_irq() 481 tx_bi->xdpf = NULL; in ixgbe_clean_xdp_tx_irq() 550 if (tx_bi->xdpf) in ixgbe_xsk_clean_tx_ring() 555 tx_bi->xdpf = NULL; in ixgbe_xsk_clean_tx_ring()
|
/linux/drivers/net/ethernet/sfc/siena/ |
H A D | tx.c | 213 struct xdp_frame *xdpf; in efx_siena_xdp_tx_buffers() local 255 xdpf = xdpfs[i]; in efx_siena_xdp_tx_buffers() 263 len = xdpf->len; in efx_siena_xdp_tx_buffers() 267 xdpf->data, len, in efx_siena_xdp_tx_buffers() 274 tx_buffer->xdpf = xdpf; in efx_siena_xdp_tx_buffers()
|
H A D | rx.c | 257 struct xdp_frame *xdpf; in efx_do_xdp() local 312 xdpf = xdp_convert_buff_to_frame(&xdp); in efx_do_xdp() 313 err = efx_siena_xdp_tx_buffers(efx, 1, &xdpf, true); in efx_do_xdp()
|
/linux/drivers/net/ethernet/freescale/dpaa/ |
H A D | dpaa_eth.c | 2224 memmove(aligned_data, xdpf->data, xdpf->len); in dpaa_a050385_wa_xdpf() 2251 memcpy(new_buff + headroom, xdpf->data, xdpf->len); in dpaa_a050385_wa_xdpf() 2258 new_xdpf->len = xdpf->len; in dpaa_a050385_wa_xdpf() 2519 buff_start = xdpf->data - xdpf->headroom; in dpaa_xdp_xmit_frame() 2526 swbp->xdpf = xdpf; in dpaa_xdp_xmit_frame() 2531 qm_fd_set_contig(&fd, xdpf->headroom, xdpf->len); in dpaa_xdp_xmit_frame() 2534 xdpf->headroom + xdpf->len, in dpaa_xdp_xmit_frame() 2567 struct xdp_frame *xdpf; in dpaa_run_xdp() local 2616 if (unlikely(!xdpf)) { in dpaa_run_xdp() 3043 struct xdp_frame *xdpf; in dpaa_xdp_xmit() local [all …]
|
/linux/drivers/net/ethernet/fungible/funeth/ |
H A D | funeth_tx.c | 500 xdp_return_frame(q->info[reclaim_idx].xdpf); in fun_xdpq_clean() 514 bool fun_xdp_tx(struct funeth_txq *q, struct xdp_frame *xdpf) in fun_xdp_tx() argument 516 unsigned int idx, nfrags = 1, ndesc = 1, tot_len = xdpf->len; in fun_xdp_tx() 525 if (unlikely(xdp_frame_has_frags(xdpf))) { in fun_xdp_tx() 526 si = xdp_get_shared_info_from_frame(xdpf); in fun_xdp_tx() 527 tot_len = xdp_get_frame_len(xdpf); in fun_xdp_tx() 539 if (unlikely(fun_map_pkt(q->dma_dev, si, xdpf->data, xdpf->len, dma, in fun_xdp_tx() 558 q->info[idx].xdpf = xdpf; in fun_xdp_tx() 618 xdp_return_frame(q->info[idx].xdpf); in fun_xdpq_purge()
|
/linux/drivers/net/ethernet/aquantia/atlantic/ |
H A D | aq_ring.c | 320 } else if (buff->xdpf) { in aq_ring_tx_clean() 325 xdp_return_frame_rx_napi(buff->xdpf); in aq_ring_tx_clean() 330 buff->xdpf = NULL; in aq_ring_tx_clean() 377 struct xdp_frame *xdpf = frames[i]; in aq_xdp_xmit() local 390 struct xdp_frame *xdpf; in aq_xdp_build_skb() local 393 xdpf = xdp_convert_buff_to_frame(xdp); in aq_xdp_build_skb() 394 if (unlikely(!xdpf)) in aq_xdp_build_skb() 397 skb = xdp_build_skb_from_frame(xdpf, dev); in aq_xdp_build_skb() 412 struct xdp_frame *xdpf; in aq_xdp_run_prog() local 443 xdpf = xdp_convert_buff_to_frame(xdp); in aq_xdp_run_prog() [all …]
|
H A D | aq_nic.c | 576 struct xdp_frame *xdpf, in aq_nic_map_xdp() argument 592 sinfo = xdp_get_shared_info_from_frame(xdpf); in aq_nic_map_xdp() 593 total_len = xdpf->len; in aq_nic_map_xdp() 595 if (xdp_frame_has_frags(xdpf)) { in aq_nic_map_xdp() 599 dx_buff->pa = dma_map_single(dev, xdpf->data, dx_buff->len, in aq_nic_map_xdp() 649 dx_buff->xdpf = xdpf; in aq_nic_map_xdp() 800 dx_buff->xdpf = NULL; in aq_nic_map_skb() 830 struct xdp_frame *xdpf) in aq_nic_xmit_xdpf() argument 840 if (xdp_frame_has_frags(xdpf)) { in aq_nic_xmit_xdpf() 841 sinfo = xdp_get_shared_info_from_frame(xdpf); in aq_nic_xmit_xdpf() [all …]
|
/linux/drivers/net/ethernet/ti/ |
H A D | cpsw_priv.c | 55 struct xdp_frame *xdpf; in cpsw_tx_handler() local 62 xdpf = cpsw_handle_to_xdpf(token); in cpsw_tx_handler() 63 xmeta = (void *)xdpf + CPSW_XMETA_OFFSET; in cpsw_tx_handler() 66 xdp_return_frame(xdpf); in cpsw_tx_handler() 1309 dma, xdpf->len, port); in cpsw_xdp_tx_frame() 1311 if (sizeof(*xmeta) > xdpf->headroom) in cpsw_xdp_tx_frame() 1315 xdpf->data, xdpf->len, port); in cpsw_xdp_tx_frame() 1330 struct xdp_frame *xdpf; in cpsw_run_xdp() local 1347 xdpf = xdp_convert_buff_to_frame(xdp); in cpsw_run_xdp() 1348 if (unlikely(!xdpf)) in cpsw_run_xdp() [all …]
|
H A D | am65-cpsw-nuss.c | 929 u32 pkt_len = xdpf->len; in am65_cpsw_xdp_tx_frame() 960 *(swdata) = xdpf; in am65_cpsw_xdp_tx_frame() 1003 struct xdp_frame *xdpf; in am65_cpsw_run_xdp() local 1025 if (unlikely(!xdpf)) in am65_cpsw_run_xdp() 1316 struct xdp_frame *xdpf; in am65_cpsw_nuss_tx_compl_packet_xdp() local 1323 xdpf = *(swdata); in am65_cpsw_nuss_tx_compl_packet_xdp() 1336 return xdpf; in am65_cpsw_nuss_tx_compl_packet_xdp() 1365 struct xdp_frame *xdpf; in am65_cpsw_nuss_tx_compl_packets() local 1395 total_bytes = xdpf->len; in am65_cpsw_nuss_tx_compl_packets() 1399 xdp_return_frame(xdpf); in am65_cpsw_nuss_tx_compl_packets() [all …]
|
/linux/drivers/net/ethernet/sfc/ |
H A D | tx.c | 426 struct xdp_frame *xdpf; in efx_xdp_tx_buffers() local 468 xdpf = xdpfs[i]; in efx_xdp_tx_buffers() 476 len = xdpf->len; in efx_xdp_tx_buffers() 480 xdpf->data, len, in efx_xdp_tx_buffers() 487 tx_buffer->xdpf = xdpf; in efx_xdp_tx_buffers()
|
/linux/drivers/net/ethernet/socionext/ |
H A D | netsec.c | 272 struct xdp_frame *xdpf; member 675 bytes += desc->xdpf->len; in netsec_clean_tx_dring() 677 xdp_return_frame_rx_napi(desc->xdpf); in netsec_clean_tx_dring() 821 dring->desc[idx].xdpf = buf; in netsec_set_tx_de() 829 struct xdp_frame *xdpf, bool is_ndo) in netsec_xdp_queue_one() argument 851 dma_handle = dma_map_single(priv->dev, xdpf->data, xdpf->len, in netsec_xdp_queue_one() 866 sizeof(*xdpf); in netsec_xdp_queue_one() 873 tx_desc.addr = xdpf->data; in netsec_xdp_queue_one() 874 tx_desc.len = xdpf->len; in netsec_xdp_queue_one() 888 if (unlikely(!xdpf)) in netsec_xdp_xmit_back() [all …]
|
/linux/drivers/net/ethernet/microchip/lan966x/ |
H A D | lan966x_fdma.c | 435 xdp_return_frame_bulk(dcb_buf->data.xdpf, &bq); in lan966x_fdma_tx_clear_buf() 710 struct xdp_frame *xdpf; in lan966x_fdma_xmit_xdpf() local 732 xdpf = ptr; in lan966x_fdma_xmit_xdpf() 734 if (xdpf->headroom < IFH_LEN_BYTES) { in lan966x_fdma_xmit_xdpf() 739 ifh = xdpf->data - IFH_LEN_BYTES; in lan966x_fdma_xmit_xdpf() 745 xdpf->data - IFH_LEN_BYTES, in lan966x_fdma_xmit_xdpf() 746 xdpf->len + IFH_LEN_BYTES, in lan966x_fdma_xmit_xdpf() 753 next_dcb_buf->data.xdpf = xdpf; in lan966x_fdma_xmit_xdpf() 754 next_dcb_buf->len = xdpf->len + IFH_LEN_BYTES; in lan966x_fdma_xmit_xdpf() 758 xdpf->len + IFH_LEN_BYTES, in lan966x_fdma_xmit_xdpf()
|
H A D | lan966x_xdp.c | 62 struct xdp_frame *xdpf = frames[i]; in lan966x_xdp_xmit() local 65 err = lan966x_fdma_xmit_xdpf(port, xdpf, 0); in lan966x_xdp_xmit()
|
/linux/drivers/net/ethernet/qlogic/qede/ |
H A D | qede_fp.c | 332 xdp->xdpf = xdpf; in qede_xdp_xmit() 345 struct xdp_frame *xdpf; in qede_xdp_transmit() local 362 xdpf = frames[i]; in qede_xdp_transmit() 364 mapping = dma_map_single(dmadev, xdpf->data, xdpf->len, in qede_xdp_transmit() 370 NULL, xdpf))) in qede_xdp_transmit() 404 struct xdp_frame *xdpf; in qede_xdp_tx_int() local 412 xdpf = xdp_info->xdpf; in qede_xdp_tx_int() 414 if (xdpf) { in qede_xdp_tx_int() 415 dma_unmap_single(dev, xdp_info->mapping, xdpf->len, in qede_xdp_tx_int() 417 xdp_return_frame(xdpf); in qede_xdp_tx_int() [all …]
|
/linux/drivers/net/ethernet/broadcom/bnxt/ |
H A D | bnxt_xdp.c | 110 struct xdp_frame *xdpf) in __bnxt_xmit_xdp_redirect() argument 116 tx_buf->xdpf = xdpf; in __bnxt_xmit_xdp_redirect() 145 xdp_return_frame(tx_buf->xdpf); in bnxt_tx_int_xdp() 147 tx_buf->xdpf = NULL; in bnxt_tx_int_xdp()
|
/linux/drivers/net/ethernet/engleder/ |
H A D | tsnep_main.c | 632 len = xdpf->len; in tsnep_xdp_tx_map() 637 xdpf->data; in tsnep_xdp_tx_map() 645 virt_to_page(xdpf->data); in tsnep_xdp_tx_map() 650 dma += sizeof(*xdpf) + xdpf->headroom; in tsnep_xdp_tx_map() 682 if (unlikely(xdp_frame_has_frags(xdpf))) in tsnep_xdp_xmit_frame_ring() 693 entry->xdpf = xdpf; in tsnep_xdp_xmit_frame_ring() 698 entry->xdpf = NULL; in tsnep_xdp_xmit_frame_ring() 731 if (unlikely(!xdpf)) in tsnep_xdp_xmit_back() 742 xmit = tsnep_xdp_xmit_frame_ring(xdpf, tx, type); in tsnep_xdp_xmit_back() 843 xdp_frame_has_frags(entry->xdpf)) in tsnep_tx_poll() [all …]
|
/linux/drivers/net/ethernet/intel/i40e/ |
H A D | i40e_xsk.c | 637 xdp_return_frame(tx_bi->xdpf); in i40e_clean_xdp_tx_buffer() 677 if (tx_bi->xdpf) { in i40e_clean_xdp_tx_irq() 679 tx_bi->xdpf = NULL; in i40e_clean_xdp_tx_irq() 778 if (tx_bi->xdpf) in i40e_xsk_clean_tx_ring() 783 tx_bi->xdpf = NULL; in i40e_xsk_clean_tx_ring()
|