1c8349639SHoratiu Vultur // SPDX-License-Identifier: GPL-2.0+ 2c8349639SHoratiu Vultur 37292bb06SHoratiu Vultur #include <linux/bpf.h> 4*a825b611SHoratiu Vultur #include <linux/filter.h> 57292bb06SHoratiu Vultur 6c8349639SHoratiu Vultur #include "lan966x_main.h" 7c8349639SHoratiu Vultur 8c8349639SHoratiu Vultur static int lan966x_fdma_channel_active(struct lan966x *lan966x) 9c8349639SHoratiu Vultur { 10c8349639SHoratiu Vultur return lan_rd(lan966x, FDMA_CH_ACTIVE); 11c8349639SHoratiu Vultur } 12c8349639SHoratiu Vultur 13c8349639SHoratiu Vultur static struct page *lan966x_fdma_rx_alloc_page(struct lan966x_rx *rx, 14c8349639SHoratiu Vultur struct lan966x_db *db) 15c8349639SHoratiu Vultur { 16c8349639SHoratiu Vultur struct page *page; 17c8349639SHoratiu Vultur 1811871abaSHoratiu Vultur page = page_pool_dev_alloc_pages(rx->page_pool); 19c8349639SHoratiu Vultur if (unlikely(!page)) 20c8349639SHoratiu Vultur return NULL; 21c8349639SHoratiu Vultur 227292bb06SHoratiu Vultur db->dataptr = page_pool_get_dma_addr(page) + XDP_PACKET_HEADROOM; 23c8349639SHoratiu Vultur 24c8349639SHoratiu Vultur return page; 25c8349639SHoratiu Vultur } 26c8349639SHoratiu Vultur 27c8349639SHoratiu Vultur static void lan966x_fdma_rx_free_pages(struct lan966x_rx *rx) 28c8349639SHoratiu Vultur { 29c8349639SHoratiu Vultur int i, j; 30c8349639SHoratiu Vultur 31c8349639SHoratiu Vultur for (i = 0; i < FDMA_DCB_MAX; ++i) { 3211871abaSHoratiu Vultur for (j = 0; j < FDMA_RX_DCB_MAX_DBS; ++j) 3311871abaSHoratiu Vultur page_pool_put_full_page(rx->page_pool, 3411871abaSHoratiu Vultur rx->page[i][j], false); 35c8349639SHoratiu Vultur } 36c8349639SHoratiu Vultur } 37c8349639SHoratiu Vultur 384a00b0c7SHoratiu Vultur static void lan966x_fdma_rx_free_page(struct lan966x_rx *rx) 394a00b0c7SHoratiu Vultur { 404a00b0c7SHoratiu Vultur struct page *page; 414a00b0c7SHoratiu Vultur 424a00b0c7SHoratiu Vultur page = rx->page[rx->dcb_index][rx->db_index]; 434a00b0c7SHoratiu Vultur if (unlikely(!page)) 444a00b0c7SHoratiu Vultur return; 454a00b0c7SHoratiu Vultur 4611871abaSHoratiu Vultur page_pool_recycle_direct(rx->page_pool, page); 474a00b0c7SHoratiu Vultur } 484a00b0c7SHoratiu Vultur 49c8349639SHoratiu Vultur static void lan966x_fdma_rx_add_dcb(struct lan966x_rx *rx, 50c8349639SHoratiu Vultur struct lan966x_rx_dcb *dcb, 51c8349639SHoratiu Vultur u64 nextptr) 52c8349639SHoratiu Vultur { 53c8349639SHoratiu Vultur struct lan966x_db *db; 54c8349639SHoratiu Vultur int i; 55c8349639SHoratiu Vultur 56c8349639SHoratiu Vultur for (i = 0; i < FDMA_RX_DCB_MAX_DBS; ++i) { 57c8349639SHoratiu Vultur db = &dcb->db[i]; 58c8349639SHoratiu Vultur db->status = FDMA_DCB_STATUS_INTR; 59c8349639SHoratiu Vultur } 60c8349639SHoratiu Vultur 61c8349639SHoratiu Vultur dcb->nextptr = FDMA_DCB_INVALID_DATA; 62c8349639SHoratiu Vultur dcb->info = FDMA_DCB_INFO_DATAL(PAGE_SIZE << rx->page_order); 63c8349639SHoratiu Vultur 64c8349639SHoratiu Vultur rx->last_entry->nextptr = nextptr; 65c8349639SHoratiu Vultur rx->last_entry = dcb; 66c8349639SHoratiu Vultur } 67c8349639SHoratiu Vultur 6811871abaSHoratiu Vultur static int lan966x_fdma_rx_alloc_page_pool(struct lan966x_rx *rx) 6911871abaSHoratiu Vultur { 7011871abaSHoratiu Vultur struct lan966x *lan966x = rx->lan966x; 7111871abaSHoratiu Vultur struct page_pool_params pp_params = { 7211871abaSHoratiu Vultur .order = rx->page_order, 7311871abaSHoratiu Vultur .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV, 7411871abaSHoratiu Vultur .pool_size = FDMA_DCB_MAX, 7511871abaSHoratiu Vultur .nid = NUMA_NO_NODE, 7611871abaSHoratiu Vultur .dev = lan966x->dev, 7711871abaSHoratiu Vultur .dma_dir = DMA_FROM_DEVICE, 787292bb06SHoratiu Vultur .offset = XDP_PACKET_HEADROOM, 7911871abaSHoratiu Vultur .max_len = rx->max_mtu - 8011871abaSHoratiu Vultur SKB_DATA_ALIGN(sizeof(struct skb_shared_info)), 8111871abaSHoratiu Vultur }; 8211871abaSHoratiu Vultur 83560c7223SHoratiu Vultur if (lan966x_xdp_present(lan966x)) 84560c7223SHoratiu Vultur pp_params.dma_dir = DMA_BIDIRECTIONAL; 85560c7223SHoratiu Vultur 8611871abaSHoratiu Vultur rx->page_pool = page_pool_create(&pp_params); 8777ddda44SHoratiu Vultur 8877ddda44SHoratiu Vultur for (int i = 0; i < lan966x->num_phys_ports; ++i) { 8977ddda44SHoratiu Vultur struct lan966x_port *port; 9077ddda44SHoratiu Vultur 9177ddda44SHoratiu Vultur if (!lan966x->ports[i]) 9277ddda44SHoratiu Vultur continue; 9377ddda44SHoratiu Vultur 9477ddda44SHoratiu Vultur port = lan966x->ports[i]; 9577ddda44SHoratiu Vultur xdp_rxq_info_unreg_mem_model(&port->xdp_rxq); 9677ddda44SHoratiu Vultur xdp_rxq_info_reg_mem_model(&port->xdp_rxq, MEM_TYPE_PAGE_POOL, 9777ddda44SHoratiu Vultur rx->page_pool); 9877ddda44SHoratiu Vultur } 9977ddda44SHoratiu Vultur 10011871abaSHoratiu Vultur return PTR_ERR_OR_ZERO(rx->page_pool); 10111871abaSHoratiu Vultur } 10211871abaSHoratiu Vultur 103c8349639SHoratiu Vultur static int lan966x_fdma_rx_alloc(struct lan966x_rx *rx) 104c8349639SHoratiu Vultur { 105c8349639SHoratiu Vultur struct lan966x *lan966x = rx->lan966x; 106c8349639SHoratiu Vultur struct lan966x_rx_dcb *dcb; 107c8349639SHoratiu Vultur struct lan966x_db *db; 108c8349639SHoratiu Vultur struct page *page; 109c8349639SHoratiu Vultur int i, j; 110c8349639SHoratiu Vultur int size; 111c8349639SHoratiu Vultur 11211871abaSHoratiu Vultur if (lan966x_fdma_rx_alloc_page_pool(rx)) 11311871abaSHoratiu Vultur return PTR_ERR(rx->page_pool); 11411871abaSHoratiu Vultur 115c8349639SHoratiu Vultur /* calculate how many pages are needed to allocate the dcbs */ 116c8349639SHoratiu Vultur size = sizeof(struct lan966x_rx_dcb) * FDMA_DCB_MAX; 117c8349639SHoratiu Vultur size = ALIGN(size, PAGE_SIZE); 118c8349639SHoratiu Vultur 119c8349639SHoratiu Vultur rx->dcbs = dma_alloc_coherent(lan966x->dev, size, &rx->dma, GFP_KERNEL); 120c8349639SHoratiu Vultur if (!rx->dcbs) 121c8349639SHoratiu Vultur return -ENOMEM; 122c8349639SHoratiu Vultur 123c8349639SHoratiu Vultur rx->last_entry = rx->dcbs; 124c8349639SHoratiu Vultur rx->db_index = 0; 125c8349639SHoratiu Vultur rx->dcb_index = 0; 126c8349639SHoratiu Vultur 127c8349639SHoratiu Vultur /* Now for each dcb allocate the dbs */ 128c8349639SHoratiu Vultur for (i = 0; i < FDMA_DCB_MAX; ++i) { 129c8349639SHoratiu Vultur dcb = &rx->dcbs[i]; 130c8349639SHoratiu Vultur dcb->info = 0; 131c8349639SHoratiu Vultur 132c8349639SHoratiu Vultur /* For each db allocate a page and map it to the DB dataptr. */ 133c8349639SHoratiu Vultur for (j = 0; j < FDMA_RX_DCB_MAX_DBS; ++j) { 134c8349639SHoratiu Vultur db = &dcb->db[j]; 135c8349639SHoratiu Vultur page = lan966x_fdma_rx_alloc_page(rx, db); 136c8349639SHoratiu Vultur if (!page) 137c8349639SHoratiu Vultur return -ENOMEM; 138c8349639SHoratiu Vultur 139c8349639SHoratiu Vultur db->status = 0; 140c8349639SHoratiu Vultur rx->page[i][j] = page; 141c8349639SHoratiu Vultur } 142c8349639SHoratiu Vultur 143c8349639SHoratiu Vultur lan966x_fdma_rx_add_dcb(rx, dcb, rx->dma + sizeof(*dcb) * i); 144c8349639SHoratiu Vultur } 145c8349639SHoratiu Vultur 146c8349639SHoratiu Vultur return 0; 147c8349639SHoratiu Vultur } 148c8349639SHoratiu Vultur 1494a00b0c7SHoratiu Vultur static void lan966x_fdma_rx_advance_dcb(struct lan966x_rx *rx) 1504a00b0c7SHoratiu Vultur { 1514a00b0c7SHoratiu Vultur rx->dcb_index++; 1524a00b0c7SHoratiu Vultur rx->dcb_index &= FDMA_DCB_MAX - 1; 1534a00b0c7SHoratiu Vultur } 1544a00b0c7SHoratiu Vultur 155c8349639SHoratiu Vultur static void lan966x_fdma_rx_free(struct lan966x_rx *rx) 156c8349639SHoratiu Vultur { 157c8349639SHoratiu Vultur struct lan966x *lan966x = rx->lan966x; 158c8349639SHoratiu Vultur u32 size; 159c8349639SHoratiu Vultur 160c8349639SHoratiu Vultur /* Now it is possible to do the cleanup of dcb */ 161c8349639SHoratiu Vultur size = sizeof(struct lan966x_tx_dcb) * FDMA_DCB_MAX; 162c8349639SHoratiu Vultur size = ALIGN(size, PAGE_SIZE); 163c8349639SHoratiu Vultur dma_free_coherent(lan966x->dev, size, rx->dcbs, rx->dma); 164c8349639SHoratiu Vultur } 165c8349639SHoratiu Vultur 166c8349639SHoratiu Vultur static void lan966x_fdma_rx_start(struct lan966x_rx *rx) 167c8349639SHoratiu Vultur { 168c8349639SHoratiu Vultur struct lan966x *lan966x = rx->lan966x; 169c8349639SHoratiu Vultur u32 mask; 170c8349639SHoratiu Vultur 171c8349639SHoratiu Vultur /* When activating a channel, first is required to write the first DCB 172c8349639SHoratiu Vultur * address and then to activate it 173c8349639SHoratiu Vultur */ 174c8349639SHoratiu Vultur lan_wr(lower_32_bits((u64)rx->dma), lan966x, 175c8349639SHoratiu Vultur FDMA_DCB_LLP(rx->channel_id)); 176c8349639SHoratiu Vultur lan_wr(upper_32_bits((u64)rx->dma), lan966x, 177c8349639SHoratiu Vultur FDMA_DCB_LLP1(rx->channel_id)); 178c8349639SHoratiu Vultur 179c8349639SHoratiu Vultur lan_wr(FDMA_CH_CFG_CH_DCB_DB_CNT_SET(FDMA_RX_DCB_MAX_DBS) | 180c8349639SHoratiu Vultur FDMA_CH_CFG_CH_INTR_DB_EOF_ONLY_SET(1) | 181c8349639SHoratiu Vultur FDMA_CH_CFG_CH_INJ_PORT_SET(0) | 182c8349639SHoratiu Vultur FDMA_CH_CFG_CH_MEM_SET(1), 183c8349639SHoratiu Vultur lan966x, FDMA_CH_CFG(rx->channel_id)); 184c8349639SHoratiu Vultur 185c8349639SHoratiu Vultur /* Start fdma */ 186c8349639SHoratiu Vultur lan_rmw(FDMA_PORT_CTRL_XTR_STOP_SET(0), 187c8349639SHoratiu Vultur FDMA_PORT_CTRL_XTR_STOP, 188c8349639SHoratiu Vultur lan966x, FDMA_PORT_CTRL(0)); 189c8349639SHoratiu Vultur 190c8349639SHoratiu Vultur /* Enable interrupts */ 191c8349639SHoratiu Vultur mask = lan_rd(lan966x, FDMA_INTR_DB_ENA); 192c8349639SHoratiu Vultur mask = FDMA_INTR_DB_ENA_INTR_DB_ENA_GET(mask); 193c8349639SHoratiu Vultur mask |= BIT(rx->channel_id); 194c8349639SHoratiu Vultur lan_rmw(FDMA_INTR_DB_ENA_INTR_DB_ENA_SET(mask), 195c8349639SHoratiu Vultur FDMA_INTR_DB_ENA_INTR_DB_ENA, 196c8349639SHoratiu Vultur lan966x, FDMA_INTR_DB_ENA); 197c8349639SHoratiu Vultur 198c8349639SHoratiu Vultur /* Activate the channel */ 199c8349639SHoratiu Vultur lan_rmw(FDMA_CH_ACTIVATE_CH_ACTIVATE_SET(BIT(rx->channel_id)), 200c8349639SHoratiu Vultur FDMA_CH_ACTIVATE_CH_ACTIVATE, 201c8349639SHoratiu Vultur lan966x, FDMA_CH_ACTIVATE); 202c8349639SHoratiu Vultur } 203c8349639SHoratiu Vultur 204c8349639SHoratiu Vultur static void lan966x_fdma_rx_disable(struct lan966x_rx *rx) 205c8349639SHoratiu Vultur { 206c8349639SHoratiu Vultur struct lan966x *lan966x = rx->lan966x; 207c8349639SHoratiu Vultur u32 val; 208c8349639SHoratiu Vultur 209c8349639SHoratiu Vultur /* Disable the channel */ 210c8349639SHoratiu Vultur lan_rmw(FDMA_CH_DISABLE_CH_DISABLE_SET(BIT(rx->channel_id)), 211c8349639SHoratiu Vultur FDMA_CH_DISABLE_CH_DISABLE, 212c8349639SHoratiu Vultur lan966x, FDMA_CH_DISABLE); 213c8349639SHoratiu Vultur 214c8349639SHoratiu Vultur readx_poll_timeout_atomic(lan966x_fdma_channel_active, lan966x, 215c8349639SHoratiu Vultur val, !(val & BIT(rx->channel_id)), 216c8349639SHoratiu Vultur READL_SLEEP_US, READL_TIMEOUT_US); 217c8349639SHoratiu Vultur 218c8349639SHoratiu Vultur lan_rmw(FDMA_CH_DB_DISCARD_DB_DISCARD_SET(BIT(rx->channel_id)), 219c8349639SHoratiu Vultur FDMA_CH_DB_DISCARD_DB_DISCARD, 220c8349639SHoratiu Vultur lan966x, FDMA_CH_DB_DISCARD); 221c8349639SHoratiu Vultur } 222c8349639SHoratiu Vultur 223c8349639SHoratiu Vultur static void lan966x_fdma_rx_reload(struct lan966x_rx *rx) 224c8349639SHoratiu Vultur { 225c8349639SHoratiu Vultur struct lan966x *lan966x = rx->lan966x; 226c8349639SHoratiu Vultur 227c8349639SHoratiu Vultur lan_rmw(FDMA_CH_RELOAD_CH_RELOAD_SET(BIT(rx->channel_id)), 228c8349639SHoratiu Vultur FDMA_CH_RELOAD_CH_RELOAD, 229c8349639SHoratiu Vultur lan966x, FDMA_CH_RELOAD); 230c8349639SHoratiu Vultur } 231c8349639SHoratiu Vultur 232c8349639SHoratiu Vultur static void lan966x_fdma_tx_add_dcb(struct lan966x_tx *tx, 233c8349639SHoratiu Vultur struct lan966x_tx_dcb *dcb) 234c8349639SHoratiu Vultur { 235c8349639SHoratiu Vultur dcb->nextptr = FDMA_DCB_INVALID_DATA; 236c8349639SHoratiu Vultur dcb->info = 0; 237c8349639SHoratiu Vultur } 238c8349639SHoratiu Vultur 239c8349639SHoratiu Vultur static int lan966x_fdma_tx_alloc(struct lan966x_tx *tx) 240c8349639SHoratiu Vultur { 241c8349639SHoratiu Vultur struct lan966x *lan966x = tx->lan966x; 242c8349639SHoratiu Vultur struct lan966x_tx_dcb *dcb; 243c8349639SHoratiu Vultur struct lan966x_db *db; 244c8349639SHoratiu Vultur int size; 245c8349639SHoratiu Vultur int i, j; 246c8349639SHoratiu Vultur 247c8349639SHoratiu Vultur tx->dcbs_buf = kcalloc(FDMA_DCB_MAX, sizeof(struct lan966x_tx_dcb_buf), 248c8349639SHoratiu Vultur GFP_KERNEL); 249c8349639SHoratiu Vultur if (!tx->dcbs_buf) 250c8349639SHoratiu Vultur return -ENOMEM; 251c8349639SHoratiu Vultur 252c8349639SHoratiu Vultur /* calculate how many pages are needed to allocate the dcbs */ 253c8349639SHoratiu Vultur size = sizeof(struct lan966x_tx_dcb) * FDMA_DCB_MAX; 254c8349639SHoratiu Vultur size = ALIGN(size, PAGE_SIZE); 255c8349639SHoratiu Vultur tx->dcbs = dma_alloc_coherent(lan966x->dev, size, &tx->dma, GFP_KERNEL); 256c8349639SHoratiu Vultur if (!tx->dcbs) 257c8349639SHoratiu Vultur goto out; 258c8349639SHoratiu Vultur 259c8349639SHoratiu Vultur /* Now for each dcb allocate the db */ 260c8349639SHoratiu Vultur for (i = 0; i < FDMA_DCB_MAX; ++i) { 261c8349639SHoratiu Vultur dcb = &tx->dcbs[i]; 262c8349639SHoratiu Vultur 263c8349639SHoratiu Vultur for (j = 0; j < FDMA_TX_DCB_MAX_DBS; ++j) { 264c8349639SHoratiu Vultur db = &dcb->db[j]; 265c8349639SHoratiu Vultur db->dataptr = 0; 266c8349639SHoratiu Vultur db->status = 0; 267c8349639SHoratiu Vultur } 268c8349639SHoratiu Vultur 269c8349639SHoratiu Vultur lan966x_fdma_tx_add_dcb(tx, dcb); 270c8349639SHoratiu Vultur } 271c8349639SHoratiu Vultur 272c8349639SHoratiu Vultur return 0; 273c8349639SHoratiu Vultur 274c8349639SHoratiu Vultur out: 275c8349639SHoratiu Vultur kfree(tx->dcbs_buf); 276c8349639SHoratiu Vultur return -ENOMEM; 277c8349639SHoratiu Vultur } 278c8349639SHoratiu Vultur 279c8349639SHoratiu Vultur static void lan966x_fdma_tx_free(struct lan966x_tx *tx) 280c8349639SHoratiu Vultur { 281c8349639SHoratiu Vultur struct lan966x *lan966x = tx->lan966x; 282c8349639SHoratiu Vultur int size; 283c8349639SHoratiu Vultur 284c8349639SHoratiu Vultur kfree(tx->dcbs_buf); 285c8349639SHoratiu Vultur 286c8349639SHoratiu Vultur size = sizeof(struct lan966x_tx_dcb) * FDMA_DCB_MAX; 287c8349639SHoratiu Vultur size = ALIGN(size, PAGE_SIZE); 288c8349639SHoratiu Vultur dma_free_coherent(lan966x->dev, size, tx->dcbs, tx->dma); 289c8349639SHoratiu Vultur } 290c8349639SHoratiu Vultur 291c8349639SHoratiu Vultur static void lan966x_fdma_tx_activate(struct lan966x_tx *tx) 292c8349639SHoratiu Vultur { 293c8349639SHoratiu Vultur struct lan966x *lan966x = tx->lan966x; 294c8349639SHoratiu Vultur u32 mask; 295c8349639SHoratiu Vultur 296c8349639SHoratiu Vultur /* When activating a channel, first is required to write the first DCB 297c8349639SHoratiu Vultur * address and then to activate it 298c8349639SHoratiu Vultur */ 299c8349639SHoratiu Vultur lan_wr(lower_32_bits((u64)tx->dma), lan966x, 300c8349639SHoratiu Vultur FDMA_DCB_LLP(tx->channel_id)); 301c8349639SHoratiu Vultur lan_wr(upper_32_bits((u64)tx->dma), lan966x, 302c8349639SHoratiu Vultur FDMA_DCB_LLP1(tx->channel_id)); 303c8349639SHoratiu Vultur 304c8349639SHoratiu Vultur lan_wr(FDMA_CH_CFG_CH_DCB_DB_CNT_SET(FDMA_TX_DCB_MAX_DBS) | 305c8349639SHoratiu Vultur FDMA_CH_CFG_CH_INTR_DB_EOF_ONLY_SET(1) | 306c8349639SHoratiu Vultur FDMA_CH_CFG_CH_INJ_PORT_SET(0) | 307c8349639SHoratiu Vultur FDMA_CH_CFG_CH_MEM_SET(1), 308c8349639SHoratiu Vultur lan966x, FDMA_CH_CFG(tx->channel_id)); 309c8349639SHoratiu Vultur 310c8349639SHoratiu Vultur /* Start fdma */ 311c8349639SHoratiu Vultur lan_rmw(FDMA_PORT_CTRL_INJ_STOP_SET(0), 312c8349639SHoratiu Vultur FDMA_PORT_CTRL_INJ_STOP, 313c8349639SHoratiu Vultur lan966x, FDMA_PORT_CTRL(0)); 314c8349639SHoratiu Vultur 315c8349639SHoratiu Vultur /* Enable interrupts */ 316c8349639SHoratiu Vultur mask = lan_rd(lan966x, FDMA_INTR_DB_ENA); 317c8349639SHoratiu Vultur mask = FDMA_INTR_DB_ENA_INTR_DB_ENA_GET(mask); 318c8349639SHoratiu Vultur mask |= BIT(tx->channel_id); 319c8349639SHoratiu Vultur lan_rmw(FDMA_INTR_DB_ENA_INTR_DB_ENA_SET(mask), 320c8349639SHoratiu Vultur FDMA_INTR_DB_ENA_INTR_DB_ENA, 321c8349639SHoratiu Vultur lan966x, FDMA_INTR_DB_ENA); 322c8349639SHoratiu Vultur 323c8349639SHoratiu Vultur /* Activate the channel */ 324c8349639SHoratiu Vultur lan_rmw(FDMA_CH_ACTIVATE_CH_ACTIVATE_SET(BIT(tx->channel_id)), 325c8349639SHoratiu Vultur FDMA_CH_ACTIVATE_CH_ACTIVATE, 326c8349639SHoratiu Vultur lan966x, FDMA_CH_ACTIVATE); 327c8349639SHoratiu Vultur } 328c8349639SHoratiu Vultur 329c8349639SHoratiu Vultur static void lan966x_fdma_tx_disable(struct lan966x_tx *tx) 330c8349639SHoratiu Vultur { 331c8349639SHoratiu Vultur struct lan966x *lan966x = tx->lan966x; 332c8349639SHoratiu Vultur u32 val; 333c8349639SHoratiu Vultur 334c8349639SHoratiu Vultur /* Disable the channel */ 335c8349639SHoratiu Vultur lan_rmw(FDMA_CH_DISABLE_CH_DISABLE_SET(BIT(tx->channel_id)), 336c8349639SHoratiu Vultur FDMA_CH_DISABLE_CH_DISABLE, 337c8349639SHoratiu Vultur lan966x, FDMA_CH_DISABLE); 338c8349639SHoratiu Vultur 339c8349639SHoratiu Vultur readx_poll_timeout_atomic(lan966x_fdma_channel_active, lan966x, 340c8349639SHoratiu Vultur val, !(val & BIT(tx->channel_id)), 341c8349639SHoratiu Vultur READL_SLEEP_US, READL_TIMEOUT_US); 342c8349639SHoratiu Vultur 343c8349639SHoratiu Vultur lan_rmw(FDMA_CH_DB_DISCARD_DB_DISCARD_SET(BIT(tx->channel_id)), 344c8349639SHoratiu Vultur FDMA_CH_DB_DISCARD_DB_DISCARD, 345c8349639SHoratiu Vultur lan966x, FDMA_CH_DB_DISCARD); 346c8349639SHoratiu Vultur 347c8349639SHoratiu Vultur tx->activated = false; 3484a4b6848SHoratiu Vultur tx->last_in_use = -1; 349c8349639SHoratiu Vultur } 350c8349639SHoratiu Vultur 351c8349639SHoratiu Vultur static void lan966x_fdma_tx_reload(struct lan966x_tx *tx) 352c8349639SHoratiu Vultur { 353c8349639SHoratiu Vultur struct lan966x *lan966x = tx->lan966x; 354c8349639SHoratiu Vultur 355c8349639SHoratiu Vultur /* Write the registers to reload the channel */ 356c8349639SHoratiu Vultur lan_rmw(FDMA_CH_RELOAD_CH_RELOAD_SET(BIT(tx->channel_id)), 357c8349639SHoratiu Vultur FDMA_CH_RELOAD_CH_RELOAD, 358c8349639SHoratiu Vultur lan966x, FDMA_CH_RELOAD); 359c8349639SHoratiu Vultur } 360c8349639SHoratiu Vultur 361c8349639SHoratiu Vultur static void lan966x_fdma_wakeup_netdev(struct lan966x *lan966x) 362c8349639SHoratiu Vultur { 363c8349639SHoratiu Vultur struct lan966x_port *port; 364c8349639SHoratiu Vultur int i; 365c8349639SHoratiu Vultur 366c8349639SHoratiu Vultur for (i = 0; i < lan966x->num_phys_ports; ++i) { 367c8349639SHoratiu Vultur port = lan966x->ports[i]; 368c8349639SHoratiu Vultur if (!port) 369c8349639SHoratiu Vultur continue; 370c8349639SHoratiu Vultur 371c8349639SHoratiu Vultur if (netif_queue_stopped(port->dev)) 372c8349639SHoratiu Vultur netif_wake_queue(port->dev); 373c8349639SHoratiu Vultur } 374c8349639SHoratiu Vultur } 375c8349639SHoratiu Vultur 3762ea1cbacSHoratiu Vultur static void lan966x_fdma_stop_netdev(struct lan966x *lan966x) 3772ea1cbacSHoratiu Vultur { 3782ea1cbacSHoratiu Vultur struct lan966x_port *port; 3792ea1cbacSHoratiu Vultur int i; 3802ea1cbacSHoratiu Vultur 3812ea1cbacSHoratiu Vultur for (i = 0; i < lan966x->num_phys_ports; ++i) { 3822ea1cbacSHoratiu Vultur port = lan966x->ports[i]; 3832ea1cbacSHoratiu Vultur if (!port) 3842ea1cbacSHoratiu Vultur continue; 3852ea1cbacSHoratiu Vultur 3862ea1cbacSHoratiu Vultur netif_stop_queue(port->dev); 3872ea1cbacSHoratiu Vultur } 3882ea1cbacSHoratiu Vultur } 3892ea1cbacSHoratiu Vultur 390c8349639SHoratiu Vultur static void lan966x_fdma_tx_clear_buf(struct lan966x *lan966x, int weight) 391c8349639SHoratiu Vultur { 392c8349639SHoratiu Vultur struct lan966x_tx *tx = &lan966x->tx; 393c8349639SHoratiu Vultur struct lan966x_tx_dcb_buf *dcb_buf; 394*a825b611SHoratiu Vultur struct xdp_frame_bulk bq; 395c8349639SHoratiu Vultur struct lan966x_db *db; 396c8349639SHoratiu Vultur unsigned long flags; 397c8349639SHoratiu Vultur bool clear = false; 398c8349639SHoratiu Vultur int i; 399c8349639SHoratiu Vultur 400*a825b611SHoratiu Vultur xdp_frame_bulk_init(&bq); 401*a825b611SHoratiu Vultur 402c8349639SHoratiu Vultur spin_lock_irqsave(&lan966x->tx_lock, flags); 403c8349639SHoratiu Vultur for (i = 0; i < FDMA_DCB_MAX; ++i) { 404c8349639SHoratiu Vultur dcb_buf = &tx->dcbs_buf[i]; 405c8349639SHoratiu Vultur 406c8349639SHoratiu Vultur if (!dcb_buf->used) 407c8349639SHoratiu Vultur continue; 408c8349639SHoratiu Vultur 409c8349639SHoratiu Vultur db = &tx->dcbs[i].db[0]; 410c8349639SHoratiu Vultur if (!(db->status & FDMA_DCB_STATUS_DONE)) 411c8349639SHoratiu Vultur continue; 412c8349639SHoratiu Vultur 413c8349639SHoratiu Vultur dcb_buf->dev->stats.tx_packets++; 41449f5eea8SHoratiu Vultur dcb_buf->dev->stats.tx_bytes += dcb_buf->len; 415c8349639SHoratiu Vultur 416c8349639SHoratiu Vultur dcb_buf->used = false; 41719c6f534SHoratiu Vultur if (dcb_buf->use_skb) { 418c8349639SHoratiu Vultur dma_unmap_single(lan966x->dev, 419c8349639SHoratiu Vultur dcb_buf->dma_addr, 42049f5eea8SHoratiu Vultur dcb_buf->len, 421c8349639SHoratiu Vultur DMA_TO_DEVICE); 42219c6f534SHoratiu Vultur 423c8349639SHoratiu Vultur if (!dcb_buf->ptp) 42419c6f534SHoratiu Vultur napi_consume_skb(dcb_buf->data.skb, weight); 42519c6f534SHoratiu Vultur } else { 426*a825b611SHoratiu Vultur if (dcb_buf->xdp_ndo) 427*a825b611SHoratiu Vultur dma_unmap_single(lan966x->dev, 428*a825b611SHoratiu Vultur dcb_buf->dma_addr, 429*a825b611SHoratiu Vultur dcb_buf->len, 430*a825b611SHoratiu Vultur DMA_TO_DEVICE); 431*a825b611SHoratiu Vultur 432*a825b611SHoratiu Vultur if (dcb_buf->xdp_ndo) 433*a825b611SHoratiu Vultur xdp_return_frame_bulk(dcb_buf->data.xdpf, &bq); 434*a825b611SHoratiu Vultur else 43519c6f534SHoratiu Vultur xdp_return_frame_rx_napi(dcb_buf->data.xdpf); 43619c6f534SHoratiu Vultur } 437c8349639SHoratiu Vultur 438c8349639SHoratiu Vultur clear = true; 439c8349639SHoratiu Vultur } 440c8349639SHoratiu Vultur 441*a825b611SHoratiu Vultur xdp_flush_frame_bulk(&bq); 442*a825b611SHoratiu Vultur 443c8349639SHoratiu Vultur if (clear) 444c8349639SHoratiu Vultur lan966x_fdma_wakeup_netdev(lan966x); 445c8349639SHoratiu Vultur 446c8349639SHoratiu Vultur spin_unlock_irqrestore(&lan966x->tx_lock, flags); 447c8349639SHoratiu Vultur } 448c8349639SHoratiu Vultur 449c8349639SHoratiu Vultur static bool lan966x_fdma_rx_more_frames(struct lan966x_rx *rx) 450c8349639SHoratiu Vultur { 451c8349639SHoratiu Vultur struct lan966x_db *db; 452c8349639SHoratiu Vultur 453c8349639SHoratiu Vultur /* Check if there is any data */ 454c8349639SHoratiu Vultur db = &rx->dcbs[rx->dcb_index].db[rx->db_index]; 455c8349639SHoratiu Vultur if (unlikely(!(db->status & FDMA_DCB_STATUS_DONE))) 456c8349639SHoratiu Vultur return false; 457c8349639SHoratiu Vultur 458c8349639SHoratiu Vultur return true; 459c8349639SHoratiu Vultur } 460c8349639SHoratiu Vultur 4614a00b0c7SHoratiu Vultur static int lan966x_fdma_rx_check_frame(struct lan966x_rx *rx, u64 *src_port) 462c8349639SHoratiu Vultur { 463c8349639SHoratiu Vultur struct lan966x *lan966x = rx->lan966x; 4646a2159beSHoratiu Vultur struct lan966x_port *port; 465c8349639SHoratiu Vultur struct lan966x_db *db; 466c8349639SHoratiu Vultur struct page *page; 467c8349639SHoratiu Vultur 468c8349639SHoratiu Vultur db = &rx->dcbs[rx->dcb_index].db[rx->db_index]; 469c8349639SHoratiu Vultur page = rx->page[rx->dcb_index][rx->db_index]; 4704a00b0c7SHoratiu Vultur if (unlikely(!page)) 4714a00b0c7SHoratiu Vultur return FDMA_ERROR; 472fc57062fSHoratiu Vultur 4737292bb06SHoratiu Vultur dma_sync_single_for_cpu(lan966x->dev, 4747292bb06SHoratiu Vultur (dma_addr_t)db->dataptr + XDP_PACKET_HEADROOM, 475fc57062fSHoratiu Vultur FDMA_DCB_STATUS_BLOCKL(db->status), 476fc57062fSHoratiu Vultur DMA_FROM_DEVICE); 477fc57062fSHoratiu Vultur 4787292bb06SHoratiu Vultur lan966x_ifh_get_src_port(page_address(page) + XDP_PACKET_HEADROOM, 4797292bb06SHoratiu Vultur src_port); 4804a00b0c7SHoratiu Vultur if (WARN_ON(*src_port >= lan966x->num_phys_ports)) 4814a00b0c7SHoratiu Vultur return FDMA_ERROR; 4824a00b0c7SHoratiu Vultur 4836a2159beSHoratiu Vultur port = lan966x->ports[*src_port]; 4846a2159beSHoratiu Vultur if (!lan966x_xdp_port_present(port)) 4854a00b0c7SHoratiu Vultur return FDMA_PASS; 4866a2159beSHoratiu Vultur 4876a2159beSHoratiu Vultur return lan966x_xdp_run(port, page, FDMA_DCB_STATUS_BLOCKL(db->status)); 4884a00b0c7SHoratiu Vultur } 4894a00b0c7SHoratiu Vultur 4904a00b0c7SHoratiu Vultur static struct sk_buff *lan966x_fdma_rx_get_frame(struct lan966x_rx *rx, 4914a00b0c7SHoratiu Vultur u64 src_port) 4924a00b0c7SHoratiu Vultur { 4934a00b0c7SHoratiu Vultur struct lan966x *lan966x = rx->lan966x; 4944a00b0c7SHoratiu Vultur struct lan966x_db *db; 4954a00b0c7SHoratiu Vultur struct sk_buff *skb; 4964a00b0c7SHoratiu Vultur struct page *page; 4974a00b0c7SHoratiu Vultur u64 timestamp; 4984a00b0c7SHoratiu Vultur 4994a00b0c7SHoratiu Vultur /* Get the received frame and unmap it */ 5004a00b0c7SHoratiu Vultur db = &rx->dcbs[rx->dcb_index].db[rx->db_index]; 5014a00b0c7SHoratiu Vultur page = rx->page[rx->dcb_index][rx->db_index]; 5024a00b0c7SHoratiu Vultur 5034a00b0c7SHoratiu Vultur skb = build_skb(page_address(page), PAGE_SIZE << rx->page_order); 5044a00b0c7SHoratiu Vultur if (unlikely(!skb)) 5054a00b0c7SHoratiu Vultur goto free_page; 5064a00b0c7SHoratiu Vultur 50711871abaSHoratiu Vultur skb_mark_for_recycle(skb); 50811871abaSHoratiu Vultur 5097292bb06SHoratiu Vultur skb_reserve(skb, XDP_PACKET_HEADROOM); 5104a00b0c7SHoratiu Vultur skb_put(skb, FDMA_DCB_STATUS_BLOCKL(db->status)); 5114a00b0c7SHoratiu Vultur 5124a00b0c7SHoratiu Vultur lan966x_ifh_get_timestamp(skb->data, ×tamp); 5134a00b0c7SHoratiu Vultur 514c8349639SHoratiu Vultur skb->dev = lan966x->ports[src_port]->dev; 515e83163b6SHoratiu Vultur skb_pull(skb, IFH_LEN_BYTES); 516c8349639SHoratiu Vultur 517c8349639SHoratiu Vultur if (likely(!(skb->dev->features & NETIF_F_RXFCS))) 518c8349639SHoratiu Vultur skb_trim(skb, skb->len - ETH_FCS_LEN); 519c8349639SHoratiu Vultur 520c8349639SHoratiu Vultur lan966x_ptp_rxtstamp(lan966x, skb, timestamp); 521c8349639SHoratiu Vultur skb->protocol = eth_type_trans(skb, skb->dev); 522c8349639SHoratiu Vultur 523c8349639SHoratiu Vultur if (lan966x->bridge_mask & BIT(src_port)) { 524c8349639SHoratiu Vultur skb->offload_fwd_mark = 1; 525c8349639SHoratiu Vultur 526c8349639SHoratiu Vultur skb_reset_network_header(skb); 527c8349639SHoratiu Vultur if (!lan966x_hw_offload(lan966x, src_port, skb)) 528c8349639SHoratiu Vultur skb->offload_fwd_mark = 0; 529c8349639SHoratiu Vultur } 530c8349639SHoratiu Vultur 531c8349639SHoratiu Vultur skb->dev->stats.rx_bytes += skb->len; 532c8349639SHoratiu Vultur skb->dev->stats.rx_packets++; 533c8349639SHoratiu Vultur 534c8349639SHoratiu Vultur return skb; 535c8349639SHoratiu Vultur 5364a00b0c7SHoratiu Vultur free_page: 53711871abaSHoratiu Vultur page_pool_recycle_direct(rx->page_pool, page); 538c8349639SHoratiu Vultur 539c8349639SHoratiu Vultur return NULL; 540c8349639SHoratiu Vultur } 541c8349639SHoratiu Vultur 542c8349639SHoratiu Vultur static int lan966x_fdma_napi_poll(struct napi_struct *napi, int weight) 543c8349639SHoratiu Vultur { 544c8349639SHoratiu Vultur struct lan966x *lan966x = container_of(napi, struct lan966x, napi); 545c8349639SHoratiu Vultur struct lan966x_rx *rx = &lan966x->rx; 546c8349639SHoratiu Vultur int dcb_reload = rx->dcb_index; 547c8349639SHoratiu Vultur struct lan966x_rx_dcb *old_dcb; 548c8349639SHoratiu Vultur struct lan966x_db *db; 549*a825b611SHoratiu Vultur bool redirect = false; 550c8349639SHoratiu Vultur struct sk_buff *skb; 551c8349639SHoratiu Vultur struct page *page; 552c8349639SHoratiu Vultur int counter = 0; 5534a00b0c7SHoratiu Vultur u64 src_port; 554c8349639SHoratiu Vultur u64 nextptr; 555c8349639SHoratiu Vultur 556c8349639SHoratiu Vultur lan966x_fdma_tx_clear_buf(lan966x, weight); 557c8349639SHoratiu Vultur 558c8349639SHoratiu Vultur /* Get all received skb */ 559c8349639SHoratiu Vultur while (counter < weight) { 560c8349639SHoratiu Vultur if (!lan966x_fdma_rx_more_frames(rx)) 561c8349639SHoratiu Vultur break; 562c8349639SHoratiu Vultur 563c8349639SHoratiu Vultur counter++; 5644a00b0c7SHoratiu Vultur 5654a00b0c7SHoratiu Vultur switch (lan966x_fdma_rx_check_frame(rx, &src_port)) { 5664a00b0c7SHoratiu Vultur case FDMA_PASS: 5674a00b0c7SHoratiu Vultur break; 5684a00b0c7SHoratiu Vultur case FDMA_ERROR: 5694a00b0c7SHoratiu Vultur lan966x_fdma_rx_free_page(rx); 5704a00b0c7SHoratiu Vultur lan966x_fdma_rx_advance_dcb(rx); 5714a00b0c7SHoratiu Vultur goto allocate_new; 572*a825b611SHoratiu Vultur case FDMA_REDIRECT: 573*a825b611SHoratiu Vultur redirect = true; 574*a825b611SHoratiu Vultur fallthrough; 57519c6f534SHoratiu Vultur case FDMA_TX: 57619c6f534SHoratiu Vultur lan966x_fdma_rx_advance_dcb(rx); 57719c6f534SHoratiu Vultur continue; 5786a2159beSHoratiu Vultur case FDMA_DROP: 5796a2159beSHoratiu Vultur lan966x_fdma_rx_free_page(rx); 5806a2159beSHoratiu Vultur lan966x_fdma_rx_advance_dcb(rx); 5816a2159beSHoratiu Vultur continue; 582c8349639SHoratiu Vultur } 583c8349639SHoratiu Vultur 5844a00b0c7SHoratiu Vultur skb = lan966x_fdma_rx_get_frame(rx, src_port); 5854a00b0c7SHoratiu Vultur lan966x_fdma_rx_advance_dcb(rx); 5864a00b0c7SHoratiu Vultur if (!skb) 5874a00b0c7SHoratiu Vultur goto allocate_new; 5884a00b0c7SHoratiu Vultur 5894a00b0c7SHoratiu Vultur napi_gro_receive(&lan966x->napi, skb); 5904a00b0c7SHoratiu Vultur } 5914a00b0c7SHoratiu Vultur 5924a00b0c7SHoratiu Vultur allocate_new: 593c8349639SHoratiu Vultur /* Allocate new pages and map them */ 594c8349639SHoratiu Vultur while (dcb_reload != rx->dcb_index) { 595c8349639SHoratiu Vultur db = &rx->dcbs[dcb_reload].db[rx->db_index]; 596c8349639SHoratiu Vultur page = lan966x_fdma_rx_alloc_page(rx, db); 597c8349639SHoratiu Vultur if (unlikely(!page)) 598c8349639SHoratiu Vultur break; 599c8349639SHoratiu Vultur rx->page[dcb_reload][rx->db_index] = page; 600c8349639SHoratiu Vultur 601c8349639SHoratiu Vultur old_dcb = &rx->dcbs[dcb_reload]; 602c8349639SHoratiu Vultur dcb_reload++; 603c8349639SHoratiu Vultur dcb_reload &= FDMA_DCB_MAX - 1; 604c8349639SHoratiu Vultur 605c8349639SHoratiu Vultur nextptr = rx->dma + ((unsigned long)old_dcb - 606c8349639SHoratiu Vultur (unsigned long)rx->dcbs); 607c8349639SHoratiu Vultur lan966x_fdma_rx_add_dcb(rx, old_dcb, nextptr); 608c8349639SHoratiu Vultur lan966x_fdma_rx_reload(rx); 609c8349639SHoratiu Vultur } 610c8349639SHoratiu Vultur 611c8349639SHoratiu Vultur if (counter < weight && napi_complete_done(napi, counter)) 612c8349639SHoratiu Vultur lan_wr(0xff, lan966x, FDMA_INTR_DB_ENA); 613c8349639SHoratiu Vultur 614*a825b611SHoratiu Vultur if (redirect) 615*a825b611SHoratiu Vultur xdp_do_flush(); 616*a825b611SHoratiu Vultur 617c8349639SHoratiu Vultur return counter; 618c8349639SHoratiu Vultur } 619c8349639SHoratiu Vultur 620c8349639SHoratiu Vultur irqreturn_t lan966x_fdma_irq_handler(int irq, void *args) 621c8349639SHoratiu Vultur { 622c8349639SHoratiu Vultur struct lan966x *lan966x = args; 623c8349639SHoratiu Vultur u32 db, err, err_type; 624c8349639SHoratiu Vultur 625c8349639SHoratiu Vultur db = lan_rd(lan966x, FDMA_INTR_DB); 626c8349639SHoratiu Vultur err = lan_rd(lan966x, FDMA_INTR_ERR); 627c8349639SHoratiu Vultur 628c8349639SHoratiu Vultur if (db) { 629c8349639SHoratiu Vultur lan_wr(0, lan966x, FDMA_INTR_DB_ENA); 630c8349639SHoratiu Vultur lan_wr(db, lan966x, FDMA_INTR_DB); 631c8349639SHoratiu Vultur 632c8349639SHoratiu Vultur napi_schedule(&lan966x->napi); 633c8349639SHoratiu Vultur } 634c8349639SHoratiu Vultur 635c8349639SHoratiu Vultur if (err) { 636c8349639SHoratiu Vultur err_type = lan_rd(lan966x, FDMA_ERRORS); 637c8349639SHoratiu Vultur 638c8349639SHoratiu Vultur WARN(1, "Unexpected error: %d, error_type: %d\n", err, err_type); 639c8349639SHoratiu Vultur 640c8349639SHoratiu Vultur lan_wr(err, lan966x, FDMA_INTR_ERR); 641c8349639SHoratiu Vultur lan_wr(err_type, lan966x, FDMA_ERRORS); 642c8349639SHoratiu Vultur } 643c8349639SHoratiu Vultur 644c8349639SHoratiu Vultur return IRQ_HANDLED; 645c8349639SHoratiu Vultur } 646c8349639SHoratiu Vultur 647c8349639SHoratiu Vultur static int lan966x_fdma_get_next_dcb(struct lan966x_tx *tx) 648c8349639SHoratiu Vultur { 649c8349639SHoratiu Vultur struct lan966x_tx_dcb_buf *dcb_buf; 650c8349639SHoratiu Vultur int i; 651c8349639SHoratiu Vultur 652c8349639SHoratiu Vultur for (i = 0; i < FDMA_DCB_MAX; ++i) { 653c8349639SHoratiu Vultur dcb_buf = &tx->dcbs_buf[i]; 654c8349639SHoratiu Vultur if (!dcb_buf->used && i != tx->last_in_use) 655c8349639SHoratiu Vultur return i; 656c8349639SHoratiu Vultur } 657c8349639SHoratiu Vultur 658c8349639SHoratiu Vultur return -1; 659c8349639SHoratiu Vultur } 660c8349639SHoratiu Vultur 6613d66bc57SHoratiu Vultur static void lan966x_fdma_tx_setup_dcb(struct lan966x_tx *tx, 6623d66bc57SHoratiu Vultur int next_to_use, int len, 6633d66bc57SHoratiu Vultur dma_addr_t dma_addr) 6643d66bc57SHoratiu Vultur { 6653d66bc57SHoratiu Vultur struct lan966x_tx_dcb *next_dcb; 6663d66bc57SHoratiu Vultur struct lan966x_db *next_db; 6673d66bc57SHoratiu Vultur 6683d66bc57SHoratiu Vultur next_dcb = &tx->dcbs[next_to_use]; 6693d66bc57SHoratiu Vultur next_dcb->nextptr = FDMA_DCB_INVALID_DATA; 6703d66bc57SHoratiu Vultur 6713d66bc57SHoratiu Vultur next_db = &next_dcb->db[0]; 6723d66bc57SHoratiu Vultur next_db->dataptr = dma_addr; 6733d66bc57SHoratiu Vultur next_db->status = FDMA_DCB_STATUS_SOF | 6743d66bc57SHoratiu Vultur FDMA_DCB_STATUS_EOF | 6753d66bc57SHoratiu Vultur FDMA_DCB_STATUS_INTR | 6763d66bc57SHoratiu Vultur FDMA_DCB_STATUS_BLOCKO(0) | 6773d66bc57SHoratiu Vultur FDMA_DCB_STATUS_BLOCKL(len); 6783d66bc57SHoratiu Vultur } 6793d66bc57SHoratiu Vultur 6803d66bc57SHoratiu Vultur static void lan966x_fdma_tx_start(struct lan966x_tx *tx, int next_to_use) 6813d66bc57SHoratiu Vultur { 6823d66bc57SHoratiu Vultur struct lan966x *lan966x = tx->lan966x; 6833d66bc57SHoratiu Vultur struct lan966x_tx_dcb *dcb; 6843d66bc57SHoratiu Vultur 6853d66bc57SHoratiu Vultur if (likely(lan966x->tx.activated)) { 6863d66bc57SHoratiu Vultur /* Connect current dcb to the next db */ 6873d66bc57SHoratiu Vultur dcb = &tx->dcbs[tx->last_in_use]; 6883d66bc57SHoratiu Vultur dcb->nextptr = tx->dma + (next_to_use * 6893d66bc57SHoratiu Vultur sizeof(struct lan966x_tx_dcb)); 6903d66bc57SHoratiu Vultur 6913d66bc57SHoratiu Vultur lan966x_fdma_tx_reload(tx); 6923d66bc57SHoratiu Vultur } else { 6933d66bc57SHoratiu Vultur /* Because it is first time, then just activate */ 6943d66bc57SHoratiu Vultur lan966x->tx.activated = true; 6953d66bc57SHoratiu Vultur lan966x_fdma_tx_activate(tx); 6963d66bc57SHoratiu Vultur } 6973d66bc57SHoratiu Vultur 6983d66bc57SHoratiu Vultur /* Move to next dcb because this last in use */ 6993d66bc57SHoratiu Vultur tx->last_in_use = next_to_use; 7003d66bc57SHoratiu Vultur } 7013d66bc57SHoratiu Vultur 70219c6f534SHoratiu Vultur int lan966x_fdma_xmit_xdpf(struct lan966x_port *port, 70319c6f534SHoratiu Vultur struct xdp_frame *xdpf, 704*a825b611SHoratiu Vultur struct page *page, 705*a825b611SHoratiu Vultur bool dma_map) 70619c6f534SHoratiu Vultur { 70719c6f534SHoratiu Vultur struct lan966x *lan966x = port->lan966x; 70819c6f534SHoratiu Vultur struct lan966x_tx_dcb_buf *next_dcb_buf; 70919c6f534SHoratiu Vultur struct lan966x_tx *tx = &lan966x->tx; 71019c6f534SHoratiu Vultur dma_addr_t dma_addr; 71119c6f534SHoratiu Vultur int next_to_use; 71219c6f534SHoratiu Vultur __be32 *ifh; 71319c6f534SHoratiu Vultur int ret = 0; 71419c6f534SHoratiu Vultur 71519c6f534SHoratiu Vultur spin_lock(&lan966x->tx_lock); 71619c6f534SHoratiu Vultur 71719c6f534SHoratiu Vultur /* Get next index */ 71819c6f534SHoratiu Vultur next_to_use = lan966x_fdma_get_next_dcb(tx); 71919c6f534SHoratiu Vultur if (next_to_use < 0) { 72019c6f534SHoratiu Vultur netif_stop_queue(port->dev); 72119c6f534SHoratiu Vultur ret = NETDEV_TX_BUSY; 72219c6f534SHoratiu Vultur goto out; 72319c6f534SHoratiu Vultur } 72419c6f534SHoratiu Vultur 72519c6f534SHoratiu Vultur /* Generate new IFH */ 726*a825b611SHoratiu Vultur if (dma_map) { 727*a825b611SHoratiu Vultur if (xdpf->headroom < IFH_LEN_BYTES) { 728*a825b611SHoratiu Vultur ret = NETDEV_TX_OK; 729*a825b611SHoratiu Vultur goto out; 730*a825b611SHoratiu Vultur } 731*a825b611SHoratiu Vultur 732*a825b611SHoratiu Vultur ifh = xdpf->data - IFH_LEN_BYTES; 733*a825b611SHoratiu Vultur memset(ifh, 0x0, sizeof(__be32) * IFH_LEN); 734*a825b611SHoratiu Vultur lan966x_ifh_set_bypass(ifh, 1); 735*a825b611SHoratiu Vultur lan966x_ifh_set_port(ifh, BIT_ULL(port->chip_port)); 736*a825b611SHoratiu Vultur 737*a825b611SHoratiu Vultur dma_addr = dma_map_single(lan966x->dev, 738*a825b611SHoratiu Vultur xdpf->data - IFH_LEN_BYTES, 739*a825b611SHoratiu Vultur xdpf->len + IFH_LEN_BYTES, 740*a825b611SHoratiu Vultur DMA_TO_DEVICE); 741*a825b611SHoratiu Vultur if (dma_mapping_error(lan966x->dev, dma_addr)) { 742*a825b611SHoratiu Vultur ret = NETDEV_TX_OK; 743*a825b611SHoratiu Vultur goto out; 744*a825b611SHoratiu Vultur } 745*a825b611SHoratiu Vultur 746*a825b611SHoratiu Vultur /* Setup next dcb */ 747*a825b611SHoratiu Vultur lan966x_fdma_tx_setup_dcb(tx, next_to_use, 748*a825b611SHoratiu Vultur xdpf->len + IFH_LEN_BYTES, 749*a825b611SHoratiu Vultur dma_addr); 750*a825b611SHoratiu Vultur } else { 75119c6f534SHoratiu Vultur ifh = page_address(page) + XDP_PACKET_HEADROOM; 75219c6f534SHoratiu Vultur memset(ifh, 0x0, sizeof(__be32) * IFH_LEN); 75319c6f534SHoratiu Vultur lan966x_ifh_set_bypass(ifh, 1); 75419c6f534SHoratiu Vultur lan966x_ifh_set_port(ifh, BIT_ULL(port->chip_port)); 75519c6f534SHoratiu Vultur 75619c6f534SHoratiu Vultur dma_addr = page_pool_get_dma_addr(page); 757*a825b611SHoratiu Vultur dma_sync_single_for_device(lan966x->dev, 758*a825b611SHoratiu Vultur dma_addr + XDP_PACKET_HEADROOM, 75919c6f534SHoratiu Vultur xdpf->len + IFH_LEN_BYTES, 76019c6f534SHoratiu Vultur DMA_TO_DEVICE); 76119c6f534SHoratiu Vultur 76219c6f534SHoratiu Vultur /* Setup next dcb */ 763*a825b611SHoratiu Vultur lan966x_fdma_tx_setup_dcb(tx, next_to_use, 764*a825b611SHoratiu Vultur xdpf->len + IFH_LEN_BYTES, 76519c6f534SHoratiu Vultur dma_addr + XDP_PACKET_HEADROOM); 766*a825b611SHoratiu Vultur } 76719c6f534SHoratiu Vultur 76819c6f534SHoratiu Vultur /* Fill up the buffer */ 76919c6f534SHoratiu Vultur next_dcb_buf = &tx->dcbs_buf[next_to_use]; 77019c6f534SHoratiu Vultur next_dcb_buf->use_skb = false; 77119c6f534SHoratiu Vultur next_dcb_buf->data.xdpf = xdpf; 772*a825b611SHoratiu Vultur next_dcb_buf->xdp_ndo = dma_map; 77319c6f534SHoratiu Vultur next_dcb_buf->len = xdpf->len + IFH_LEN_BYTES; 77419c6f534SHoratiu Vultur next_dcb_buf->dma_addr = dma_addr; 77519c6f534SHoratiu Vultur next_dcb_buf->used = true; 77619c6f534SHoratiu Vultur next_dcb_buf->ptp = false; 77719c6f534SHoratiu Vultur next_dcb_buf->dev = port->dev; 77819c6f534SHoratiu Vultur 77919c6f534SHoratiu Vultur /* Start the transmission */ 78019c6f534SHoratiu Vultur lan966x_fdma_tx_start(tx, next_to_use); 78119c6f534SHoratiu Vultur 78219c6f534SHoratiu Vultur out: 78319c6f534SHoratiu Vultur spin_unlock(&lan966x->tx_lock); 78419c6f534SHoratiu Vultur 78519c6f534SHoratiu Vultur return ret; 78619c6f534SHoratiu Vultur } 78719c6f534SHoratiu Vultur 788c8349639SHoratiu Vultur int lan966x_fdma_xmit(struct sk_buff *skb, __be32 *ifh, struct net_device *dev) 789c8349639SHoratiu Vultur { 790c8349639SHoratiu Vultur struct lan966x_port *port = netdev_priv(dev); 791c8349639SHoratiu Vultur struct lan966x *lan966x = port->lan966x; 792c8349639SHoratiu Vultur struct lan966x_tx_dcb_buf *next_dcb_buf; 793c8349639SHoratiu Vultur struct lan966x_tx *tx = &lan966x->tx; 794c8349639SHoratiu Vultur int needed_headroom; 795c8349639SHoratiu Vultur int needed_tailroom; 796c8349639SHoratiu Vultur dma_addr_t dma_addr; 797c8349639SHoratiu Vultur int next_to_use; 798c8349639SHoratiu Vultur int err; 799c8349639SHoratiu Vultur 800c8349639SHoratiu Vultur /* Get next index */ 801c8349639SHoratiu Vultur next_to_use = lan966x_fdma_get_next_dcb(tx); 802c8349639SHoratiu Vultur if (next_to_use < 0) { 803c8349639SHoratiu Vultur netif_stop_queue(dev); 804c8349639SHoratiu Vultur return NETDEV_TX_BUSY; 805c8349639SHoratiu Vultur } 806c8349639SHoratiu Vultur 807c8349639SHoratiu Vultur if (skb_put_padto(skb, ETH_ZLEN)) { 808c8349639SHoratiu Vultur dev->stats.tx_dropped++; 809c8349639SHoratiu Vultur return NETDEV_TX_OK; 810c8349639SHoratiu Vultur } 811c8349639SHoratiu Vultur 812c8349639SHoratiu Vultur /* skb processing */ 813e83163b6SHoratiu Vultur needed_headroom = max_t(int, IFH_LEN_BYTES - skb_headroom(skb), 0); 814c8349639SHoratiu Vultur needed_tailroom = max_t(int, ETH_FCS_LEN - skb_tailroom(skb), 0); 815c8349639SHoratiu Vultur if (needed_headroom || needed_tailroom || skb_header_cloned(skb)) { 816c8349639SHoratiu Vultur err = pskb_expand_head(skb, needed_headroom, needed_tailroom, 817c8349639SHoratiu Vultur GFP_ATOMIC); 818c8349639SHoratiu Vultur if (unlikely(err)) { 819c8349639SHoratiu Vultur dev->stats.tx_dropped++; 820c8349639SHoratiu Vultur err = NETDEV_TX_OK; 821c8349639SHoratiu Vultur goto release; 822c8349639SHoratiu Vultur } 823c8349639SHoratiu Vultur } 824c8349639SHoratiu Vultur 825c8349639SHoratiu Vultur skb_tx_timestamp(skb); 826e83163b6SHoratiu Vultur skb_push(skb, IFH_LEN_BYTES); 827e83163b6SHoratiu Vultur memcpy(skb->data, ifh, IFH_LEN_BYTES); 828c8349639SHoratiu Vultur skb_put(skb, 4); 829c8349639SHoratiu Vultur 830c8349639SHoratiu Vultur dma_addr = dma_map_single(lan966x->dev, skb->data, skb->len, 831c8349639SHoratiu Vultur DMA_TO_DEVICE); 832c8349639SHoratiu Vultur if (dma_mapping_error(lan966x->dev, dma_addr)) { 833c8349639SHoratiu Vultur dev->stats.tx_dropped++; 834c8349639SHoratiu Vultur err = NETDEV_TX_OK; 835c8349639SHoratiu Vultur goto release; 836c8349639SHoratiu Vultur } 837c8349639SHoratiu Vultur 838c8349639SHoratiu Vultur /* Setup next dcb */ 8393d66bc57SHoratiu Vultur lan966x_fdma_tx_setup_dcb(tx, next_to_use, skb->len, dma_addr); 840c8349639SHoratiu Vultur 841c8349639SHoratiu Vultur /* Fill up the buffer */ 842c8349639SHoratiu Vultur next_dcb_buf = &tx->dcbs_buf[next_to_use]; 84319c6f534SHoratiu Vultur next_dcb_buf->use_skb = true; 84419c6f534SHoratiu Vultur next_dcb_buf->data.skb = skb; 845*a825b611SHoratiu Vultur next_dcb_buf->xdp_ndo = false; 84649f5eea8SHoratiu Vultur next_dcb_buf->len = skb->len; 847c8349639SHoratiu Vultur next_dcb_buf->dma_addr = dma_addr; 848c8349639SHoratiu Vultur next_dcb_buf->used = true; 849c8349639SHoratiu Vultur next_dcb_buf->ptp = false; 850c8349639SHoratiu Vultur next_dcb_buf->dev = dev; 851c8349639SHoratiu Vultur 852c8349639SHoratiu Vultur if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP && 853c8349639SHoratiu Vultur LAN966X_SKB_CB(skb)->rew_op == IFH_REW_OP_TWO_STEP_PTP) 854c8349639SHoratiu Vultur next_dcb_buf->ptp = true; 855c8349639SHoratiu Vultur 8563d66bc57SHoratiu Vultur /* Start the transmission */ 8573d66bc57SHoratiu Vultur lan966x_fdma_tx_start(tx, next_to_use); 858c8349639SHoratiu Vultur 859c8349639SHoratiu Vultur return NETDEV_TX_OK; 860c8349639SHoratiu Vultur 861c8349639SHoratiu Vultur release: 862c8349639SHoratiu Vultur if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP && 863c8349639SHoratiu Vultur LAN966X_SKB_CB(skb)->rew_op == IFH_REW_OP_TWO_STEP_PTP) 864c8349639SHoratiu Vultur lan966x_ptp_txtstamp_release(port, skb); 865c8349639SHoratiu Vultur 866c8349639SHoratiu Vultur dev_kfree_skb_any(skb); 867c8349639SHoratiu Vultur return err; 868c8349639SHoratiu Vultur } 869c8349639SHoratiu Vultur 8702ea1cbacSHoratiu Vultur static int lan966x_fdma_get_max_mtu(struct lan966x *lan966x) 8712ea1cbacSHoratiu Vultur { 8722ea1cbacSHoratiu Vultur int max_mtu = 0; 8732ea1cbacSHoratiu Vultur int i; 8742ea1cbacSHoratiu Vultur 8752ea1cbacSHoratiu Vultur for (i = 0; i < lan966x->num_phys_ports; ++i) { 876872ad758SHoratiu Vultur struct lan966x_port *port; 8772ea1cbacSHoratiu Vultur int mtu; 8782ea1cbacSHoratiu Vultur 879872ad758SHoratiu Vultur port = lan966x->ports[i]; 880872ad758SHoratiu Vultur if (!port) 8812ea1cbacSHoratiu Vultur continue; 8822ea1cbacSHoratiu Vultur 883872ad758SHoratiu Vultur mtu = lan_rd(lan966x, DEV_MAC_MAXLEN_CFG(port->chip_port)); 8842ea1cbacSHoratiu Vultur if (mtu > max_mtu) 8852ea1cbacSHoratiu Vultur max_mtu = mtu; 8862ea1cbacSHoratiu Vultur } 8872ea1cbacSHoratiu Vultur 8882ea1cbacSHoratiu Vultur return max_mtu; 8892ea1cbacSHoratiu Vultur } 8902ea1cbacSHoratiu Vultur 8912ea1cbacSHoratiu Vultur static int lan966x_qsys_sw_status(struct lan966x *lan966x) 8922ea1cbacSHoratiu Vultur { 8932ea1cbacSHoratiu Vultur return lan_rd(lan966x, QSYS_SW_STATUS(CPU_PORT)); 8942ea1cbacSHoratiu Vultur } 8952ea1cbacSHoratiu Vultur 8962ea1cbacSHoratiu Vultur static int lan966x_fdma_reload(struct lan966x *lan966x, int new_mtu) 8972ea1cbacSHoratiu Vultur { 89811871abaSHoratiu Vultur struct page_pool *page_pool; 8994a4b6848SHoratiu Vultur dma_addr_t rx_dma; 9004a4b6848SHoratiu Vultur void *rx_dcbs; 9012ea1cbacSHoratiu Vultur u32 size; 9022ea1cbacSHoratiu Vultur int err; 9032ea1cbacSHoratiu Vultur 9042ea1cbacSHoratiu Vultur /* Store these for later to free them */ 9052ea1cbacSHoratiu Vultur rx_dma = lan966x->rx.dma; 9062ea1cbacSHoratiu Vultur rx_dcbs = lan966x->rx.dcbs; 90711871abaSHoratiu Vultur page_pool = lan966x->rx.page_pool; 9082ea1cbacSHoratiu Vultur 9092ea1cbacSHoratiu Vultur napi_synchronize(&lan966x->napi); 9102ea1cbacSHoratiu Vultur napi_disable(&lan966x->napi); 9112ea1cbacSHoratiu Vultur lan966x_fdma_stop_netdev(lan966x); 9122ea1cbacSHoratiu Vultur 9132ea1cbacSHoratiu Vultur lan966x_fdma_rx_disable(&lan966x->rx); 9142ea1cbacSHoratiu Vultur lan966x_fdma_rx_free_pages(&lan966x->rx); 9152ea1cbacSHoratiu Vultur lan966x->rx.page_order = round_up(new_mtu, PAGE_SIZE) / PAGE_SIZE - 1; 91611871abaSHoratiu Vultur lan966x->rx.max_mtu = new_mtu; 9172ea1cbacSHoratiu Vultur err = lan966x_fdma_rx_alloc(&lan966x->rx); 9182ea1cbacSHoratiu Vultur if (err) 9192ea1cbacSHoratiu Vultur goto restore; 9202ea1cbacSHoratiu Vultur lan966x_fdma_rx_start(&lan966x->rx); 9212ea1cbacSHoratiu Vultur 9222ea1cbacSHoratiu Vultur size = sizeof(struct lan966x_rx_dcb) * FDMA_DCB_MAX; 9232ea1cbacSHoratiu Vultur size = ALIGN(size, PAGE_SIZE); 9242ea1cbacSHoratiu Vultur dma_free_coherent(lan966x->dev, size, rx_dcbs, rx_dma); 9252ea1cbacSHoratiu Vultur 92611871abaSHoratiu Vultur page_pool_destroy(page_pool); 92711871abaSHoratiu Vultur 9282ea1cbacSHoratiu Vultur lan966x_fdma_wakeup_netdev(lan966x); 9292ea1cbacSHoratiu Vultur napi_enable(&lan966x->napi); 9302ea1cbacSHoratiu Vultur 9312ea1cbacSHoratiu Vultur return err; 9322ea1cbacSHoratiu Vultur restore: 93311871abaSHoratiu Vultur lan966x->rx.page_pool = page_pool; 9342ea1cbacSHoratiu Vultur lan966x->rx.dma = rx_dma; 935f0a65f81SHoratiu Vultur lan966x->rx.dcbs = rx_dcbs; 9362ea1cbacSHoratiu Vultur lan966x_fdma_rx_start(&lan966x->rx); 9372ea1cbacSHoratiu Vultur 9382ea1cbacSHoratiu Vultur return err; 9392ea1cbacSHoratiu Vultur } 9402ea1cbacSHoratiu Vultur 94111871abaSHoratiu Vultur static int lan966x_fdma_get_max_frame(struct lan966x *lan966x) 94211871abaSHoratiu Vultur { 94311871abaSHoratiu Vultur return lan966x_fdma_get_max_mtu(lan966x) + 94411871abaSHoratiu Vultur IFH_LEN_BYTES + 94511871abaSHoratiu Vultur SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + 9467292bb06SHoratiu Vultur VLAN_HLEN * 2 + 9477292bb06SHoratiu Vultur XDP_PACKET_HEADROOM; 94811871abaSHoratiu Vultur } 94911871abaSHoratiu Vultur 950560c7223SHoratiu Vultur static int __lan966x_fdma_reload(struct lan966x *lan966x, int max_mtu) 9512ea1cbacSHoratiu Vultur { 9522ea1cbacSHoratiu Vultur int err; 9532ea1cbacSHoratiu Vultur u32 val; 9542ea1cbacSHoratiu Vultur 9552ea1cbacSHoratiu Vultur /* Disable the CPU port */ 9562ea1cbacSHoratiu Vultur lan_rmw(QSYS_SW_PORT_MODE_PORT_ENA_SET(0), 9572ea1cbacSHoratiu Vultur QSYS_SW_PORT_MODE_PORT_ENA, 9582ea1cbacSHoratiu Vultur lan966x, QSYS_SW_PORT_MODE(CPU_PORT)); 9592ea1cbacSHoratiu Vultur 9602ea1cbacSHoratiu Vultur /* Flush the CPU queues */ 9612ea1cbacSHoratiu Vultur readx_poll_timeout(lan966x_qsys_sw_status, lan966x, 9622ea1cbacSHoratiu Vultur val, !(QSYS_SW_STATUS_EQ_AVAIL_GET(val)), 9632ea1cbacSHoratiu Vultur READL_SLEEP_US, READL_TIMEOUT_US); 9642ea1cbacSHoratiu Vultur 9652ea1cbacSHoratiu Vultur /* Add a sleep in case there are frames between the queues and the CPU 9662ea1cbacSHoratiu Vultur * port 9672ea1cbacSHoratiu Vultur */ 9682ea1cbacSHoratiu Vultur usleep_range(1000, 2000); 9692ea1cbacSHoratiu Vultur 9702ea1cbacSHoratiu Vultur err = lan966x_fdma_reload(lan966x, max_mtu); 9712ea1cbacSHoratiu Vultur 9722ea1cbacSHoratiu Vultur /* Enable back the CPU port */ 9732ea1cbacSHoratiu Vultur lan_rmw(QSYS_SW_PORT_MODE_PORT_ENA_SET(1), 9742ea1cbacSHoratiu Vultur QSYS_SW_PORT_MODE_PORT_ENA, 9752ea1cbacSHoratiu Vultur lan966x, QSYS_SW_PORT_MODE(CPU_PORT)); 9762ea1cbacSHoratiu Vultur 9772ea1cbacSHoratiu Vultur return err; 9782ea1cbacSHoratiu Vultur } 9792ea1cbacSHoratiu Vultur 980560c7223SHoratiu Vultur int lan966x_fdma_change_mtu(struct lan966x *lan966x) 981560c7223SHoratiu Vultur { 982560c7223SHoratiu Vultur int max_mtu; 983560c7223SHoratiu Vultur 984560c7223SHoratiu Vultur max_mtu = lan966x_fdma_get_max_frame(lan966x); 985560c7223SHoratiu Vultur if (max_mtu == lan966x->rx.max_mtu) 986560c7223SHoratiu Vultur return 0; 987560c7223SHoratiu Vultur 988560c7223SHoratiu Vultur return __lan966x_fdma_reload(lan966x, max_mtu); 989560c7223SHoratiu Vultur } 990560c7223SHoratiu Vultur 991560c7223SHoratiu Vultur int lan966x_fdma_reload_page_pool(struct lan966x *lan966x) 992560c7223SHoratiu Vultur { 993560c7223SHoratiu Vultur int max_mtu; 994560c7223SHoratiu Vultur 995560c7223SHoratiu Vultur max_mtu = lan966x_fdma_get_max_frame(lan966x); 996560c7223SHoratiu Vultur return __lan966x_fdma_reload(lan966x, max_mtu); 997560c7223SHoratiu Vultur } 998560c7223SHoratiu Vultur 999c8349639SHoratiu Vultur void lan966x_fdma_netdev_init(struct lan966x *lan966x, struct net_device *dev) 1000c8349639SHoratiu Vultur { 1001c8349639SHoratiu Vultur if (lan966x->fdma_ndev) 1002c8349639SHoratiu Vultur return; 1003c8349639SHoratiu Vultur 1004c8349639SHoratiu Vultur lan966x->fdma_ndev = dev; 1005b48b89f9SJakub Kicinski netif_napi_add(dev, &lan966x->napi, lan966x_fdma_napi_poll); 1006c8349639SHoratiu Vultur napi_enable(&lan966x->napi); 1007c8349639SHoratiu Vultur } 1008c8349639SHoratiu Vultur 1009c8349639SHoratiu Vultur void lan966x_fdma_netdev_deinit(struct lan966x *lan966x, struct net_device *dev) 1010c8349639SHoratiu Vultur { 1011c8349639SHoratiu Vultur if (lan966x->fdma_ndev == dev) { 1012c8349639SHoratiu Vultur netif_napi_del(&lan966x->napi); 1013c8349639SHoratiu Vultur lan966x->fdma_ndev = NULL; 1014c8349639SHoratiu Vultur } 1015c8349639SHoratiu Vultur } 1016c8349639SHoratiu Vultur 1017c8349639SHoratiu Vultur int lan966x_fdma_init(struct lan966x *lan966x) 1018c8349639SHoratiu Vultur { 1019c8349639SHoratiu Vultur int err; 1020c8349639SHoratiu Vultur 1021c8349639SHoratiu Vultur if (!lan966x->fdma) 1022c8349639SHoratiu Vultur return 0; 1023c8349639SHoratiu Vultur 1024c8349639SHoratiu Vultur lan966x->rx.lan966x = lan966x; 1025c8349639SHoratiu Vultur lan966x->rx.channel_id = FDMA_XTR_CHANNEL; 102611871abaSHoratiu Vultur lan966x->rx.max_mtu = lan966x_fdma_get_max_frame(lan966x); 1027c8349639SHoratiu Vultur lan966x->tx.lan966x = lan966x; 1028c8349639SHoratiu Vultur lan966x->tx.channel_id = FDMA_INJ_CHANNEL; 1029c8349639SHoratiu Vultur lan966x->tx.last_in_use = -1; 1030c8349639SHoratiu Vultur 1031c8349639SHoratiu Vultur err = lan966x_fdma_rx_alloc(&lan966x->rx); 1032c8349639SHoratiu Vultur if (err) 1033c8349639SHoratiu Vultur return err; 1034c8349639SHoratiu Vultur 1035c8349639SHoratiu Vultur err = lan966x_fdma_tx_alloc(&lan966x->tx); 1036c8349639SHoratiu Vultur if (err) { 1037c8349639SHoratiu Vultur lan966x_fdma_rx_free(&lan966x->rx); 1038c8349639SHoratiu Vultur return err; 1039c8349639SHoratiu Vultur } 1040c8349639SHoratiu Vultur 1041c8349639SHoratiu Vultur lan966x_fdma_rx_start(&lan966x->rx); 1042c8349639SHoratiu Vultur 1043c8349639SHoratiu Vultur return 0; 1044c8349639SHoratiu Vultur } 1045c8349639SHoratiu Vultur 1046c8349639SHoratiu Vultur void lan966x_fdma_deinit(struct lan966x *lan966x) 1047c8349639SHoratiu Vultur { 1048c8349639SHoratiu Vultur if (!lan966x->fdma) 1049c8349639SHoratiu Vultur return; 1050c8349639SHoratiu Vultur 1051c8349639SHoratiu Vultur lan966x_fdma_rx_disable(&lan966x->rx); 1052c8349639SHoratiu Vultur lan966x_fdma_tx_disable(&lan966x->tx); 1053c8349639SHoratiu Vultur 1054c8349639SHoratiu Vultur napi_synchronize(&lan966x->napi); 1055c8349639SHoratiu Vultur napi_disable(&lan966x->napi); 1056c8349639SHoratiu Vultur 1057c8349639SHoratiu Vultur lan966x_fdma_rx_free_pages(&lan966x->rx); 1058c8349639SHoratiu Vultur lan966x_fdma_rx_free(&lan966x->rx); 105911871abaSHoratiu Vultur page_pool_destroy(lan966x->rx.page_pool); 1060c8349639SHoratiu Vultur lan966x_fdma_tx_free(&lan966x->tx); 1061c8349639SHoratiu Vultur } 1062