1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (C) 2021 Gerhard Engleder <gerhard@engleder-embedded.com> */ 3 4 /* TSN endpoint Ethernet MAC driver 5 * 6 * The TSN endpoint Ethernet MAC is a FPGA based network device for real-time 7 * communication. It is designed for endpoints within TSN (Time Sensitive 8 * Networking) networks; e.g., for PLCs in the industrial automation case. 9 * 10 * It supports multiple TX/RX queue pairs. The first TX/RX queue pair is used 11 * by the driver. 12 * 13 * More information can be found here: 14 * - www.embedded-experts.at/tsn 15 * - www.engleder-embedded.com 16 */ 17 18 #include "tsnep.h" 19 #include "tsnep_hw.h" 20 21 #include <linux/module.h> 22 #include <linux/of.h> 23 #include <linux/of_net.h> 24 #include <linux/of_mdio.h> 25 #include <linux/interrupt.h> 26 #include <linux/etherdevice.h> 27 #include <linux/phy.h> 28 #include <linux/iopoll.h> 29 #include <linux/bpf.h> 30 #include <linux/bpf_trace.h> 31 #include <net/page_pool/helpers.h> 32 #include <net/xdp_sock_drv.h> 33 34 #define TSNEP_RX_OFFSET (max(NET_SKB_PAD, XDP_PACKET_HEADROOM) + NET_IP_ALIGN) 35 #define TSNEP_HEADROOM ALIGN(TSNEP_RX_OFFSET, 4) 36 #define TSNEP_MAX_RX_BUF_SIZE (PAGE_SIZE - TSNEP_HEADROOM - \ 37 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) 38 /* XSK buffer shall store at least Q-in-Q frame */ 39 #define TSNEP_XSK_RX_BUF_SIZE (ALIGN(TSNEP_RX_INLINE_METADATA_SIZE + \ 40 ETH_FRAME_LEN + ETH_FCS_LEN + \ 41 VLAN_HLEN * 2, 4)) 42 43 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 44 #define DMA_ADDR_HIGH(dma_addr) ((u32)(((dma_addr) >> 32) & 0xFFFFFFFF)) 45 #else 46 #define DMA_ADDR_HIGH(dma_addr) ((u32)(0)) 47 #endif 48 #define DMA_ADDR_LOW(dma_addr) ((u32)((dma_addr) & 0xFFFFFFFF)) 49 50 #define TSNEP_COALESCE_USECS_DEFAULT 64 51 #define TSNEP_COALESCE_USECS_MAX ((ECM_INT_DELAY_MASK >> ECM_INT_DELAY_SHIFT) * \ 52 ECM_INT_DELAY_BASE_US + ECM_INT_DELAY_BASE_US - 1) 53 54 /* mapping type */ 55 #define TSNEP_TX_TYPE_MAP BIT(0) 56 #define TSNEP_TX_TYPE_MAP_PAGE BIT(1) 57 #define TSNEP_TX_TYPE_INLINE BIT(2) 58 /* buffer type */ 59 #define TSNEP_TX_TYPE_SKB BIT(8) 60 #define TSNEP_TX_TYPE_SKB_MAP (TSNEP_TX_TYPE_SKB | TSNEP_TX_TYPE_MAP) 61 #define TSNEP_TX_TYPE_SKB_INLINE (TSNEP_TX_TYPE_SKB | TSNEP_TX_TYPE_INLINE) 62 #define TSNEP_TX_TYPE_SKB_FRAG BIT(9) 63 #define TSNEP_TX_TYPE_SKB_FRAG_MAP_PAGE (TSNEP_TX_TYPE_SKB_FRAG | TSNEP_TX_TYPE_MAP_PAGE) 64 #define TSNEP_TX_TYPE_SKB_FRAG_INLINE (TSNEP_TX_TYPE_SKB_FRAG | TSNEP_TX_TYPE_INLINE) 65 #define TSNEP_TX_TYPE_XDP_TX BIT(10) 66 #define TSNEP_TX_TYPE_XDP_NDO BIT(11) 67 #define TSNEP_TX_TYPE_XDP_NDO_MAP_PAGE (TSNEP_TX_TYPE_XDP_NDO | TSNEP_TX_TYPE_MAP_PAGE) 68 #define TSNEP_TX_TYPE_XDP (TSNEP_TX_TYPE_XDP_TX | TSNEP_TX_TYPE_XDP_NDO) 69 #define TSNEP_TX_TYPE_XSK BIT(12) 70 71 #define TSNEP_XDP_TX BIT(0) 72 #define TSNEP_XDP_REDIRECT BIT(1) 73 74 static void tsnep_enable_irq(struct tsnep_adapter *adapter, u32 mask) 75 { 76 iowrite32(mask, adapter->addr + ECM_INT_ENABLE); 77 } 78 79 static void tsnep_disable_irq(struct tsnep_adapter *adapter, u32 mask) 80 { 81 mask |= ECM_INT_DISABLE; 82 iowrite32(mask, adapter->addr + ECM_INT_ENABLE); 83 } 84 85 static irqreturn_t tsnep_irq(int irq, void *arg) 86 { 87 struct tsnep_adapter *adapter = arg; 88 u32 active = ioread32(adapter->addr + ECM_INT_ACTIVE); 89 90 /* acknowledge interrupt */ 91 if (active != 0) 92 iowrite32(active, adapter->addr + ECM_INT_ACKNOWLEDGE); 93 94 /* handle link interrupt */ 95 if ((active & ECM_INT_LINK) != 0) 96 phy_mac_interrupt(adapter->netdev->phydev); 97 98 /* handle TX/RX queue 0 interrupt */ 99 if ((active & adapter->queue[0].irq_mask) != 0) { 100 if (napi_schedule_prep(&adapter->queue[0].napi)) { 101 tsnep_disable_irq(adapter, adapter->queue[0].irq_mask); 102 /* schedule after masking to avoid races */ 103 __napi_schedule(&adapter->queue[0].napi); 104 } 105 } 106 107 return IRQ_HANDLED; 108 } 109 110 static irqreturn_t tsnep_irq_txrx(int irq, void *arg) 111 { 112 struct tsnep_queue *queue = arg; 113 114 /* handle TX/RX queue interrupt */ 115 if (napi_schedule_prep(&queue->napi)) { 116 tsnep_disable_irq(queue->adapter, queue->irq_mask); 117 /* schedule after masking to avoid races */ 118 __napi_schedule(&queue->napi); 119 } 120 121 return IRQ_HANDLED; 122 } 123 124 int tsnep_set_irq_coalesce(struct tsnep_queue *queue, u32 usecs) 125 { 126 if (usecs > TSNEP_COALESCE_USECS_MAX) 127 return -ERANGE; 128 129 usecs /= ECM_INT_DELAY_BASE_US; 130 usecs <<= ECM_INT_DELAY_SHIFT; 131 usecs &= ECM_INT_DELAY_MASK; 132 133 queue->irq_delay &= ~ECM_INT_DELAY_MASK; 134 queue->irq_delay |= usecs; 135 iowrite8(queue->irq_delay, queue->irq_delay_addr); 136 137 return 0; 138 } 139 140 u32 tsnep_get_irq_coalesce(struct tsnep_queue *queue) 141 { 142 u32 usecs; 143 144 usecs = (queue->irq_delay & ECM_INT_DELAY_MASK); 145 usecs >>= ECM_INT_DELAY_SHIFT; 146 usecs *= ECM_INT_DELAY_BASE_US; 147 148 return usecs; 149 } 150 151 static int tsnep_mdiobus_read(struct mii_bus *bus, int addr, int regnum) 152 { 153 struct tsnep_adapter *adapter = bus->priv; 154 u32 md; 155 int retval; 156 157 md = ECM_MD_READ; 158 if (!adapter->suppress_preamble) 159 md |= ECM_MD_PREAMBLE; 160 md |= (regnum << ECM_MD_ADDR_SHIFT) & ECM_MD_ADDR_MASK; 161 md |= (addr << ECM_MD_PHY_ADDR_SHIFT) & ECM_MD_PHY_ADDR_MASK; 162 iowrite32(md, adapter->addr + ECM_MD_CONTROL); 163 retval = readl_poll_timeout_atomic(adapter->addr + ECM_MD_STATUS, md, 164 !(md & ECM_MD_BUSY), 16, 1000); 165 if (retval != 0) 166 return retval; 167 168 return (md & ECM_MD_DATA_MASK) >> ECM_MD_DATA_SHIFT; 169 } 170 171 static int tsnep_mdiobus_write(struct mii_bus *bus, int addr, int regnum, 172 u16 val) 173 { 174 struct tsnep_adapter *adapter = bus->priv; 175 u32 md; 176 int retval; 177 178 md = ECM_MD_WRITE; 179 if (!adapter->suppress_preamble) 180 md |= ECM_MD_PREAMBLE; 181 md |= (regnum << ECM_MD_ADDR_SHIFT) & ECM_MD_ADDR_MASK; 182 md |= (addr << ECM_MD_PHY_ADDR_SHIFT) & ECM_MD_PHY_ADDR_MASK; 183 md |= ((u32)val << ECM_MD_DATA_SHIFT) & ECM_MD_DATA_MASK; 184 iowrite32(md, adapter->addr + ECM_MD_CONTROL); 185 retval = readl_poll_timeout_atomic(adapter->addr + ECM_MD_STATUS, md, 186 !(md & ECM_MD_BUSY), 16, 1000); 187 if (retval != 0) 188 return retval; 189 190 return 0; 191 } 192 193 static void tsnep_set_link_mode(struct tsnep_adapter *adapter) 194 { 195 u32 mode; 196 197 switch (adapter->phydev->speed) { 198 case SPEED_100: 199 mode = ECM_LINK_MODE_100; 200 break; 201 case SPEED_1000: 202 mode = ECM_LINK_MODE_1000; 203 break; 204 default: 205 mode = ECM_LINK_MODE_OFF; 206 break; 207 } 208 iowrite32(mode, adapter->addr + ECM_STATUS); 209 } 210 211 static void tsnep_phy_link_status_change(struct net_device *netdev) 212 { 213 struct tsnep_adapter *adapter = netdev_priv(netdev); 214 struct phy_device *phydev = netdev->phydev; 215 216 if (phydev->link) 217 tsnep_set_link_mode(adapter); 218 219 phy_print_status(netdev->phydev); 220 } 221 222 static int tsnep_phy_loopback(struct tsnep_adapter *adapter, bool enable) 223 { 224 int retval; 225 226 retval = phy_loopback(adapter->phydev, enable); 227 228 /* PHY link state change is not signaled if loopback is enabled, it 229 * would delay a working loopback anyway, let's ensure that loopback 230 * is working immediately by setting link mode directly 231 */ 232 if (!retval && enable) 233 tsnep_set_link_mode(adapter); 234 235 return retval; 236 } 237 238 static int tsnep_phy_open(struct tsnep_adapter *adapter) 239 { 240 struct phy_device *phydev; 241 struct ethtool_eee ethtool_eee; 242 int retval; 243 244 retval = phy_connect_direct(adapter->netdev, adapter->phydev, 245 tsnep_phy_link_status_change, 246 adapter->phy_mode); 247 if (retval) 248 return retval; 249 phydev = adapter->netdev->phydev; 250 251 /* MAC supports only 100Mbps|1000Mbps full duplex 252 * SPE (Single Pair Ethernet) is also an option but not implemented yet 253 */ 254 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_10baseT_Half_BIT); 255 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_10baseT_Full_BIT); 256 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_100baseT_Half_BIT); 257 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_1000baseT_Half_BIT); 258 259 /* disable EEE autoneg, EEE not supported by TSNEP */ 260 memset(ðtool_eee, 0, sizeof(ethtool_eee)); 261 phy_ethtool_set_eee(adapter->phydev, ðtool_eee); 262 263 adapter->phydev->irq = PHY_MAC_INTERRUPT; 264 phy_start(adapter->phydev); 265 266 return 0; 267 } 268 269 static void tsnep_phy_close(struct tsnep_adapter *adapter) 270 { 271 phy_stop(adapter->netdev->phydev); 272 phy_disconnect(adapter->netdev->phydev); 273 } 274 275 static void tsnep_tx_ring_cleanup(struct tsnep_tx *tx) 276 { 277 struct device *dmadev = tx->adapter->dmadev; 278 int i; 279 280 memset(tx->entry, 0, sizeof(tx->entry)); 281 282 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) { 283 if (tx->page[i]) { 284 dma_free_coherent(dmadev, PAGE_SIZE, tx->page[i], 285 tx->page_dma[i]); 286 tx->page[i] = NULL; 287 tx->page_dma[i] = 0; 288 } 289 } 290 } 291 292 static int tsnep_tx_ring_create(struct tsnep_tx *tx) 293 { 294 struct device *dmadev = tx->adapter->dmadev; 295 struct tsnep_tx_entry *entry; 296 struct tsnep_tx_entry *next_entry; 297 int i, j; 298 int retval; 299 300 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) { 301 tx->page[i] = 302 dma_alloc_coherent(dmadev, PAGE_SIZE, &tx->page_dma[i], 303 GFP_KERNEL); 304 if (!tx->page[i]) { 305 retval = -ENOMEM; 306 goto alloc_failed; 307 } 308 for (j = 0; j < TSNEP_RING_ENTRIES_PER_PAGE; j++) { 309 entry = &tx->entry[TSNEP_RING_ENTRIES_PER_PAGE * i + j]; 310 entry->desc_wb = (struct tsnep_tx_desc_wb *) 311 (((u8 *)tx->page[i]) + TSNEP_DESC_SIZE * j); 312 entry->desc = (struct tsnep_tx_desc *) 313 (((u8 *)entry->desc_wb) + TSNEP_DESC_OFFSET); 314 entry->desc_dma = tx->page_dma[i] + TSNEP_DESC_SIZE * j; 315 entry->owner_user_flag = false; 316 } 317 } 318 for (i = 0; i < TSNEP_RING_SIZE; i++) { 319 entry = &tx->entry[i]; 320 next_entry = &tx->entry[(i + 1) & TSNEP_RING_MASK]; 321 entry->desc->next = __cpu_to_le64(next_entry->desc_dma); 322 } 323 324 return 0; 325 326 alloc_failed: 327 tsnep_tx_ring_cleanup(tx); 328 return retval; 329 } 330 331 static void tsnep_tx_init(struct tsnep_tx *tx) 332 { 333 dma_addr_t dma; 334 335 dma = tx->entry[0].desc_dma | TSNEP_RESET_OWNER_COUNTER; 336 iowrite32(DMA_ADDR_LOW(dma), tx->addr + TSNEP_TX_DESC_ADDR_LOW); 337 iowrite32(DMA_ADDR_HIGH(dma), tx->addr + TSNEP_TX_DESC_ADDR_HIGH); 338 tx->write = 0; 339 tx->read = 0; 340 tx->owner_counter = 1; 341 tx->increment_owner_counter = TSNEP_RING_SIZE - 1; 342 } 343 344 static void tsnep_tx_enable(struct tsnep_tx *tx) 345 { 346 struct netdev_queue *nq; 347 348 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index); 349 350 __netif_tx_lock_bh(nq); 351 netif_tx_wake_queue(nq); 352 __netif_tx_unlock_bh(nq); 353 } 354 355 static void tsnep_tx_disable(struct tsnep_tx *tx, struct napi_struct *napi) 356 { 357 struct netdev_queue *nq; 358 u32 val; 359 360 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index); 361 362 __netif_tx_lock_bh(nq); 363 netif_tx_stop_queue(nq); 364 __netif_tx_unlock_bh(nq); 365 366 /* wait until TX is done in hardware */ 367 readx_poll_timeout(ioread32, tx->addr + TSNEP_CONTROL, val, 368 ((val & TSNEP_CONTROL_TX_ENABLE) == 0), 10000, 369 1000000); 370 371 /* wait until TX is also done in software */ 372 while (READ_ONCE(tx->read) != tx->write) { 373 napi_schedule(napi); 374 napi_synchronize(napi); 375 } 376 } 377 378 static void tsnep_tx_activate(struct tsnep_tx *tx, int index, int length, 379 bool last) 380 { 381 struct tsnep_tx_entry *entry = &tx->entry[index]; 382 383 entry->properties = 0; 384 /* xdpf and zc are union with skb */ 385 if (entry->skb) { 386 entry->properties = length & TSNEP_DESC_LENGTH_MASK; 387 entry->properties |= TSNEP_DESC_INTERRUPT_FLAG; 388 if ((entry->type & TSNEP_TX_TYPE_SKB) && 389 (skb_shinfo(entry->skb)->tx_flags & SKBTX_IN_PROGRESS)) 390 entry->properties |= TSNEP_DESC_EXTENDED_WRITEBACK_FLAG; 391 392 /* toggle user flag to prevent false acknowledge 393 * 394 * Only the first fragment is acknowledged. For all other 395 * fragments no acknowledge is done and the last written owner 396 * counter stays in the writeback descriptor. Therefore, it is 397 * possible that the last written owner counter is identical to 398 * the new incremented owner counter and a false acknowledge is 399 * detected before the real acknowledge has been done by 400 * hardware. 401 * 402 * The user flag is used to prevent this situation. The user 403 * flag is copied to the writeback descriptor by the hardware 404 * and is used as additional acknowledge data. By toggeling the 405 * user flag only for the first fragment (which is 406 * acknowledged), it is guaranteed that the last acknowledge 407 * done for this descriptor has used a different user flag and 408 * cannot be detected as false acknowledge. 409 */ 410 entry->owner_user_flag = !entry->owner_user_flag; 411 } 412 if (last) 413 entry->properties |= TSNEP_TX_DESC_LAST_FRAGMENT_FLAG; 414 if (index == tx->increment_owner_counter) { 415 tx->owner_counter++; 416 if (tx->owner_counter == 4) 417 tx->owner_counter = 1; 418 tx->increment_owner_counter--; 419 if (tx->increment_owner_counter < 0) 420 tx->increment_owner_counter = TSNEP_RING_SIZE - 1; 421 } 422 entry->properties |= 423 (tx->owner_counter << TSNEP_DESC_OWNER_COUNTER_SHIFT) & 424 TSNEP_DESC_OWNER_COUNTER_MASK; 425 if (entry->owner_user_flag) 426 entry->properties |= TSNEP_TX_DESC_OWNER_USER_FLAG; 427 entry->desc->more_properties = 428 __cpu_to_le32(entry->len & TSNEP_DESC_LENGTH_MASK); 429 if (entry->type & TSNEP_TX_TYPE_INLINE) 430 entry->properties |= TSNEP_TX_DESC_DATA_AFTER_DESC_FLAG; 431 432 /* descriptor properties shall be written last, because valid data is 433 * signaled there 434 */ 435 dma_wmb(); 436 437 entry->desc->properties = __cpu_to_le32(entry->properties); 438 } 439 440 static int tsnep_tx_desc_available(struct tsnep_tx *tx) 441 { 442 if (tx->read <= tx->write) 443 return TSNEP_RING_SIZE - tx->write + tx->read - 1; 444 else 445 return tx->read - tx->write - 1; 446 } 447 448 static int tsnep_tx_map_frag(skb_frag_t *frag, struct tsnep_tx_entry *entry, 449 struct device *dmadev, dma_addr_t *dma) 450 { 451 unsigned int len; 452 int mapped; 453 454 len = skb_frag_size(frag); 455 if (likely(len > TSNEP_DESC_SIZE_DATA_AFTER_INLINE)) { 456 *dma = skb_frag_dma_map(dmadev, frag, 0, len, DMA_TO_DEVICE); 457 if (dma_mapping_error(dmadev, *dma)) 458 return -ENOMEM; 459 entry->type = TSNEP_TX_TYPE_SKB_FRAG_MAP_PAGE; 460 mapped = 1; 461 } else { 462 void *fragdata = skb_frag_address_safe(frag); 463 464 if (likely(fragdata)) { 465 memcpy(&entry->desc->tx, fragdata, len); 466 } else { 467 struct page *page = skb_frag_page(frag); 468 469 fragdata = kmap_local_page(page); 470 memcpy(&entry->desc->tx, fragdata + skb_frag_off(frag), 471 len); 472 kunmap_local(fragdata); 473 } 474 entry->type = TSNEP_TX_TYPE_SKB_FRAG_INLINE; 475 mapped = 0; 476 } 477 478 return mapped; 479 } 480 481 static int tsnep_tx_map(struct sk_buff *skb, struct tsnep_tx *tx, int count) 482 { 483 struct device *dmadev = tx->adapter->dmadev; 484 struct tsnep_tx_entry *entry; 485 unsigned int len; 486 int map_len = 0; 487 dma_addr_t dma; 488 int i, mapped; 489 490 for (i = 0; i < count; i++) { 491 entry = &tx->entry[(tx->write + i) & TSNEP_RING_MASK]; 492 493 if (!i) { 494 len = skb_headlen(skb); 495 if (likely(len > TSNEP_DESC_SIZE_DATA_AFTER_INLINE)) { 496 dma = dma_map_single(dmadev, skb->data, len, 497 DMA_TO_DEVICE); 498 if (dma_mapping_error(dmadev, dma)) 499 return -ENOMEM; 500 entry->type = TSNEP_TX_TYPE_SKB_MAP; 501 mapped = 1; 502 } else { 503 memcpy(&entry->desc->tx, skb->data, len); 504 entry->type = TSNEP_TX_TYPE_SKB_INLINE; 505 mapped = 0; 506 } 507 } else { 508 skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1]; 509 510 len = skb_frag_size(frag); 511 mapped = tsnep_tx_map_frag(frag, entry, dmadev, &dma); 512 if (mapped < 0) 513 return mapped; 514 } 515 516 entry->len = len; 517 if (likely(mapped)) { 518 dma_unmap_addr_set(entry, dma, dma); 519 entry->desc->tx = __cpu_to_le64(dma); 520 } 521 522 map_len += len; 523 } 524 525 return map_len; 526 } 527 528 static int tsnep_tx_unmap(struct tsnep_tx *tx, int index, int count) 529 { 530 struct device *dmadev = tx->adapter->dmadev; 531 struct tsnep_tx_entry *entry; 532 int map_len = 0; 533 int i; 534 535 for (i = 0; i < count; i++) { 536 entry = &tx->entry[(index + i) & TSNEP_RING_MASK]; 537 538 if (entry->len) { 539 if (entry->type & TSNEP_TX_TYPE_MAP) 540 dma_unmap_single(dmadev, 541 dma_unmap_addr(entry, dma), 542 dma_unmap_len(entry, len), 543 DMA_TO_DEVICE); 544 else if (entry->type & TSNEP_TX_TYPE_MAP_PAGE) 545 dma_unmap_page(dmadev, 546 dma_unmap_addr(entry, dma), 547 dma_unmap_len(entry, len), 548 DMA_TO_DEVICE); 549 map_len += entry->len; 550 entry->len = 0; 551 } 552 } 553 554 return map_len; 555 } 556 557 static netdev_tx_t tsnep_xmit_frame_ring(struct sk_buff *skb, 558 struct tsnep_tx *tx) 559 { 560 int count = 1; 561 struct tsnep_tx_entry *entry; 562 int length; 563 int i; 564 int retval; 565 566 if (skb_shinfo(skb)->nr_frags > 0) 567 count += skb_shinfo(skb)->nr_frags; 568 569 if (tsnep_tx_desc_available(tx) < count) { 570 /* ring full, shall not happen because queue is stopped if full 571 * below 572 */ 573 netif_stop_subqueue(tx->adapter->netdev, tx->queue_index); 574 575 return NETDEV_TX_BUSY; 576 } 577 578 entry = &tx->entry[tx->write]; 579 entry->skb = skb; 580 581 retval = tsnep_tx_map(skb, tx, count); 582 if (retval < 0) { 583 tsnep_tx_unmap(tx, tx->write, count); 584 dev_kfree_skb_any(entry->skb); 585 entry->skb = NULL; 586 587 tx->dropped++; 588 589 return NETDEV_TX_OK; 590 } 591 length = retval; 592 593 if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) 594 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 595 596 for (i = 0; i < count; i++) 597 tsnep_tx_activate(tx, (tx->write + i) & TSNEP_RING_MASK, length, 598 i == count - 1); 599 tx->write = (tx->write + count) & TSNEP_RING_MASK; 600 601 skb_tx_timestamp(skb); 602 603 /* descriptor properties shall be valid before hardware is notified */ 604 dma_wmb(); 605 606 iowrite32(TSNEP_CONTROL_TX_ENABLE, tx->addr + TSNEP_CONTROL); 607 608 if (tsnep_tx_desc_available(tx) < (MAX_SKB_FRAGS + 1)) { 609 /* ring can get full with next frame */ 610 netif_stop_subqueue(tx->adapter->netdev, tx->queue_index); 611 } 612 613 return NETDEV_TX_OK; 614 } 615 616 static int tsnep_xdp_tx_map(struct xdp_frame *xdpf, struct tsnep_tx *tx, 617 struct skb_shared_info *shinfo, int count, u32 type) 618 { 619 struct device *dmadev = tx->adapter->dmadev; 620 struct tsnep_tx_entry *entry; 621 struct page *page; 622 skb_frag_t *frag; 623 unsigned int len; 624 int map_len = 0; 625 dma_addr_t dma; 626 void *data; 627 int i; 628 629 frag = NULL; 630 len = xdpf->len; 631 for (i = 0; i < count; i++) { 632 entry = &tx->entry[(tx->write + i) & TSNEP_RING_MASK]; 633 if (type & TSNEP_TX_TYPE_XDP_NDO) { 634 data = unlikely(frag) ? skb_frag_address(frag) : 635 xdpf->data; 636 dma = dma_map_single(dmadev, data, len, DMA_TO_DEVICE); 637 if (dma_mapping_error(dmadev, dma)) 638 return -ENOMEM; 639 640 entry->type = TSNEP_TX_TYPE_XDP_NDO_MAP_PAGE; 641 } else { 642 page = unlikely(frag) ? skb_frag_page(frag) : 643 virt_to_page(xdpf->data); 644 dma = page_pool_get_dma_addr(page); 645 if (unlikely(frag)) 646 dma += skb_frag_off(frag); 647 else 648 dma += sizeof(*xdpf) + xdpf->headroom; 649 dma_sync_single_for_device(dmadev, dma, len, 650 DMA_BIDIRECTIONAL); 651 652 entry->type = TSNEP_TX_TYPE_XDP_TX; 653 } 654 655 entry->len = len; 656 dma_unmap_addr_set(entry, dma, dma); 657 658 entry->desc->tx = __cpu_to_le64(dma); 659 660 map_len += len; 661 662 if (i + 1 < count) { 663 frag = &shinfo->frags[i]; 664 len = skb_frag_size(frag); 665 } 666 } 667 668 return map_len; 669 } 670 671 /* This function requires __netif_tx_lock is held by the caller. */ 672 static bool tsnep_xdp_xmit_frame_ring(struct xdp_frame *xdpf, 673 struct tsnep_tx *tx, u32 type) 674 { 675 struct skb_shared_info *shinfo = xdp_get_shared_info_from_frame(xdpf); 676 struct tsnep_tx_entry *entry; 677 int count, length, retval, i; 678 679 count = 1; 680 if (unlikely(xdp_frame_has_frags(xdpf))) 681 count += shinfo->nr_frags; 682 683 /* ensure that TX ring is not filled up by XDP, always MAX_SKB_FRAGS 684 * will be available for normal TX path and queue is stopped there if 685 * necessary 686 */ 687 if (tsnep_tx_desc_available(tx) < (MAX_SKB_FRAGS + 1 + count)) 688 return false; 689 690 entry = &tx->entry[tx->write]; 691 entry->xdpf = xdpf; 692 693 retval = tsnep_xdp_tx_map(xdpf, tx, shinfo, count, type); 694 if (retval < 0) { 695 tsnep_tx_unmap(tx, tx->write, count); 696 entry->xdpf = NULL; 697 698 tx->dropped++; 699 700 return false; 701 } 702 length = retval; 703 704 for (i = 0; i < count; i++) 705 tsnep_tx_activate(tx, (tx->write + i) & TSNEP_RING_MASK, length, 706 i == count - 1); 707 tx->write = (tx->write + count) & TSNEP_RING_MASK; 708 709 /* descriptor properties shall be valid before hardware is notified */ 710 dma_wmb(); 711 712 return true; 713 } 714 715 static void tsnep_xdp_xmit_flush(struct tsnep_tx *tx) 716 { 717 iowrite32(TSNEP_CONTROL_TX_ENABLE, tx->addr + TSNEP_CONTROL); 718 } 719 720 static bool tsnep_xdp_xmit_back(struct tsnep_adapter *adapter, 721 struct xdp_buff *xdp, 722 struct netdev_queue *tx_nq, struct tsnep_tx *tx) 723 { 724 struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); 725 bool xmit; 726 727 if (unlikely(!xdpf)) 728 return false; 729 730 __netif_tx_lock(tx_nq, smp_processor_id()); 731 732 xmit = tsnep_xdp_xmit_frame_ring(xdpf, tx, TSNEP_TX_TYPE_XDP_TX); 733 734 /* Avoid transmit queue timeout since we share it with the slow path */ 735 if (xmit) 736 txq_trans_cond_update(tx_nq); 737 738 __netif_tx_unlock(tx_nq); 739 740 return xmit; 741 } 742 743 static int tsnep_xdp_tx_map_zc(struct xdp_desc *xdpd, struct tsnep_tx *tx) 744 { 745 struct tsnep_tx_entry *entry; 746 dma_addr_t dma; 747 748 entry = &tx->entry[tx->write]; 749 entry->zc = true; 750 751 dma = xsk_buff_raw_get_dma(tx->xsk_pool, xdpd->addr); 752 xsk_buff_raw_dma_sync_for_device(tx->xsk_pool, dma, xdpd->len); 753 754 entry->type = TSNEP_TX_TYPE_XSK; 755 entry->len = xdpd->len; 756 757 entry->desc->tx = __cpu_to_le64(dma); 758 759 return xdpd->len; 760 } 761 762 static void tsnep_xdp_xmit_frame_ring_zc(struct xdp_desc *xdpd, 763 struct tsnep_tx *tx) 764 { 765 int length; 766 767 length = tsnep_xdp_tx_map_zc(xdpd, tx); 768 769 tsnep_tx_activate(tx, tx->write, length, true); 770 tx->write = (tx->write + 1) & TSNEP_RING_MASK; 771 } 772 773 static void tsnep_xdp_xmit_zc(struct tsnep_tx *tx) 774 { 775 int desc_available = tsnep_tx_desc_available(tx); 776 struct xdp_desc *descs = tx->xsk_pool->tx_descs; 777 int batch, i; 778 779 /* ensure that TX ring is not filled up by XDP, always MAX_SKB_FRAGS 780 * will be available for normal TX path and queue is stopped there if 781 * necessary 782 */ 783 if (desc_available <= (MAX_SKB_FRAGS + 1)) 784 return; 785 desc_available -= MAX_SKB_FRAGS + 1; 786 787 batch = xsk_tx_peek_release_desc_batch(tx->xsk_pool, desc_available); 788 for (i = 0; i < batch; i++) 789 tsnep_xdp_xmit_frame_ring_zc(&descs[i], tx); 790 791 if (batch) { 792 /* descriptor properties shall be valid before hardware is 793 * notified 794 */ 795 dma_wmb(); 796 797 tsnep_xdp_xmit_flush(tx); 798 } 799 } 800 801 static bool tsnep_tx_poll(struct tsnep_tx *tx, int napi_budget) 802 { 803 struct tsnep_tx_entry *entry; 804 struct netdev_queue *nq; 805 int xsk_frames = 0; 806 int budget = 128; 807 int length; 808 int count; 809 810 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index); 811 __netif_tx_lock(nq, smp_processor_id()); 812 813 do { 814 if (tx->read == tx->write) 815 break; 816 817 entry = &tx->entry[tx->read]; 818 if ((__le32_to_cpu(entry->desc_wb->properties) & 819 TSNEP_TX_DESC_OWNER_MASK) != 820 (entry->properties & TSNEP_TX_DESC_OWNER_MASK)) 821 break; 822 823 /* descriptor properties shall be read first, because valid data 824 * is signaled there 825 */ 826 dma_rmb(); 827 828 count = 1; 829 if ((entry->type & TSNEP_TX_TYPE_SKB) && 830 skb_shinfo(entry->skb)->nr_frags > 0) 831 count += skb_shinfo(entry->skb)->nr_frags; 832 else if ((entry->type & TSNEP_TX_TYPE_XDP) && 833 xdp_frame_has_frags(entry->xdpf)) 834 count += xdp_get_shared_info_from_frame(entry->xdpf)->nr_frags; 835 836 length = tsnep_tx_unmap(tx, tx->read, count); 837 838 if ((entry->type & TSNEP_TX_TYPE_SKB) && 839 (skb_shinfo(entry->skb)->tx_flags & SKBTX_IN_PROGRESS) && 840 (__le32_to_cpu(entry->desc_wb->properties) & 841 TSNEP_DESC_EXTENDED_WRITEBACK_FLAG)) { 842 struct skb_shared_hwtstamps hwtstamps; 843 u64 timestamp; 844 845 if (skb_shinfo(entry->skb)->tx_flags & 846 SKBTX_HW_TSTAMP_USE_CYCLES) 847 timestamp = 848 __le64_to_cpu(entry->desc_wb->counter); 849 else 850 timestamp = 851 __le64_to_cpu(entry->desc_wb->timestamp); 852 853 memset(&hwtstamps, 0, sizeof(hwtstamps)); 854 hwtstamps.hwtstamp = ns_to_ktime(timestamp); 855 856 skb_tstamp_tx(entry->skb, &hwtstamps); 857 } 858 859 if (entry->type & TSNEP_TX_TYPE_SKB) 860 napi_consume_skb(entry->skb, napi_budget); 861 else if (entry->type & TSNEP_TX_TYPE_XDP) 862 xdp_return_frame_rx_napi(entry->xdpf); 863 else 864 xsk_frames++; 865 /* xdpf and zc are union with skb */ 866 entry->skb = NULL; 867 868 tx->read = (tx->read + count) & TSNEP_RING_MASK; 869 870 tx->packets++; 871 tx->bytes += length + ETH_FCS_LEN; 872 873 budget--; 874 } while (likely(budget)); 875 876 if (tx->xsk_pool) { 877 if (xsk_frames) 878 xsk_tx_completed(tx->xsk_pool, xsk_frames); 879 if (xsk_uses_need_wakeup(tx->xsk_pool)) 880 xsk_set_tx_need_wakeup(tx->xsk_pool); 881 tsnep_xdp_xmit_zc(tx); 882 } 883 884 if ((tsnep_tx_desc_available(tx) >= ((MAX_SKB_FRAGS + 1) * 2)) && 885 netif_tx_queue_stopped(nq)) { 886 netif_tx_wake_queue(nq); 887 } 888 889 __netif_tx_unlock(nq); 890 891 return budget != 0; 892 } 893 894 static bool tsnep_tx_pending(struct tsnep_tx *tx) 895 { 896 struct tsnep_tx_entry *entry; 897 struct netdev_queue *nq; 898 bool pending = false; 899 900 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index); 901 __netif_tx_lock(nq, smp_processor_id()); 902 903 if (tx->read != tx->write) { 904 entry = &tx->entry[tx->read]; 905 if ((__le32_to_cpu(entry->desc_wb->properties) & 906 TSNEP_TX_DESC_OWNER_MASK) == 907 (entry->properties & TSNEP_TX_DESC_OWNER_MASK)) 908 pending = true; 909 } 910 911 __netif_tx_unlock(nq); 912 913 return pending; 914 } 915 916 static int tsnep_tx_open(struct tsnep_tx *tx) 917 { 918 int retval; 919 920 retval = tsnep_tx_ring_create(tx); 921 if (retval) 922 return retval; 923 924 tsnep_tx_init(tx); 925 926 return 0; 927 } 928 929 static void tsnep_tx_close(struct tsnep_tx *tx) 930 { 931 tsnep_tx_ring_cleanup(tx); 932 } 933 934 static void tsnep_rx_ring_cleanup(struct tsnep_rx *rx) 935 { 936 struct device *dmadev = rx->adapter->dmadev; 937 struct tsnep_rx_entry *entry; 938 int i; 939 940 for (i = 0; i < TSNEP_RING_SIZE; i++) { 941 entry = &rx->entry[i]; 942 if (!rx->xsk_pool && entry->page) 943 page_pool_put_full_page(rx->page_pool, entry->page, 944 false); 945 if (rx->xsk_pool && entry->xdp) 946 xsk_buff_free(entry->xdp); 947 /* xdp is union with page */ 948 entry->page = NULL; 949 } 950 951 if (rx->page_pool) 952 page_pool_destroy(rx->page_pool); 953 954 memset(rx->entry, 0, sizeof(rx->entry)); 955 956 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) { 957 if (rx->page[i]) { 958 dma_free_coherent(dmadev, PAGE_SIZE, rx->page[i], 959 rx->page_dma[i]); 960 rx->page[i] = NULL; 961 rx->page_dma[i] = 0; 962 } 963 } 964 } 965 966 static int tsnep_rx_ring_create(struct tsnep_rx *rx) 967 { 968 struct device *dmadev = rx->adapter->dmadev; 969 struct tsnep_rx_entry *entry; 970 struct page_pool_params pp_params = { 0 }; 971 struct tsnep_rx_entry *next_entry; 972 int i, j; 973 int retval; 974 975 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) { 976 rx->page[i] = 977 dma_alloc_coherent(dmadev, PAGE_SIZE, &rx->page_dma[i], 978 GFP_KERNEL); 979 if (!rx->page[i]) { 980 retval = -ENOMEM; 981 goto failed; 982 } 983 for (j = 0; j < TSNEP_RING_ENTRIES_PER_PAGE; j++) { 984 entry = &rx->entry[TSNEP_RING_ENTRIES_PER_PAGE * i + j]; 985 entry->desc_wb = (struct tsnep_rx_desc_wb *) 986 (((u8 *)rx->page[i]) + TSNEP_DESC_SIZE * j); 987 entry->desc = (struct tsnep_rx_desc *) 988 (((u8 *)entry->desc_wb) + TSNEP_DESC_OFFSET); 989 entry->desc_dma = rx->page_dma[i] + TSNEP_DESC_SIZE * j; 990 } 991 } 992 993 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 994 pp_params.order = 0; 995 pp_params.pool_size = TSNEP_RING_SIZE; 996 pp_params.nid = dev_to_node(dmadev); 997 pp_params.dev = dmadev; 998 pp_params.dma_dir = DMA_BIDIRECTIONAL; 999 pp_params.max_len = TSNEP_MAX_RX_BUF_SIZE; 1000 pp_params.offset = TSNEP_RX_OFFSET; 1001 rx->page_pool = page_pool_create(&pp_params); 1002 if (IS_ERR(rx->page_pool)) { 1003 retval = PTR_ERR(rx->page_pool); 1004 rx->page_pool = NULL; 1005 goto failed; 1006 } 1007 1008 for (i = 0; i < TSNEP_RING_SIZE; i++) { 1009 entry = &rx->entry[i]; 1010 next_entry = &rx->entry[(i + 1) & TSNEP_RING_MASK]; 1011 entry->desc->next = __cpu_to_le64(next_entry->desc_dma); 1012 } 1013 1014 return 0; 1015 1016 failed: 1017 tsnep_rx_ring_cleanup(rx); 1018 return retval; 1019 } 1020 1021 static void tsnep_rx_init(struct tsnep_rx *rx) 1022 { 1023 dma_addr_t dma; 1024 1025 dma = rx->entry[0].desc_dma | TSNEP_RESET_OWNER_COUNTER; 1026 iowrite32(DMA_ADDR_LOW(dma), rx->addr + TSNEP_RX_DESC_ADDR_LOW); 1027 iowrite32(DMA_ADDR_HIGH(dma), rx->addr + TSNEP_RX_DESC_ADDR_HIGH); 1028 rx->write = 0; 1029 rx->read = 0; 1030 rx->owner_counter = 1; 1031 rx->increment_owner_counter = TSNEP_RING_SIZE - 1; 1032 } 1033 1034 static void tsnep_rx_enable(struct tsnep_rx *rx) 1035 { 1036 /* descriptor properties shall be valid before hardware is notified */ 1037 dma_wmb(); 1038 1039 iowrite32(TSNEP_CONTROL_RX_ENABLE, rx->addr + TSNEP_CONTROL); 1040 } 1041 1042 static void tsnep_rx_disable(struct tsnep_rx *rx) 1043 { 1044 u32 val; 1045 1046 iowrite32(TSNEP_CONTROL_RX_DISABLE, rx->addr + TSNEP_CONTROL); 1047 readx_poll_timeout(ioread32, rx->addr + TSNEP_CONTROL, val, 1048 ((val & TSNEP_CONTROL_RX_ENABLE) == 0), 10000, 1049 1000000); 1050 } 1051 1052 static int tsnep_rx_desc_available(struct tsnep_rx *rx) 1053 { 1054 if (rx->read <= rx->write) 1055 return TSNEP_RING_SIZE - rx->write + rx->read - 1; 1056 else 1057 return rx->read - rx->write - 1; 1058 } 1059 1060 static void tsnep_rx_free_page_buffer(struct tsnep_rx *rx) 1061 { 1062 struct page **page; 1063 1064 /* last entry of page_buffer is always zero, because ring cannot be 1065 * filled completely 1066 */ 1067 page = rx->page_buffer; 1068 while (*page) { 1069 page_pool_put_full_page(rx->page_pool, *page, false); 1070 *page = NULL; 1071 page++; 1072 } 1073 } 1074 1075 static int tsnep_rx_alloc_page_buffer(struct tsnep_rx *rx) 1076 { 1077 int i; 1078 1079 /* alloc for all ring entries except the last one, because ring cannot 1080 * be filled completely 1081 */ 1082 for (i = 0; i < TSNEP_RING_SIZE - 1; i++) { 1083 rx->page_buffer[i] = page_pool_dev_alloc_pages(rx->page_pool); 1084 if (!rx->page_buffer[i]) { 1085 tsnep_rx_free_page_buffer(rx); 1086 1087 return -ENOMEM; 1088 } 1089 } 1090 1091 return 0; 1092 } 1093 1094 static void tsnep_rx_set_page(struct tsnep_rx *rx, struct tsnep_rx_entry *entry, 1095 struct page *page) 1096 { 1097 entry->page = page; 1098 entry->len = TSNEP_MAX_RX_BUF_SIZE; 1099 entry->dma = page_pool_get_dma_addr(entry->page); 1100 entry->desc->rx = __cpu_to_le64(entry->dma + TSNEP_RX_OFFSET); 1101 } 1102 1103 static int tsnep_rx_alloc_buffer(struct tsnep_rx *rx, int index) 1104 { 1105 struct tsnep_rx_entry *entry = &rx->entry[index]; 1106 struct page *page; 1107 1108 page = page_pool_dev_alloc_pages(rx->page_pool); 1109 if (unlikely(!page)) 1110 return -ENOMEM; 1111 tsnep_rx_set_page(rx, entry, page); 1112 1113 return 0; 1114 } 1115 1116 static void tsnep_rx_reuse_buffer(struct tsnep_rx *rx, int index) 1117 { 1118 struct tsnep_rx_entry *entry = &rx->entry[index]; 1119 struct tsnep_rx_entry *read = &rx->entry[rx->read]; 1120 1121 tsnep_rx_set_page(rx, entry, read->page); 1122 read->page = NULL; 1123 } 1124 1125 static void tsnep_rx_activate(struct tsnep_rx *rx, int index) 1126 { 1127 struct tsnep_rx_entry *entry = &rx->entry[index]; 1128 1129 /* TSNEP_MAX_RX_BUF_SIZE and TSNEP_XSK_RX_BUF_SIZE are multiple of 4 */ 1130 entry->properties = entry->len & TSNEP_DESC_LENGTH_MASK; 1131 entry->properties |= TSNEP_DESC_INTERRUPT_FLAG; 1132 if (index == rx->increment_owner_counter) { 1133 rx->owner_counter++; 1134 if (rx->owner_counter == 4) 1135 rx->owner_counter = 1; 1136 rx->increment_owner_counter--; 1137 if (rx->increment_owner_counter < 0) 1138 rx->increment_owner_counter = TSNEP_RING_SIZE - 1; 1139 } 1140 entry->properties |= 1141 (rx->owner_counter << TSNEP_DESC_OWNER_COUNTER_SHIFT) & 1142 TSNEP_DESC_OWNER_COUNTER_MASK; 1143 1144 /* descriptor properties shall be written last, because valid data is 1145 * signaled there 1146 */ 1147 dma_wmb(); 1148 1149 entry->desc->properties = __cpu_to_le32(entry->properties); 1150 } 1151 1152 static int tsnep_rx_alloc(struct tsnep_rx *rx, int count, bool reuse) 1153 { 1154 bool alloc_failed = false; 1155 int i, index; 1156 1157 for (i = 0; i < count && !alloc_failed; i++) { 1158 index = (rx->write + i) & TSNEP_RING_MASK; 1159 1160 if (unlikely(tsnep_rx_alloc_buffer(rx, index))) { 1161 rx->alloc_failed++; 1162 alloc_failed = true; 1163 1164 /* reuse only if no other allocation was successful */ 1165 if (i == 0 && reuse) 1166 tsnep_rx_reuse_buffer(rx, index); 1167 else 1168 break; 1169 } 1170 1171 tsnep_rx_activate(rx, index); 1172 } 1173 1174 if (i) 1175 rx->write = (rx->write + i) & TSNEP_RING_MASK; 1176 1177 return i; 1178 } 1179 1180 static int tsnep_rx_refill(struct tsnep_rx *rx, int count, bool reuse) 1181 { 1182 int desc_refilled; 1183 1184 desc_refilled = tsnep_rx_alloc(rx, count, reuse); 1185 if (desc_refilled) 1186 tsnep_rx_enable(rx); 1187 1188 return desc_refilled; 1189 } 1190 1191 static void tsnep_rx_set_xdp(struct tsnep_rx *rx, struct tsnep_rx_entry *entry, 1192 struct xdp_buff *xdp) 1193 { 1194 entry->xdp = xdp; 1195 entry->len = TSNEP_XSK_RX_BUF_SIZE; 1196 entry->dma = xsk_buff_xdp_get_dma(entry->xdp); 1197 entry->desc->rx = __cpu_to_le64(entry->dma); 1198 } 1199 1200 static void tsnep_rx_reuse_buffer_zc(struct tsnep_rx *rx, int index) 1201 { 1202 struct tsnep_rx_entry *entry = &rx->entry[index]; 1203 struct tsnep_rx_entry *read = &rx->entry[rx->read]; 1204 1205 tsnep_rx_set_xdp(rx, entry, read->xdp); 1206 read->xdp = NULL; 1207 } 1208 1209 static int tsnep_rx_alloc_zc(struct tsnep_rx *rx, int count, bool reuse) 1210 { 1211 u32 allocated; 1212 int i; 1213 1214 allocated = xsk_buff_alloc_batch(rx->xsk_pool, rx->xdp_batch, count); 1215 for (i = 0; i < allocated; i++) { 1216 int index = (rx->write + i) & TSNEP_RING_MASK; 1217 struct tsnep_rx_entry *entry = &rx->entry[index]; 1218 1219 tsnep_rx_set_xdp(rx, entry, rx->xdp_batch[i]); 1220 tsnep_rx_activate(rx, index); 1221 } 1222 if (i == 0) { 1223 rx->alloc_failed++; 1224 1225 if (reuse) { 1226 tsnep_rx_reuse_buffer_zc(rx, rx->write); 1227 tsnep_rx_activate(rx, rx->write); 1228 } 1229 } 1230 1231 if (i) 1232 rx->write = (rx->write + i) & TSNEP_RING_MASK; 1233 1234 return i; 1235 } 1236 1237 static void tsnep_rx_free_zc(struct tsnep_rx *rx) 1238 { 1239 int i; 1240 1241 for (i = 0; i < TSNEP_RING_SIZE; i++) { 1242 struct tsnep_rx_entry *entry = &rx->entry[i]; 1243 1244 if (entry->xdp) 1245 xsk_buff_free(entry->xdp); 1246 entry->xdp = NULL; 1247 } 1248 } 1249 1250 static int tsnep_rx_refill_zc(struct tsnep_rx *rx, int count, bool reuse) 1251 { 1252 int desc_refilled; 1253 1254 desc_refilled = tsnep_rx_alloc_zc(rx, count, reuse); 1255 if (desc_refilled) 1256 tsnep_rx_enable(rx); 1257 1258 return desc_refilled; 1259 } 1260 1261 static bool tsnep_xdp_run_prog(struct tsnep_rx *rx, struct bpf_prog *prog, 1262 struct xdp_buff *xdp, int *status, 1263 struct netdev_queue *tx_nq, struct tsnep_tx *tx) 1264 { 1265 unsigned int length; 1266 unsigned int sync; 1267 u32 act; 1268 1269 length = xdp->data_end - xdp->data_hard_start - XDP_PACKET_HEADROOM; 1270 1271 act = bpf_prog_run_xdp(prog, xdp); 1272 switch (act) { 1273 case XDP_PASS: 1274 return false; 1275 case XDP_TX: 1276 if (!tsnep_xdp_xmit_back(rx->adapter, xdp, tx_nq, tx)) 1277 goto out_failure; 1278 *status |= TSNEP_XDP_TX; 1279 return true; 1280 case XDP_REDIRECT: 1281 if (xdp_do_redirect(rx->adapter->netdev, xdp, prog) < 0) 1282 goto out_failure; 1283 *status |= TSNEP_XDP_REDIRECT; 1284 return true; 1285 default: 1286 bpf_warn_invalid_xdp_action(rx->adapter->netdev, prog, act); 1287 fallthrough; 1288 case XDP_ABORTED: 1289 out_failure: 1290 trace_xdp_exception(rx->adapter->netdev, prog, act); 1291 fallthrough; 1292 case XDP_DROP: 1293 /* Due xdp_adjust_tail: DMA sync for_device cover max len CPU 1294 * touch 1295 */ 1296 sync = xdp->data_end - xdp->data_hard_start - 1297 XDP_PACKET_HEADROOM; 1298 sync = max(sync, length); 1299 page_pool_put_page(rx->page_pool, virt_to_head_page(xdp->data), 1300 sync, true); 1301 return true; 1302 } 1303 } 1304 1305 static bool tsnep_xdp_run_prog_zc(struct tsnep_rx *rx, struct bpf_prog *prog, 1306 struct xdp_buff *xdp, int *status, 1307 struct netdev_queue *tx_nq, 1308 struct tsnep_tx *tx) 1309 { 1310 u32 act; 1311 1312 act = bpf_prog_run_xdp(prog, xdp); 1313 1314 /* XDP_REDIRECT is the main action for zero-copy */ 1315 if (likely(act == XDP_REDIRECT)) { 1316 if (xdp_do_redirect(rx->adapter->netdev, xdp, prog) < 0) 1317 goto out_failure; 1318 *status |= TSNEP_XDP_REDIRECT; 1319 return true; 1320 } 1321 1322 switch (act) { 1323 case XDP_PASS: 1324 return false; 1325 case XDP_TX: 1326 if (!tsnep_xdp_xmit_back(rx->adapter, xdp, tx_nq, tx)) 1327 goto out_failure; 1328 *status |= TSNEP_XDP_TX; 1329 return true; 1330 default: 1331 bpf_warn_invalid_xdp_action(rx->adapter->netdev, prog, act); 1332 fallthrough; 1333 case XDP_ABORTED: 1334 out_failure: 1335 trace_xdp_exception(rx->adapter->netdev, prog, act); 1336 fallthrough; 1337 case XDP_DROP: 1338 xsk_buff_free(xdp); 1339 return true; 1340 } 1341 } 1342 1343 static void tsnep_finalize_xdp(struct tsnep_adapter *adapter, int status, 1344 struct netdev_queue *tx_nq, struct tsnep_tx *tx) 1345 { 1346 if (status & TSNEP_XDP_TX) { 1347 __netif_tx_lock(tx_nq, smp_processor_id()); 1348 tsnep_xdp_xmit_flush(tx); 1349 __netif_tx_unlock(tx_nq); 1350 } 1351 1352 if (status & TSNEP_XDP_REDIRECT) 1353 xdp_do_flush(); 1354 } 1355 1356 static struct sk_buff *tsnep_build_skb(struct tsnep_rx *rx, struct page *page, 1357 int length) 1358 { 1359 struct sk_buff *skb; 1360 1361 skb = napi_build_skb(page_address(page), PAGE_SIZE); 1362 if (unlikely(!skb)) 1363 return NULL; 1364 1365 /* update pointers within the skb to store the data */ 1366 skb_reserve(skb, TSNEP_RX_OFFSET + TSNEP_RX_INLINE_METADATA_SIZE); 1367 __skb_put(skb, length - ETH_FCS_LEN); 1368 1369 if (rx->adapter->hwtstamp_config.rx_filter == HWTSTAMP_FILTER_ALL) { 1370 struct skb_shared_hwtstamps *hwtstamps = skb_hwtstamps(skb); 1371 struct tsnep_rx_inline *rx_inline = 1372 (struct tsnep_rx_inline *)(page_address(page) + 1373 TSNEP_RX_OFFSET); 1374 1375 skb_shinfo(skb)->tx_flags |= 1376 SKBTX_HW_TSTAMP_NETDEV; 1377 memset(hwtstamps, 0, sizeof(*hwtstamps)); 1378 hwtstamps->netdev_data = rx_inline; 1379 } 1380 1381 skb_record_rx_queue(skb, rx->queue_index); 1382 skb->protocol = eth_type_trans(skb, rx->adapter->netdev); 1383 1384 return skb; 1385 } 1386 1387 static void tsnep_rx_page(struct tsnep_rx *rx, struct napi_struct *napi, 1388 struct page *page, int length) 1389 { 1390 struct sk_buff *skb; 1391 1392 skb = tsnep_build_skb(rx, page, length); 1393 if (skb) { 1394 skb_mark_for_recycle(skb); 1395 1396 rx->packets++; 1397 rx->bytes += length; 1398 if (skb->pkt_type == PACKET_MULTICAST) 1399 rx->multicast++; 1400 1401 napi_gro_receive(napi, skb); 1402 } else { 1403 page_pool_recycle_direct(rx->page_pool, page); 1404 1405 rx->dropped++; 1406 } 1407 } 1408 1409 static int tsnep_rx_poll(struct tsnep_rx *rx, struct napi_struct *napi, 1410 int budget) 1411 { 1412 struct device *dmadev = rx->adapter->dmadev; 1413 enum dma_data_direction dma_dir; 1414 struct tsnep_rx_entry *entry; 1415 struct netdev_queue *tx_nq; 1416 struct bpf_prog *prog; 1417 struct xdp_buff xdp; 1418 struct tsnep_tx *tx; 1419 int desc_available; 1420 int xdp_status = 0; 1421 int done = 0; 1422 int length; 1423 1424 desc_available = tsnep_rx_desc_available(rx); 1425 dma_dir = page_pool_get_dma_dir(rx->page_pool); 1426 prog = READ_ONCE(rx->adapter->xdp_prog); 1427 if (prog) { 1428 tx_nq = netdev_get_tx_queue(rx->adapter->netdev, 1429 rx->tx_queue_index); 1430 tx = &rx->adapter->tx[rx->tx_queue_index]; 1431 1432 xdp_init_buff(&xdp, PAGE_SIZE, &rx->xdp_rxq); 1433 } 1434 1435 while (likely(done < budget) && (rx->read != rx->write)) { 1436 entry = &rx->entry[rx->read]; 1437 if ((__le32_to_cpu(entry->desc_wb->properties) & 1438 TSNEP_DESC_OWNER_COUNTER_MASK) != 1439 (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK)) 1440 break; 1441 done++; 1442 1443 if (desc_available >= TSNEP_RING_RX_REFILL) { 1444 bool reuse = desc_available >= TSNEP_RING_RX_REUSE; 1445 1446 desc_available -= tsnep_rx_refill(rx, desc_available, 1447 reuse); 1448 if (!entry->page) { 1449 /* buffer has been reused for refill to prevent 1450 * empty RX ring, thus buffer cannot be used for 1451 * RX processing 1452 */ 1453 rx->read = (rx->read + 1) & TSNEP_RING_MASK; 1454 desc_available++; 1455 1456 rx->dropped++; 1457 1458 continue; 1459 } 1460 } 1461 1462 /* descriptor properties shall be read first, because valid data 1463 * is signaled there 1464 */ 1465 dma_rmb(); 1466 1467 prefetch(page_address(entry->page) + TSNEP_RX_OFFSET); 1468 length = __le32_to_cpu(entry->desc_wb->properties) & 1469 TSNEP_DESC_LENGTH_MASK; 1470 dma_sync_single_range_for_cpu(dmadev, entry->dma, 1471 TSNEP_RX_OFFSET, length, dma_dir); 1472 1473 /* RX metadata with timestamps is in front of actual data, 1474 * subtract metadata size to get length of actual data and 1475 * consider metadata size as offset of actual data during RX 1476 * processing 1477 */ 1478 length -= TSNEP_RX_INLINE_METADATA_SIZE; 1479 1480 rx->read = (rx->read + 1) & TSNEP_RING_MASK; 1481 desc_available++; 1482 1483 if (prog) { 1484 bool consume; 1485 1486 xdp_prepare_buff(&xdp, page_address(entry->page), 1487 XDP_PACKET_HEADROOM + TSNEP_RX_INLINE_METADATA_SIZE, 1488 length - ETH_FCS_LEN, false); 1489 1490 consume = tsnep_xdp_run_prog(rx, prog, &xdp, 1491 &xdp_status, tx_nq, tx); 1492 if (consume) { 1493 rx->packets++; 1494 rx->bytes += length; 1495 1496 entry->page = NULL; 1497 1498 continue; 1499 } 1500 } 1501 1502 tsnep_rx_page(rx, napi, entry->page, length); 1503 entry->page = NULL; 1504 } 1505 1506 if (xdp_status) 1507 tsnep_finalize_xdp(rx->adapter, xdp_status, tx_nq, tx); 1508 1509 if (desc_available) 1510 tsnep_rx_refill(rx, desc_available, false); 1511 1512 return done; 1513 } 1514 1515 static int tsnep_rx_poll_zc(struct tsnep_rx *rx, struct napi_struct *napi, 1516 int budget) 1517 { 1518 struct tsnep_rx_entry *entry; 1519 struct netdev_queue *tx_nq; 1520 struct bpf_prog *prog; 1521 struct tsnep_tx *tx; 1522 int desc_available; 1523 int xdp_status = 0; 1524 struct page *page; 1525 int done = 0; 1526 int length; 1527 1528 desc_available = tsnep_rx_desc_available(rx); 1529 prog = READ_ONCE(rx->adapter->xdp_prog); 1530 if (prog) { 1531 tx_nq = netdev_get_tx_queue(rx->adapter->netdev, 1532 rx->tx_queue_index); 1533 tx = &rx->adapter->tx[rx->tx_queue_index]; 1534 } 1535 1536 while (likely(done < budget) && (rx->read != rx->write)) { 1537 entry = &rx->entry[rx->read]; 1538 if ((__le32_to_cpu(entry->desc_wb->properties) & 1539 TSNEP_DESC_OWNER_COUNTER_MASK) != 1540 (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK)) 1541 break; 1542 done++; 1543 1544 if (desc_available >= TSNEP_RING_RX_REFILL) { 1545 bool reuse = desc_available >= TSNEP_RING_RX_REUSE; 1546 1547 desc_available -= tsnep_rx_refill_zc(rx, desc_available, 1548 reuse); 1549 if (!entry->xdp) { 1550 /* buffer has been reused for refill to prevent 1551 * empty RX ring, thus buffer cannot be used for 1552 * RX processing 1553 */ 1554 rx->read = (rx->read + 1) & TSNEP_RING_MASK; 1555 desc_available++; 1556 1557 rx->dropped++; 1558 1559 continue; 1560 } 1561 } 1562 1563 /* descriptor properties shall be read first, because valid data 1564 * is signaled there 1565 */ 1566 dma_rmb(); 1567 1568 prefetch(entry->xdp->data); 1569 length = __le32_to_cpu(entry->desc_wb->properties) & 1570 TSNEP_DESC_LENGTH_MASK; 1571 xsk_buff_set_size(entry->xdp, length - ETH_FCS_LEN); 1572 xsk_buff_dma_sync_for_cpu(entry->xdp, rx->xsk_pool); 1573 1574 /* RX metadata with timestamps is in front of actual data, 1575 * subtract metadata size to get length of actual data and 1576 * consider metadata size as offset of actual data during RX 1577 * processing 1578 */ 1579 length -= TSNEP_RX_INLINE_METADATA_SIZE; 1580 1581 rx->read = (rx->read + 1) & TSNEP_RING_MASK; 1582 desc_available++; 1583 1584 if (prog) { 1585 bool consume; 1586 1587 entry->xdp->data += TSNEP_RX_INLINE_METADATA_SIZE; 1588 entry->xdp->data_meta += TSNEP_RX_INLINE_METADATA_SIZE; 1589 1590 consume = tsnep_xdp_run_prog_zc(rx, prog, entry->xdp, 1591 &xdp_status, tx_nq, tx); 1592 if (consume) { 1593 rx->packets++; 1594 rx->bytes += length; 1595 1596 entry->xdp = NULL; 1597 1598 continue; 1599 } 1600 } 1601 1602 page = page_pool_dev_alloc_pages(rx->page_pool); 1603 if (page) { 1604 memcpy(page_address(page) + TSNEP_RX_OFFSET, 1605 entry->xdp->data - TSNEP_RX_INLINE_METADATA_SIZE, 1606 length + TSNEP_RX_INLINE_METADATA_SIZE); 1607 tsnep_rx_page(rx, napi, page, length); 1608 } else { 1609 rx->dropped++; 1610 } 1611 xsk_buff_free(entry->xdp); 1612 entry->xdp = NULL; 1613 } 1614 1615 if (xdp_status) 1616 tsnep_finalize_xdp(rx->adapter, xdp_status, tx_nq, tx); 1617 1618 if (desc_available) 1619 desc_available -= tsnep_rx_refill_zc(rx, desc_available, false); 1620 1621 if (xsk_uses_need_wakeup(rx->xsk_pool)) { 1622 if (desc_available) 1623 xsk_set_rx_need_wakeup(rx->xsk_pool); 1624 else 1625 xsk_clear_rx_need_wakeup(rx->xsk_pool); 1626 1627 return done; 1628 } 1629 1630 return desc_available ? budget : done; 1631 } 1632 1633 static bool tsnep_rx_pending(struct tsnep_rx *rx) 1634 { 1635 struct tsnep_rx_entry *entry; 1636 1637 if (rx->read != rx->write) { 1638 entry = &rx->entry[rx->read]; 1639 if ((__le32_to_cpu(entry->desc_wb->properties) & 1640 TSNEP_DESC_OWNER_COUNTER_MASK) == 1641 (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK)) 1642 return true; 1643 } 1644 1645 return false; 1646 } 1647 1648 static int tsnep_rx_open(struct tsnep_rx *rx) 1649 { 1650 int desc_available; 1651 int retval; 1652 1653 retval = tsnep_rx_ring_create(rx); 1654 if (retval) 1655 return retval; 1656 1657 tsnep_rx_init(rx); 1658 1659 desc_available = tsnep_rx_desc_available(rx); 1660 if (rx->xsk_pool) 1661 retval = tsnep_rx_alloc_zc(rx, desc_available, false); 1662 else 1663 retval = tsnep_rx_alloc(rx, desc_available, false); 1664 if (retval != desc_available) { 1665 retval = -ENOMEM; 1666 1667 goto alloc_failed; 1668 } 1669 1670 /* prealloc pages to prevent allocation failures when XSK pool is 1671 * disabled at runtime 1672 */ 1673 if (rx->xsk_pool) { 1674 retval = tsnep_rx_alloc_page_buffer(rx); 1675 if (retval) 1676 goto alloc_failed; 1677 } 1678 1679 return 0; 1680 1681 alloc_failed: 1682 tsnep_rx_ring_cleanup(rx); 1683 return retval; 1684 } 1685 1686 static void tsnep_rx_close(struct tsnep_rx *rx) 1687 { 1688 if (rx->xsk_pool) 1689 tsnep_rx_free_page_buffer(rx); 1690 1691 tsnep_rx_ring_cleanup(rx); 1692 } 1693 1694 static void tsnep_rx_reopen(struct tsnep_rx *rx) 1695 { 1696 struct page **page = rx->page_buffer; 1697 int i; 1698 1699 tsnep_rx_init(rx); 1700 1701 for (i = 0; i < TSNEP_RING_SIZE; i++) { 1702 struct tsnep_rx_entry *entry = &rx->entry[i]; 1703 1704 /* defined initial values for properties are required for 1705 * correct owner counter checking 1706 */ 1707 entry->desc->properties = 0; 1708 entry->desc_wb->properties = 0; 1709 1710 /* prevent allocation failures by reusing kept pages */ 1711 if (*page) { 1712 tsnep_rx_set_page(rx, entry, *page); 1713 tsnep_rx_activate(rx, rx->write); 1714 rx->write++; 1715 1716 *page = NULL; 1717 page++; 1718 } 1719 } 1720 } 1721 1722 static void tsnep_rx_reopen_xsk(struct tsnep_rx *rx) 1723 { 1724 struct page **page = rx->page_buffer; 1725 u32 allocated; 1726 int i; 1727 1728 tsnep_rx_init(rx); 1729 1730 /* alloc all ring entries except the last one, because ring cannot be 1731 * filled completely, as many buffers as possible is enough as wakeup is 1732 * done if new buffers are available 1733 */ 1734 allocated = xsk_buff_alloc_batch(rx->xsk_pool, rx->xdp_batch, 1735 TSNEP_RING_SIZE - 1); 1736 1737 for (i = 0; i < TSNEP_RING_SIZE; i++) { 1738 struct tsnep_rx_entry *entry = &rx->entry[i]; 1739 1740 /* keep pages to prevent allocation failures when xsk is 1741 * disabled 1742 */ 1743 if (entry->page) { 1744 *page = entry->page; 1745 entry->page = NULL; 1746 1747 page++; 1748 } 1749 1750 /* defined initial values for properties are required for 1751 * correct owner counter checking 1752 */ 1753 entry->desc->properties = 0; 1754 entry->desc_wb->properties = 0; 1755 1756 if (allocated) { 1757 tsnep_rx_set_xdp(rx, entry, 1758 rx->xdp_batch[allocated - 1]); 1759 tsnep_rx_activate(rx, rx->write); 1760 rx->write++; 1761 1762 allocated--; 1763 } 1764 } 1765 1766 /* set need wakeup flag immediately if ring is not filled completely, 1767 * first polling would be too late as need wakeup signalisation would 1768 * be delayed for an indefinite time 1769 */ 1770 if (xsk_uses_need_wakeup(rx->xsk_pool)) { 1771 int desc_available = tsnep_rx_desc_available(rx); 1772 1773 if (desc_available) 1774 xsk_set_rx_need_wakeup(rx->xsk_pool); 1775 else 1776 xsk_clear_rx_need_wakeup(rx->xsk_pool); 1777 } 1778 } 1779 1780 static bool tsnep_pending(struct tsnep_queue *queue) 1781 { 1782 if (queue->tx && tsnep_tx_pending(queue->tx)) 1783 return true; 1784 1785 if (queue->rx && tsnep_rx_pending(queue->rx)) 1786 return true; 1787 1788 return false; 1789 } 1790 1791 static int tsnep_poll(struct napi_struct *napi, int budget) 1792 { 1793 struct tsnep_queue *queue = container_of(napi, struct tsnep_queue, 1794 napi); 1795 bool complete = true; 1796 int done = 0; 1797 1798 if (queue->tx) 1799 complete = tsnep_tx_poll(queue->tx, budget); 1800 1801 /* handle case where we are called by netpoll with a budget of 0 */ 1802 if (unlikely(budget <= 0)) 1803 return budget; 1804 1805 if (queue->rx) { 1806 done = queue->rx->xsk_pool ? 1807 tsnep_rx_poll_zc(queue->rx, napi, budget) : 1808 tsnep_rx_poll(queue->rx, napi, budget); 1809 if (done >= budget) 1810 complete = false; 1811 } 1812 1813 /* if all work not completed, return budget and keep polling */ 1814 if (!complete) 1815 return budget; 1816 1817 if (likely(napi_complete_done(napi, done))) { 1818 tsnep_enable_irq(queue->adapter, queue->irq_mask); 1819 1820 /* reschedule if work is already pending, prevent rotten packets 1821 * which are transmitted or received after polling but before 1822 * interrupt enable 1823 */ 1824 if (tsnep_pending(queue)) { 1825 tsnep_disable_irq(queue->adapter, queue->irq_mask); 1826 napi_schedule(napi); 1827 } 1828 } 1829 1830 return min(done, budget - 1); 1831 } 1832 1833 static int tsnep_request_irq(struct tsnep_queue *queue, bool first) 1834 { 1835 const char *name = netdev_name(queue->adapter->netdev); 1836 irq_handler_t handler; 1837 void *dev; 1838 int retval; 1839 1840 if (first) { 1841 sprintf(queue->name, "%s-mac", name); 1842 handler = tsnep_irq; 1843 dev = queue->adapter; 1844 } else { 1845 if (queue->tx && queue->rx) 1846 snprintf(queue->name, sizeof(queue->name), "%s-txrx-%d", 1847 name, queue->rx->queue_index); 1848 else if (queue->tx) 1849 snprintf(queue->name, sizeof(queue->name), "%s-tx-%d", 1850 name, queue->tx->queue_index); 1851 else 1852 snprintf(queue->name, sizeof(queue->name), "%s-rx-%d", 1853 name, queue->rx->queue_index); 1854 handler = tsnep_irq_txrx; 1855 dev = queue; 1856 } 1857 1858 retval = request_irq(queue->irq, handler, 0, queue->name, dev); 1859 if (retval) { 1860 /* if name is empty, then interrupt won't be freed */ 1861 memset(queue->name, 0, sizeof(queue->name)); 1862 } 1863 1864 return retval; 1865 } 1866 1867 static void tsnep_free_irq(struct tsnep_queue *queue, bool first) 1868 { 1869 void *dev; 1870 1871 if (!strlen(queue->name)) 1872 return; 1873 1874 if (first) 1875 dev = queue->adapter; 1876 else 1877 dev = queue; 1878 1879 free_irq(queue->irq, dev); 1880 memset(queue->name, 0, sizeof(queue->name)); 1881 } 1882 1883 static void tsnep_queue_close(struct tsnep_queue *queue, bool first) 1884 { 1885 struct tsnep_rx *rx = queue->rx; 1886 1887 tsnep_free_irq(queue, first); 1888 1889 if (rx) { 1890 if (xdp_rxq_info_is_reg(&rx->xdp_rxq)) 1891 xdp_rxq_info_unreg(&rx->xdp_rxq); 1892 if (xdp_rxq_info_is_reg(&rx->xdp_rxq_zc)) 1893 xdp_rxq_info_unreg(&rx->xdp_rxq_zc); 1894 } 1895 1896 netif_napi_del(&queue->napi); 1897 } 1898 1899 static int tsnep_queue_open(struct tsnep_adapter *adapter, 1900 struct tsnep_queue *queue, bool first) 1901 { 1902 struct tsnep_rx *rx = queue->rx; 1903 struct tsnep_tx *tx = queue->tx; 1904 int retval; 1905 1906 netif_napi_add(adapter->netdev, &queue->napi, tsnep_poll); 1907 1908 if (rx) { 1909 /* choose TX queue for XDP_TX */ 1910 if (tx) 1911 rx->tx_queue_index = tx->queue_index; 1912 else if (rx->queue_index < adapter->num_tx_queues) 1913 rx->tx_queue_index = rx->queue_index; 1914 else 1915 rx->tx_queue_index = 0; 1916 1917 /* prepare both memory models to eliminate possible registration 1918 * errors when memory model is switched between page pool and 1919 * XSK pool during runtime 1920 */ 1921 retval = xdp_rxq_info_reg(&rx->xdp_rxq, adapter->netdev, 1922 rx->queue_index, queue->napi.napi_id); 1923 if (retval) 1924 goto failed; 1925 retval = xdp_rxq_info_reg_mem_model(&rx->xdp_rxq, 1926 MEM_TYPE_PAGE_POOL, 1927 rx->page_pool); 1928 if (retval) 1929 goto failed; 1930 retval = xdp_rxq_info_reg(&rx->xdp_rxq_zc, adapter->netdev, 1931 rx->queue_index, queue->napi.napi_id); 1932 if (retval) 1933 goto failed; 1934 retval = xdp_rxq_info_reg_mem_model(&rx->xdp_rxq_zc, 1935 MEM_TYPE_XSK_BUFF_POOL, 1936 NULL); 1937 if (retval) 1938 goto failed; 1939 if (rx->xsk_pool) 1940 xsk_pool_set_rxq_info(rx->xsk_pool, &rx->xdp_rxq_zc); 1941 } 1942 1943 retval = tsnep_request_irq(queue, first); 1944 if (retval) { 1945 netif_err(adapter, drv, adapter->netdev, 1946 "can't get assigned irq %d.\n", queue->irq); 1947 goto failed; 1948 } 1949 1950 return 0; 1951 1952 failed: 1953 tsnep_queue_close(queue, first); 1954 1955 return retval; 1956 } 1957 1958 static void tsnep_queue_enable(struct tsnep_queue *queue) 1959 { 1960 napi_enable(&queue->napi); 1961 tsnep_enable_irq(queue->adapter, queue->irq_mask); 1962 1963 if (queue->tx) 1964 tsnep_tx_enable(queue->tx); 1965 1966 if (queue->rx) 1967 tsnep_rx_enable(queue->rx); 1968 } 1969 1970 static void tsnep_queue_disable(struct tsnep_queue *queue) 1971 { 1972 if (queue->tx) 1973 tsnep_tx_disable(queue->tx, &queue->napi); 1974 1975 napi_disable(&queue->napi); 1976 tsnep_disable_irq(queue->adapter, queue->irq_mask); 1977 1978 /* disable RX after NAPI polling has been disabled, because RX can be 1979 * enabled during NAPI polling 1980 */ 1981 if (queue->rx) 1982 tsnep_rx_disable(queue->rx); 1983 } 1984 1985 static int tsnep_netdev_open(struct net_device *netdev) 1986 { 1987 struct tsnep_adapter *adapter = netdev_priv(netdev); 1988 int i, retval; 1989 1990 for (i = 0; i < adapter->num_queues; i++) { 1991 if (adapter->queue[i].tx) { 1992 retval = tsnep_tx_open(adapter->queue[i].tx); 1993 if (retval) 1994 goto failed; 1995 } 1996 if (adapter->queue[i].rx) { 1997 retval = tsnep_rx_open(adapter->queue[i].rx); 1998 if (retval) 1999 goto failed; 2000 } 2001 2002 retval = tsnep_queue_open(adapter, &adapter->queue[i], i == 0); 2003 if (retval) 2004 goto failed; 2005 } 2006 2007 retval = netif_set_real_num_tx_queues(adapter->netdev, 2008 adapter->num_tx_queues); 2009 if (retval) 2010 goto failed; 2011 retval = netif_set_real_num_rx_queues(adapter->netdev, 2012 adapter->num_rx_queues); 2013 if (retval) 2014 goto failed; 2015 2016 tsnep_enable_irq(adapter, ECM_INT_LINK); 2017 retval = tsnep_phy_open(adapter); 2018 if (retval) 2019 goto phy_failed; 2020 2021 for (i = 0; i < adapter->num_queues; i++) 2022 tsnep_queue_enable(&adapter->queue[i]); 2023 2024 return 0; 2025 2026 phy_failed: 2027 tsnep_disable_irq(adapter, ECM_INT_LINK); 2028 failed: 2029 for (i = 0; i < adapter->num_queues; i++) { 2030 tsnep_queue_close(&adapter->queue[i], i == 0); 2031 2032 if (adapter->queue[i].rx) 2033 tsnep_rx_close(adapter->queue[i].rx); 2034 if (adapter->queue[i].tx) 2035 tsnep_tx_close(adapter->queue[i].tx); 2036 } 2037 return retval; 2038 } 2039 2040 static int tsnep_netdev_close(struct net_device *netdev) 2041 { 2042 struct tsnep_adapter *adapter = netdev_priv(netdev); 2043 int i; 2044 2045 tsnep_disable_irq(adapter, ECM_INT_LINK); 2046 tsnep_phy_close(adapter); 2047 2048 for (i = 0; i < adapter->num_queues; i++) { 2049 tsnep_queue_disable(&adapter->queue[i]); 2050 2051 tsnep_queue_close(&adapter->queue[i], i == 0); 2052 2053 if (adapter->queue[i].rx) 2054 tsnep_rx_close(adapter->queue[i].rx); 2055 if (adapter->queue[i].tx) 2056 tsnep_tx_close(adapter->queue[i].tx); 2057 } 2058 2059 return 0; 2060 } 2061 2062 int tsnep_enable_xsk(struct tsnep_queue *queue, struct xsk_buff_pool *pool) 2063 { 2064 bool running = netif_running(queue->adapter->netdev); 2065 u32 frame_size; 2066 2067 frame_size = xsk_pool_get_rx_frame_size(pool); 2068 if (frame_size < TSNEP_XSK_RX_BUF_SIZE) 2069 return -EOPNOTSUPP; 2070 2071 queue->rx->page_buffer = kcalloc(TSNEP_RING_SIZE, 2072 sizeof(*queue->rx->page_buffer), 2073 GFP_KERNEL); 2074 if (!queue->rx->page_buffer) 2075 return -ENOMEM; 2076 queue->rx->xdp_batch = kcalloc(TSNEP_RING_SIZE, 2077 sizeof(*queue->rx->xdp_batch), 2078 GFP_KERNEL); 2079 if (!queue->rx->xdp_batch) { 2080 kfree(queue->rx->page_buffer); 2081 queue->rx->page_buffer = NULL; 2082 2083 return -ENOMEM; 2084 } 2085 2086 xsk_pool_set_rxq_info(pool, &queue->rx->xdp_rxq_zc); 2087 2088 if (running) 2089 tsnep_queue_disable(queue); 2090 2091 queue->tx->xsk_pool = pool; 2092 queue->rx->xsk_pool = pool; 2093 2094 if (running) { 2095 tsnep_rx_reopen_xsk(queue->rx); 2096 tsnep_queue_enable(queue); 2097 } 2098 2099 return 0; 2100 } 2101 2102 void tsnep_disable_xsk(struct tsnep_queue *queue) 2103 { 2104 bool running = netif_running(queue->adapter->netdev); 2105 2106 if (running) 2107 tsnep_queue_disable(queue); 2108 2109 tsnep_rx_free_zc(queue->rx); 2110 2111 queue->rx->xsk_pool = NULL; 2112 queue->tx->xsk_pool = NULL; 2113 2114 if (running) { 2115 tsnep_rx_reopen(queue->rx); 2116 tsnep_queue_enable(queue); 2117 } 2118 2119 kfree(queue->rx->xdp_batch); 2120 queue->rx->xdp_batch = NULL; 2121 kfree(queue->rx->page_buffer); 2122 queue->rx->page_buffer = NULL; 2123 } 2124 2125 static netdev_tx_t tsnep_netdev_xmit_frame(struct sk_buff *skb, 2126 struct net_device *netdev) 2127 { 2128 struct tsnep_adapter *adapter = netdev_priv(netdev); 2129 u16 queue_mapping = skb_get_queue_mapping(skb); 2130 2131 if (queue_mapping >= adapter->num_tx_queues) 2132 queue_mapping = 0; 2133 2134 return tsnep_xmit_frame_ring(skb, &adapter->tx[queue_mapping]); 2135 } 2136 2137 static int tsnep_netdev_ioctl(struct net_device *netdev, struct ifreq *ifr, 2138 int cmd) 2139 { 2140 if (!netif_running(netdev)) 2141 return -EINVAL; 2142 if (cmd == SIOCSHWTSTAMP || cmd == SIOCGHWTSTAMP) 2143 return tsnep_ptp_ioctl(netdev, ifr, cmd); 2144 return phy_mii_ioctl(netdev->phydev, ifr, cmd); 2145 } 2146 2147 static void tsnep_netdev_set_multicast(struct net_device *netdev) 2148 { 2149 struct tsnep_adapter *adapter = netdev_priv(netdev); 2150 2151 u16 rx_filter = 0; 2152 2153 /* configured MAC address and broadcasts are never filtered */ 2154 if (netdev->flags & IFF_PROMISC) { 2155 rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_MULTICASTS; 2156 rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_UNICASTS; 2157 } else if (!netdev_mc_empty(netdev) || (netdev->flags & IFF_ALLMULTI)) { 2158 rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_MULTICASTS; 2159 } 2160 iowrite16(rx_filter, adapter->addr + TSNEP_RX_FILTER); 2161 } 2162 2163 static void tsnep_netdev_get_stats64(struct net_device *netdev, 2164 struct rtnl_link_stats64 *stats) 2165 { 2166 struct tsnep_adapter *adapter = netdev_priv(netdev); 2167 u32 reg; 2168 u32 val; 2169 int i; 2170 2171 for (i = 0; i < adapter->num_tx_queues; i++) { 2172 stats->tx_packets += adapter->tx[i].packets; 2173 stats->tx_bytes += adapter->tx[i].bytes; 2174 stats->tx_dropped += adapter->tx[i].dropped; 2175 } 2176 for (i = 0; i < adapter->num_rx_queues; i++) { 2177 stats->rx_packets += adapter->rx[i].packets; 2178 stats->rx_bytes += adapter->rx[i].bytes; 2179 stats->rx_dropped += adapter->rx[i].dropped; 2180 stats->multicast += adapter->rx[i].multicast; 2181 2182 reg = ioread32(adapter->addr + TSNEP_QUEUE(i) + 2183 TSNEP_RX_STATISTIC); 2184 val = (reg & TSNEP_RX_STATISTIC_NO_DESC_MASK) >> 2185 TSNEP_RX_STATISTIC_NO_DESC_SHIFT; 2186 stats->rx_dropped += val; 2187 val = (reg & TSNEP_RX_STATISTIC_BUFFER_TOO_SMALL_MASK) >> 2188 TSNEP_RX_STATISTIC_BUFFER_TOO_SMALL_SHIFT; 2189 stats->rx_dropped += val; 2190 val = (reg & TSNEP_RX_STATISTIC_FIFO_OVERFLOW_MASK) >> 2191 TSNEP_RX_STATISTIC_FIFO_OVERFLOW_SHIFT; 2192 stats->rx_errors += val; 2193 stats->rx_fifo_errors += val; 2194 val = (reg & TSNEP_RX_STATISTIC_INVALID_FRAME_MASK) >> 2195 TSNEP_RX_STATISTIC_INVALID_FRAME_SHIFT; 2196 stats->rx_errors += val; 2197 stats->rx_frame_errors += val; 2198 } 2199 2200 reg = ioread32(adapter->addr + ECM_STAT); 2201 val = (reg & ECM_STAT_RX_ERR_MASK) >> ECM_STAT_RX_ERR_SHIFT; 2202 stats->rx_errors += val; 2203 val = (reg & ECM_STAT_INV_FRM_MASK) >> ECM_STAT_INV_FRM_SHIFT; 2204 stats->rx_errors += val; 2205 stats->rx_crc_errors += val; 2206 val = (reg & ECM_STAT_FWD_RX_ERR_MASK) >> ECM_STAT_FWD_RX_ERR_SHIFT; 2207 stats->rx_errors += val; 2208 } 2209 2210 static void tsnep_mac_set_address(struct tsnep_adapter *adapter, u8 *addr) 2211 { 2212 iowrite32(*(u32 *)addr, adapter->addr + TSNEP_MAC_ADDRESS_LOW); 2213 iowrite16(*(u16 *)(addr + sizeof(u32)), 2214 adapter->addr + TSNEP_MAC_ADDRESS_HIGH); 2215 2216 ether_addr_copy(adapter->mac_address, addr); 2217 netif_info(adapter, drv, adapter->netdev, "MAC address set to %pM\n", 2218 addr); 2219 } 2220 2221 static int tsnep_netdev_set_mac_address(struct net_device *netdev, void *addr) 2222 { 2223 struct tsnep_adapter *adapter = netdev_priv(netdev); 2224 struct sockaddr *sock_addr = addr; 2225 int retval; 2226 2227 retval = eth_prepare_mac_addr_change(netdev, sock_addr); 2228 if (retval) 2229 return retval; 2230 eth_hw_addr_set(netdev, sock_addr->sa_data); 2231 tsnep_mac_set_address(adapter, sock_addr->sa_data); 2232 2233 return 0; 2234 } 2235 2236 static int tsnep_netdev_set_features(struct net_device *netdev, 2237 netdev_features_t features) 2238 { 2239 struct tsnep_adapter *adapter = netdev_priv(netdev); 2240 netdev_features_t changed = netdev->features ^ features; 2241 bool enable; 2242 int retval = 0; 2243 2244 if (changed & NETIF_F_LOOPBACK) { 2245 enable = !!(features & NETIF_F_LOOPBACK); 2246 retval = tsnep_phy_loopback(adapter, enable); 2247 } 2248 2249 return retval; 2250 } 2251 2252 static ktime_t tsnep_netdev_get_tstamp(struct net_device *netdev, 2253 const struct skb_shared_hwtstamps *hwtstamps, 2254 bool cycles) 2255 { 2256 struct tsnep_rx_inline *rx_inline = hwtstamps->netdev_data; 2257 u64 timestamp; 2258 2259 if (cycles) 2260 timestamp = __le64_to_cpu(rx_inline->counter); 2261 else 2262 timestamp = __le64_to_cpu(rx_inline->timestamp); 2263 2264 return ns_to_ktime(timestamp); 2265 } 2266 2267 static int tsnep_netdev_bpf(struct net_device *dev, struct netdev_bpf *bpf) 2268 { 2269 struct tsnep_adapter *adapter = netdev_priv(dev); 2270 2271 switch (bpf->command) { 2272 case XDP_SETUP_PROG: 2273 return tsnep_xdp_setup_prog(adapter, bpf->prog, bpf->extack); 2274 case XDP_SETUP_XSK_POOL: 2275 return tsnep_xdp_setup_pool(adapter, bpf->xsk.pool, 2276 bpf->xsk.queue_id); 2277 default: 2278 return -EOPNOTSUPP; 2279 } 2280 } 2281 2282 static struct tsnep_tx *tsnep_xdp_get_tx(struct tsnep_adapter *adapter, u32 cpu) 2283 { 2284 if (cpu >= TSNEP_MAX_QUEUES) 2285 cpu &= TSNEP_MAX_QUEUES - 1; 2286 2287 while (cpu >= adapter->num_tx_queues) 2288 cpu -= adapter->num_tx_queues; 2289 2290 return &adapter->tx[cpu]; 2291 } 2292 2293 static int tsnep_netdev_xdp_xmit(struct net_device *dev, int n, 2294 struct xdp_frame **xdp, u32 flags) 2295 { 2296 struct tsnep_adapter *adapter = netdev_priv(dev); 2297 u32 cpu = smp_processor_id(); 2298 struct netdev_queue *nq; 2299 struct tsnep_tx *tx; 2300 int nxmit; 2301 bool xmit; 2302 2303 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 2304 return -EINVAL; 2305 2306 tx = tsnep_xdp_get_tx(adapter, cpu); 2307 nq = netdev_get_tx_queue(adapter->netdev, tx->queue_index); 2308 2309 __netif_tx_lock(nq, cpu); 2310 2311 for (nxmit = 0; nxmit < n; nxmit++) { 2312 xmit = tsnep_xdp_xmit_frame_ring(xdp[nxmit], tx, 2313 TSNEP_TX_TYPE_XDP_NDO); 2314 if (!xmit) 2315 break; 2316 2317 /* avoid transmit queue timeout since we share it with the slow 2318 * path 2319 */ 2320 txq_trans_cond_update(nq); 2321 } 2322 2323 if (flags & XDP_XMIT_FLUSH) 2324 tsnep_xdp_xmit_flush(tx); 2325 2326 __netif_tx_unlock(nq); 2327 2328 return nxmit; 2329 } 2330 2331 static int tsnep_netdev_xsk_wakeup(struct net_device *dev, u32 queue_id, 2332 u32 flags) 2333 { 2334 struct tsnep_adapter *adapter = netdev_priv(dev); 2335 struct tsnep_queue *queue; 2336 2337 if (queue_id >= adapter->num_rx_queues || 2338 queue_id >= adapter->num_tx_queues) 2339 return -EINVAL; 2340 2341 queue = &adapter->queue[queue_id]; 2342 2343 if (!napi_if_scheduled_mark_missed(&queue->napi)) 2344 napi_schedule(&queue->napi); 2345 2346 return 0; 2347 } 2348 2349 static const struct net_device_ops tsnep_netdev_ops = { 2350 .ndo_open = tsnep_netdev_open, 2351 .ndo_stop = tsnep_netdev_close, 2352 .ndo_start_xmit = tsnep_netdev_xmit_frame, 2353 .ndo_eth_ioctl = tsnep_netdev_ioctl, 2354 .ndo_set_rx_mode = tsnep_netdev_set_multicast, 2355 .ndo_get_stats64 = tsnep_netdev_get_stats64, 2356 .ndo_set_mac_address = tsnep_netdev_set_mac_address, 2357 .ndo_set_features = tsnep_netdev_set_features, 2358 .ndo_get_tstamp = tsnep_netdev_get_tstamp, 2359 .ndo_setup_tc = tsnep_tc_setup, 2360 .ndo_bpf = tsnep_netdev_bpf, 2361 .ndo_xdp_xmit = tsnep_netdev_xdp_xmit, 2362 .ndo_xsk_wakeup = tsnep_netdev_xsk_wakeup, 2363 }; 2364 2365 static int tsnep_mac_init(struct tsnep_adapter *adapter) 2366 { 2367 int retval; 2368 2369 /* initialize RX filtering, at least configured MAC address and 2370 * broadcast are not filtered 2371 */ 2372 iowrite16(0, adapter->addr + TSNEP_RX_FILTER); 2373 2374 /* try to get MAC address in the following order: 2375 * - device tree 2376 * - valid MAC address already set 2377 * - MAC address register if valid 2378 * - random MAC address 2379 */ 2380 retval = of_get_mac_address(adapter->pdev->dev.of_node, 2381 adapter->mac_address); 2382 if (retval == -EPROBE_DEFER) 2383 return retval; 2384 if (retval && !is_valid_ether_addr(adapter->mac_address)) { 2385 *(u32 *)adapter->mac_address = 2386 ioread32(adapter->addr + TSNEP_MAC_ADDRESS_LOW); 2387 *(u16 *)(adapter->mac_address + sizeof(u32)) = 2388 ioread16(adapter->addr + TSNEP_MAC_ADDRESS_HIGH); 2389 if (!is_valid_ether_addr(adapter->mac_address)) 2390 eth_random_addr(adapter->mac_address); 2391 } 2392 2393 tsnep_mac_set_address(adapter, adapter->mac_address); 2394 eth_hw_addr_set(adapter->netdev, adapter->mac_address); 2395 2396 return 0; 2397 } 2398 2399 static int tsnep_mdio_init(struct tsnep_adapter *adapter) 2400 { 2401 struct device_node *np = adapter->pdev->dev.of_node; 2402 int retval; 2403 2404 if (np) { 2405 np = of_get_child_by_name(np, "mdio"); 2406 if (!np) 2407 return 0; 2408 2409 adapter->suppress_preamble = 2410 of_property_read_bool(np, "suppress-preamble"); 2411 } 2412 2413 adapter->mdiobus = devm_mdiobus_alloc(&adapter->pdev->dev); 2414 if (!adapter->mdiobus) { 2415 retval = -ENOMEM; 2416 2417 goto out; 2418 } 2419 2420 adapter->mdiobus->priv = (void *)adapter; 2421 adapter->mdiobus->parent = &adapter->pdev->dev; 2422 adapter->mdiobus->read = tsnep_mdiobus_read; 2423 adapter->mdiobus->write = tsnep_mdiobus_write; 2424 adapter->mdiobus->name = TSNEP "-mdiobus"; 2425 snprintf(adapter->mdiobus->id, MII_BUS_ID_SIZE, "%s", 2426 adapter->pdev->name); 2427 2428 /* do not scan broadcast address */ 2429 adapter->mdiobus->phy_mask = 0x0000001; 2430 2431 retval = of_mdiobus_register(adapter->mdiobus, np); 2432 2433 out: 2434 of_node_put(np); 2435 2436 return retval; 2437 } 2438 2439 static int tsnep_phy_init(struct tsnep_adapter *adapter) 2440 { 2441 struct device_node *phy_node; 2442 int retval; 2443 2444 retval = of_get_phy_mode(adapter->pdev->dev.of_node, 2445 &adapter->phy_mode); 2446 if (retval) 2447 adapter->phy_mode = PHY_INTERFACE_MODE_GMII; 2448 2449 phy_node = of_parse_phandle(adapter->pdev->dev.of_node, "phy-handle", 2450 0); 2451 adapter->phydev = of_phy_find_device(phy_node); 2452 of_node_put(phy_node); 2453 if (!adapter->phydev && adapter->mdiobus) 2454 adapter->phydev = phy_find_first(adapter->mdiobus); 2455 if (!adapter->phydev) 2456 return -EIO; 2457 2458 return 0; 2459 } 2460 2461 static int tsnep_queue_init(struct tsnep_adapter *adapter, int queue_count) 2462 { 2463 u32 irq_mask = ECM_INT_TX_0 | ECM_INT_RX_0; 2464 char name[8]; 2465 int i; 2466 int retval; 2467 2468 /* one TX/RX queue pair for netdev is mandatory */ 2469 if (platform_irq_count(adapter->pdev) == 1) 2470 retval = platform_get_irq(adapter->pdev, 0); 2471 else 2472 retval = platform_get_irq_byname(adapter->pdev, "mac"); 2473 if (retval < 0) 2474 return retval; 2475 adapter->num_tx_queues = 1; 2476 adapter->num_rx_queues = 1; 2477 adapter->num_queues = 1; 2478 adapter->queue[0].adapter = adapter; 2479 adapter->queue[0].irq = retval; 2480 adapter->queue[0].tx = &adapter->tx[0]; 2481 adapter->queue[0].tx->adapter = adapter; 2482 adapter->queue[0].tx->addr = adapter->addr + TSNEP_QUEUE(0); 2483 adapter->queue[0].tx->queue_index = 0; 2484 adapter->queue[0].rx = &adapter->rx[0]; 2485 adapter->queue[0].rx->adapter = adapter; 2486 adapter->queue[0].rx->addr = adapter->addr + TSNEP_QUEUE(0); 2487 adapter->queue[0].rx->queue_index = 0; 2488 adapter->queue[0].irq_mask = irq_mask; 2489 adapter->queue[0].irq_delay_addr = adapter->addr + ECM_INT_DELAY; 2490 retval = tsnep_set_irq_coalesce(&adapter->queue[0], 2491 TSNEP_COALESCE_USECS_DEFAULT); 2492 if (retval < 0) 2493 return retval; 2494 2495 adapter->netdev->irq = adapter->queue[0].irq; 2496 2497 /* add additional TX/RX queue pairs only if dedicated interrupt is 2498 * available 2499 */ 2500 for (i = 1; i < queue_count; i++) { 2501 sprintf(name, "txrx-%d", i); 2502 retval = platform_get_irq_byname_optional(adapter->pdev, name); 2503 if (retval < 0) 2504 break; 2505 2506 adapter->num_tx_queues++; 2507 adapter->num_rx_queues++; 2508 adapter->num_queues++; 2509 adapter->queue[i].adapter = adapter; 2510 adapter->queue[i].irq = retval; 2511 adapter->queue[i].tx = &adapter->tx[i]; 2512 adapter->queue[i].tx->adapter = adapter; 2513 adapter->queue[i].tx->addr = adapter->addr + TSNEP_QUEUE(i); 2514 adapter->queue[i].tx->queue_index = i; 2515 adapter->queue[i].rx = &adapter->rx[i]; 2516 adapter->queue[i].rx->adapter = adapter; 2517 adapter->queue[i].rx->addr = adapter->addr + TSNEP_QUEUE(i); 2518 adapter->queue[i].rx->queue_index = i; 2519 adapter->queue[i].irq_mask = 2520 irq_mask << (ECM_INT_TXRX_SHIFT * i); 2521 adapter->queue[i].irq_delay_addr = 2522 adapter->addr + ECM_INT_DELAY + ECM_INT_DELAY_OFFSET * i; 2523 retval = tsnep_set_irq_coalesce(&adapter->queue[i], 2524 TSNEP_COALESCE_USECS_DEFAULT); 2525 if (retval < 0) 2526 return retval; 2527 } 2528 2529 return 0; 2530 } 2531 2532 static int tsnep_probe(struct platform_device *pdev) 2533 { 2534 struct tsnep_adapter *adapter; 2535 struct net_device *netdev; 2536 struct resource *io; 2537 u32 type; 2538 int revision; 2539 int version; 2540 int queue_count; 2541 int retval; 2542 2543 netdev = devm_alloc_etherdev_mqs(&pdev->dev, 2544 sizeof(struct tsnep_adapter), 2545 TSNEP_MAX_QUEUES, TSNEP_MAX_QUEUES); 2546 if (!netdev) 2547 return -ENODEV; 2548 SET_NETDEV_DEV(netdev, &pdev->dev); 2549 adapter = netdev_priv(netdev); 2550 platform_set_drvdata(pdev, adapter); 2551 adapter->pdev = pdev; 2552 adapter->dmadev = &pdev->dev; 2553 adapter->netdev = netdev; 2554 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE | 2555 NETIF_MSG_LINK | NETIF_MSG_IFUP | 2556 NETIF_MSG_IFDOWN | NETIF_MSG_TX_QUEUED; 2557 2558 netdev->min_mtu = ETH_MIN_MTU; 2559 netdev->max_mtu = TSNEP_MAX_FRAME_SIZE; 2560 2561 mutex_init(&adapter->gate_control_lock); 2562 mutex_init(&adapter->rxnfc_lock); 2563 INIT_LIST_HEAD(&adapter->rxnfc_rules); 2564 2565 io = platform_get_resource(pdev, IORESOURCE_MEM, 0); 2566 adapter->addr = devm_ioremap_resource(&pdev->dev, io); 2567 if (IS_ERR(adapter->addr)) 2568 return PTR_ERR(adapter->addr); 2569 netdev->mem_start = io->start; 2570 netdev->mem_end = io->end; 2571 2572 type = ioread32(adapter->addr + ECM_TYPE); 2573 revision = (type & ECM_REVISION_MASK) >> ECM_REVISION_SHIFT; 2574 version = (type & ECM_VERSION_MASK) >> ECM_VERSION_SHIFT; 2575 queue_count = (type & ECM_QUEUE_COUNT_MASK) >> ECM_QUEUE_COUNT_SHIFT; 2576 adapter->gate_control = type & ECM_GATE_CONTROL; 2577 adapter->rxnfc_max = TSNEP_RX_ASSIGN_ETHER_TYPE_COUNT; 2578 2579 tsnep_disable_irq(adapter, ECM_INT_ALL); 2580 2581 retval = tsnep_queue_init(adapter, queue_count); 2582 if (retval) 2583 return retval; 2584 2585 retval = dma_set_mask_and_coherent(&adapter->pdev->dev, 2586 DMA_BIT_MASK(64)); 2587 if (retval) { 2588 dev_err(&adapter->pdev->dev, "no usable DMA configuration.\n"); 2589 return retval; 2590 } 2591 2592 retval = tsnep_mac_init(adapter); 2593 if (retval) 2594 return retval; 2595 2596 retval = tsnep_mdio_init(adapter); 2597 if (retval) 2598 goto mdio_init_failed; 2599 2600 retval = tsnep_phy_init(adapter); 2601 if (retval) 2602 goto phy_init_failed; 2603 2604 retval = tsnep_ptp_init(adapter); 2605 if (retval) 2606 goto ptp_init_failed; 2607 2608 retval = tsnep_tc_init(adapter); 2609 if (retval) 2610 goto tc_init_failed; 2611 2612 retval = tsnep_rxnfc_init(adapter); 2613 if (retval) 2614 goto rxnfc_init_failed; 2615 2616 netdev->netdev_ops = &tsnep_netdev_ops; 2617 netdev->ethtool_ops = &tsnep_ethtool_ops; 2618 netdev->features = NETIF_F_SG; 2619 netdev->hw_features = netdev->features | NETIF_F_LOOPBACK; 2620 2621 netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | 2622 NETDEV_XDP_ACT_NDO_XMIT | 2623 NETDEV_XDP_ACT_NDO_XMIT_SG | 2624 NETDEV_XDP_ACT_XSK_ZEROCOPY; 2625 2626 /* carrier off reporting is important to ethtool even BEFORE open */ 2627 netif_carrier_off(netdev); 2628 2629 retval = register_netdev(netdev); 2630 if (retval) 2631 goto register_failed; 2632 2633 dev_info(&adapter->pdev->dev, "device version %d.%02d\n", version, 2634 revision); 2635 if (adapter->gate_control) 2636 dev_info(&adapter->pdev->dev, "gate control detected\n"); 2637 2638 return 0; 2639 2640 register_failed: 2641 tsnep_rxnfc_cleanup(adapter); 2642 rxnfc_init_failed: 2643 tsnep_tc_cleanup(adapter); 2644 tc_init_failed: 2645 tsnep_ptp_cleanup(adapter); 2646 ptp_init_failed: 2647 phy_init_failed: 2648 if (adapter->mdiobus) 2649 mdiobus_unregister(adapter->mdiobus); 2650 mdio_init_failed: 2651 return retval; 2652 } 2653 2654 static void tsnep_remove(struct platform_device *pdev) 2655 { 2656 struct tsnep_adapter *adapter = platform_get_drvdata(pdev); 2657 2658 unregister_netdev(adapter->netdev); 2659 2660 tsnep_rxnfc_cleanup(adapter); 2661 2662 tsnep_tc_cleanup(adapter); 2663 2664 tsnep_ptp_cleanup(adapter); 2665 2666 if (adapter->mdiobus) 2667 mdiobus_unregister(adapter->mdiobus); 2668 2669 tsnep_disable_irq(adapter, ECM_INT_ALL); 2670 } 2671 2672 static const struct of_device_id tsnep_of_match[] = { 2673 { .compatible = "engleder,tsnep", }, 2674 { }, 2675 }; 2676 MODULE_DEVICE_TABLE(of, tsnep_of_match); 2677 2678 static struct platform_driver tsnep_driver = { 2679 .driver = { 2680 .name = TSNEP, 2681 .of_match_table = tsnep_of_match, 2682 }, 2683 .probe = tsnep_probe, 2684 .remove_new = tsnep_remove, 2685 }; 2686 module_platform_driver(tsnep_driver); 2687 2688 MODULE_AUTHOR("Gerhard Engleder <gerhard@engleder-embedded.com>"); 2689 MODULE_DESCRIPTION("TSN endpoint Ethernet MAC driver"); 2690 MODULE_LICENSE("GPL"); 2691