1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (C) 2021 Gerhard Engleder <gerhard@engleder-embedded.com> */ 3 4 /* TSN endpoint Ethernet MAC driver 5 * 6 * The TSN endpoint Ethernet MAC is a FPGA based network device for real-time 7 * communication. It is designed for endpoints within TSN (Time Sensitive 8 * Networking) networks; e.g., for PLCs in the industrial automation case. 9 * 10 * It supports multiple TX/RX queue pairs. The first TX/RX queue pair is used 11 * by the driver. 12 * 13 * More information can be found here: 14 * - www.embedded-experts.at/tsn 15 * - www.engleder-embedded.com 16 */ 17 18 #include "tsnep.h" 19 #include "tsnep_hw.h" 20 21 #include <linux/module.h> 22 #include <linux/of.h> 23 #include <linux/of_net.h> 24 #include <linux/of_mdio.h> 25 #include <linux/interrupt.h> 26 #include <linux/etherdevice.h> 27 #include <linux/phy.h> 28 #include <linux/iopoll.h> 29 #include <linux/bpf.h> 30 #include <linux/bpf_trace.h> 31 #include <net/page_pool/helpers.h> 32 #include <net/xdp_sock_drv.h> 33 34 #define TSNEP_RX_OFFSET (max(NET_SKB_PAD, XDP_PACKET_HEADROOM) + NET_IP_ALIGN) 35 #define TSNEP_HEADROOM ALIGN(TSNEP_RX_OFFSET, 4) 36 #define TSNEP_MAX_RX_BUF_SIZE (PAGE_SIZE - TSNEP_HEADROOM - \ 37 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) 38 /* XSK buffer shall store at least Q-in-Q frame */ 39 #define TSNEP_XSK_RX_BUF_SIZE (ALIGN(TSNEP_RX_INLINE_METADATA_SIZE + \ 40 ETH_FRAME_LEN + ETH_FCS_LEN + \ 41 VLAN_HLEN * 2, 4)) 42 43 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 44 #define DMA_ADDR_HIGH(dma_addr) ((u32)(((dma_addr) >> 32) & 0xFFFFFFFF)) 45 #else 46 #define DMA_ADDR_HIGH(dma_addr) ((u32)(0)) 47 #endif 48 #define DMA_ADDR_LOW(dma_addr) ((u32)((dma_addr) & 0xFFFFFFFF)) 49 50 #define TSNEP_COALESCE_USECS_DEFAULT 64 51 #define TSNEP_COALESCE_USECS_MAX ((ECM_INT_DELAY_MASK >> ECM_INT_DELAY_SHIFT) * \ 52 ECM_INT_DELAY_BASE_US + ECM_INT_DELAY_BASE_US - 1) 53 54 /* mapping type */ 55 #define TSNEP_TX_TYPE_MAP BIT(0) 56 #define TSNEP_TX_TYPE_MAP_PAGE BIT(1) 57 #define TSNEP_TX_TYPE_INLINE BIT(2) 58 /* buffer type */ 59 #define TSNEP_TX_TYPE_SKB BIT(8) 60 #define TSNEP_TX_TYPE_SKB_MAP (TSNEP_TX_TYPE_SKB | TSNEP_TX_TYPE_MAP) 61 #define TSNEP_TX_TYPE_SKB_INLINE (TSNEP_TX_TYPE_SKB | TSNEP_TX_TYPE_INLINE) 62 #define TSNEP_TX_TYPE_SKB_FRAG BIT(9) 63 #define TSNEP_TX_TYPE_SKB_FRAG_MAP_PAGE (TSNEP_TX_TYPE_SKB_FRAG | TSNEP_TX_TYPE_MAP_PAGE) 64 #define TSNEP_TX_TYPE_SKB_FRAG_INLINE (TSNEP_TX_TYPE_SKB_FRAG | TSNEP_TX_TYPE_INLINE) 65 #define TSNEP_TX_TYPE_XDP_TX BIT(10) 66 #define TSNEP_TX_TYPE_XDP_NDO BIT(11) 67 #define TSNEP_TX_TYPE_XDP_NDO_MAP_PAGE (TSNEP_TX_TYPE_XDP_NDO | TSNEP_TX_TYPE_MAP_PAGE) 68 #define TSNEP_TX_TYPE_XDP (TSNEP_TX_TYPE_XDP_TX | TSNEP_TX_TYPE_XDP_NDO) 69 #define TSNEP_TX_TYPE_XSK BIT(12) 70 71 #define TSNEP_XDP_TX BIT(0) 72 #define TSNEP_XDP_REDIRECT BIT(1) 73 74 static void tsnep_enable_irq(struct tsnep_adapter *adapter, u32 mask) 75 { 76 iowrite32(mask, adapter->addr + ECM_INT_ENABLE); 77 } 78 79 static void tsnep_disable_irq(struct tsnep_adapter *adapter, u32 mask) 80 { 81 mask |= ECM_INT_DISABLE; 82 iowrite32(mask, adapter->addr + ECM_INT_ENABLE); 83 } 84 85 static irqreturn_t tsnep_irq(int irq, void *arg) 86 { 87 struct tsnep_adapter *adapter = arg; 88 u32 active = ioread32(adapter->addr + ECM_INT_ACTIVE); 89 90 /* acknowledge interrupt */ 91 if (active != 0) 92 iowrite32(active, adapter->addr + ECM_INT_ACKNOWLEDGE); 93 94 /* handle link interrupt */ 95 if ((active & ECM_INT_LINK) != 0) 96 phy_mac_interrupt(adapter->netdev->phydev); 97 98 /* handle TX/RX queue 0 interrupt */ 99 if ((active & adapter->queue[0].irq_mask) != 0) { 100 if (napi_schedule_prep(&adapter->queue[0].napi)) { 101 tsnep_disable_irq(adapter, adapter->queue[0].irq_mask); 102 /* schedule after masking to avoid races */ 103 __napi_schedule(&adapter->queue[0].napi); 104 } 105 } 106 107 return IRQ_HANDLED; 108 } 109 110 static irqreturn_t tsnep_irq_txrx(int irq, void *arg) 111 { 112 struct tsnep_queue *queue = arg; 113 114 /* handle TX/RX queue interrupt */ 115 if (napi_schedule_prep(&queue->napi)) { 116 tsnep_disable_irq(queue->adapter, queue->irq_mask); 117 /* schedule after masking to avoid races */ 118 __napi_schedule(&queue->napi); 119 } 120 121 return IRQ_HANDLED; 122 } 123 124 int tsnep_set_irq_coalesce(struct tsnep_queue *queue, u32 usecs) 125 { 126 if (usecs > TSNEP_COALESCE_USECS_MAX) 127 return -ERANGE; 128 129 usecs /= ECM_INT_DELAY_BASE_US; 130 usecs <<= ECM_INT_DELAY_SHIFT; 131 usecs &= ECM_INT_DELAY_MASK; 132 133 queue->irq_delay &= ~ECM_INT_DELAY_MASK; 134 queue->irq_delay |= usecs; 135 iowrite8(queue->irq_delay, queue->irq_delay_addr); 136 137 return 0; 138 } 139 140 u32 tsnep_get_irq_coalesce(struct tsnep_queue *queue) 141 { 142 u32 usecs; 143 144 usecs = (queue->irq_delay & ECM_INT_DELAY_MASK); 145 usecs >>= ECM_INT_DELAY_SHIFT; 146 usecs *= ECM_INT_DELAY_BASE_US; 147 148 return usecs; 149 } 150 151 static int tsnep_mdiobus_read(struct mii_bus *bus, int addr, int regnum) 152 { 153 struct tsnep_adapter *adapter = bus->priv; 154 u32 md; 155 int retval; 156 157 md = ECM_MD_READ; 158 if (!adapter->suppress_preamble) 159 md |= ECM_MD_PREAMBLE; 160 md |= (regnum << ECM_MD_ADDR_SHIFT) & ECM_MD_ADDR_MASK; 161 md |= (addr << ECM_MD_PHY_ADDR_SHIFT) & ECM_MD_PHY_ADDR_MASK; 162 iowrite32(md, adapter->addr + ECM_MD_CONTROL); 163 retval = readl_poll_timeout_atomic(adapter->addr + ECM_MD_STATUS, md, 164 !(md & ECM_MD_BUSY), 16, 1000); 165 if (retval != 0) 166 return retval; 167 168 return (md & ECM_MD_DATA_MASK) >> ECM_MD_DATA_SHIFT; 169 } 170 171 static int tsnep_mdiobus_write(struct mii_bus *bus, int addr, int regnum, 172 u16 val) 173 { 174 struct tsnep_adapter *adapter = bus->priv; 175 u32 md; 176 int retval; 177 178 md = ECM_MD_WRITE; 179 if (!adapter->suppress_preamble) 180 md |= ECM_MD_PREAMBLE; 181 md |= (regnum << ECM_MD_ADDR_SHIFT) & ECM_MD_ADDR_MASK; 182 md |= (addr << ECM_MD_PHY_ADDR_SHIFT) & ECM_MD_PHY_ADDR_MASK; 183 md |= ((u32)val << ECM_MD_DATA_SHIFT) & ECM_MD_DATA_MASK; 184 iowrite32(md, adapter->addr + ECM_MD_CONTROL); 185 retval = readl_poll_timeout_atomic(adapter->addr + ECM_MD_STATUS, md, 186 !(md & ECM_MD_BUSY), 16, 1000); 187 if (retval != 0) 188 return retval; 189 190 return 0; 191 } 192 193 static void tsnep_set_link_mode(struct tsnep_adapter *adapter) 194 { 195 u32 mode; 196 197 switch (adapter->phydev->speed) { 198 case SPEED_100: 199 mode = ECM_LINK_MODE_100; 200 break; 201 case SPEED_1000: 202 mode = ECM_LINK_MODE_1000; 203 break; 204 default: 205 mode = ECM_LINK_MODE_OFF; 206 break; 207 } 208 iowrite32(mode, adapter->addr + ECM_STATUS); 209 } 210 211 static void tsnep_phy_link_status_change(struct net_device *netdev) 212 { 213 struct tsnep_adapter *adapter = netdev_priv(netdev); 214 struct phy_device *phydev = netdev->phydev; 215 216 if (phydev->link) 217 tsnep_set_link_mode(adapter); 218 219 phy_print_status(netdev->phydev); 220 } 221 222 static int tsnep_phy_loopback(struct tsnep_adapter *adapter, bool enable) 223 { 224 int retval; 225 226 retval = phy_loopback(adapter->phydev, enable); 227 228 /* PHY link state change is not signaled if loopback is enabled, it 229 * would delay a working loopback anyway, let's ensure that loopback 230 * is working immediately by setting link mode directly 231 */ 232 if (!retval && enable) { 233 netif_carrier_on(adapter->netdev); 234 tsnep_set_link_mode(adapter); 235 } 236 237 return retval; 238 } 239 240 static int tsnep_phy_open(struct tsnep_adapter *adapter) 241 { 242 struct phy_device *phydev; 243 struct ethtool_keee ethtool_keee; 244 int retval; 245 246 retval = phy_connect_direct(adapter->netdev, adapter->phydev, 247 tsnep_phy_link_status_change, 248 adapter->phy_mode); 249 if (retval) 250 return retval; 251 phydev = adapter->netdev->phydev; 252 253 /* MAC supports only 100Mbps|1000Mbps full duplex 254 * SPE (Single Pair Ethernet) is also an option but not implemented yet 255 */ 256 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_10baseT_Half_BIT); 257 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_10baseT_Full_BIT); 258 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_100baseT_Half_BIT); 259 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_1000baseT_Half_BIT); 260 261 /* disable EEE autoneg, EEE not supported by TSNEP */ 262 memset(ðtool_keee, 0, sizeof(ethtool_keee)); 263 phy_ethtool_set_eee(adapter->phydev, ðtool_keee); 264 265 adapter->phydev->irq = PHY_MAC_INTERRUPT; 266 phy_start(adapter->phydev); 267 268 return 0; 269 } 270 271 static void tsnep_phy_close(struct tsnep_adapter *adapter) 272 { 273 phy_stop(adapter->netdev->phydev); 274 phy_disconnect(adapter->netdev->phydev); 275 } 276 277 static void tsnep_tx_ring_cleanup(struct tsnep_tx *tx) 278 { 279 struct device *dmadev = tx->adapter->dmadev; 280 int i; 281 282 memset(tx->entry, 0, sizeof(tx->entry)); 283 284 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) { 285 if (tx->page[i]) { 286 dma_free_coherent(dmadev, PAGE_SIZE, tx->page[i], 287 tx->page_dma[i]); 288 tx->page[i] = NULL; 289 tx->page_dma[i] = 0; 290 } 291 } 292 } 293 294 static int tsnep_tx_ring_create(struct tsnep_tx *tx) 295 { 296 struct device *dmadev = tx->adapter->dmadev; 297 struct tsnep_tx_entry *entry; 298 struct tsnep_tx_entry *next_entry; 299 int i, j; 300 int retval; 301 302 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) { 303 tx->page[i] = 304 dma_alloc_coherent(dmadev, PAGE_SIZE, &tx->page_dma[i], 305 GFP_KERNEL); 306 if (!tx->page[i]) { 307 retval = -ENOMEM; 308 goto alloc_failed; 309 } 310 for (j = 0; j < TSNEP_RING_ENTRIES_PER_PAGE; j++) { 311 entry = &tx->entry[TSNEP_RING_ENTRIES_PER_PAGE * i + j]; 312 entry->desc_wb = (struct tsnep_tx_desc_wb *) 313 (((u8 *)tx->page[i]) + TSNEP_DESC_SIZE * j); 314 entry->desc = (struct tsnep_tx_desc *) 315 (((u8 *)entry->desc_wb) + TSNEP_DESC_OFFSET); 316 entry->desc_dma = tx->page_dma[i] + TSNEP_DESC_SIZE * j; 317 entry->owner_user_flag = false; 318 } 319 } 320 for (i = 0; i < TSNEP_RING_SIZE; i++) { 321 entry = &tx->entry[i]; 322 next_entry = &tx->entry[(i + 1) & TSNEP_RING_MASK]; 323 entry->desc->next = __cpu_to_le64(next_entry->desc_dma); 324 } 325 326 return 0; 327 328 alloc_failed: 329 tsnep_tx_ring_cleanup(tx); 330 return retval; 331 } 332 333 static void tsnep_tx_init(struct tsnep_tx *tx) 334 { 335 dma_addr_t dma; 336 337 dma = tx->entry[0].desc_dma | TSNEP_RESET_OWNER_COUNTER; 338 iowrite32(DMA_ADDR_LOW(dma), tx->addr + TSNEP_TX_DESC_ADDR_LOW); 339 iowrite32(DMA_ADDR_HIGH(dma), tx->addr + TSNEP_TX_DESC_ADDR_HIGH); 340 tx->write = 0; 341 tx->read = 0; 342 tx->owner_counter = 1; 343 tx->increment_owner_counter = TSNEP_RING_SIZE - 1; 344 } 345 346 static void tsnep_tx_enable(struct tsnep_tx *tx) 347 { 348 struct netdev_queue *nq; 349 350 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index); 351 352 __netif_tx_lock_bh(nq); 353 netif_tx_wake_queue(nq); 354 __netif_tx_unlock_bh(nq); 355 } 356 357 static void tsnep_tx_disable(struct tsnep_tx *tx, struct napi_struct *napi) 358 { 359 struct netdev_queue *nq; 360 u32 val; 361 362 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index); 363 364 __netif_tx_lock_bh(nq); 365 netif_tx_stop_queue(nq); 366 __netif_tx_unlock_bh(nq); 367 368 /* wait until TX is done in hardware */ 369 readx_poll_timeout(ioread32, tx->addr + TSNEP_CONTROL, val, 370 ((val & TSNEP_CONTROL_TX_ENABLE) == 0), 10000, 371 1000000); 372 373 /* wait until TX is also done in software */ 374 while (READ_ONCE(tx->read) != tx->write) { 375 napi_schedule(napi); 376 napi_synchronize(napi); 377 } 378 } 379 380 static void tsnep_tx_activate(struct tsnep_tx *tx, int index, int length, 381 bool last) 382 { 383 struct tsnep_tx_entry *entry = &tx->entry[index]; 384 385 entry->properties = 0; 386 /* xdpf and zc are union with skb */ 387 if (entry->skb) { 388 entry->properties = length & TSNEP_DESC_LENGTH_MASK; 389 entry->properties |= TSNEP_DESC_INTERRUPT_FLAG; 390 if ((entry->type & TSNEP_TX_TYPE_SKB) && 391 (skb_shinfo(entry->skb)->tx_flags & SKBTX_IN_PROGRESS)) 392 entry->properties |= TSNEP_DESC_EXTENDED_WRITEBACK_FLAG; 393 394 /* toggle user flag to prevent false acknowledge 395 * 396 * Only the first fragment is acknowledged. For all other 397 * fragments no acknowledge is done and the last written owner 398 * counter stays in the writeback descriptor. Therefore, it is 399 * possible that the last written owner counter is identical to 400 * the new incremented owner counter and a false acknowledge is 401 * detected before the real acknowledge has been done by 402 * hardware. 403 * 404 * The user flag is used to prevent this situation. The user 405 * flag is copied to the writeback descriptor by the hardware 406 * and is used as additional acknowledge data. By toggeling the 407 * user flag only for the first fragment (which is 408 * acknowledged), it is guaranteed that the last acknowledge 409 * done for this descriptor has used a different user flag and 410 * cannot be detected as false acknowledge. 411 */ 412 entry->owner_user_flag = !entry->owner_user_flag; 413 } 414 if (last) 415 entry->properties |= TSNEP_TX_DESC_LAST_FRAGMENT_FLAG; 416 if (index == tx->increment_owner_counter) { 417 tx->owner_counter++; 418 if (tx->owner_counter == 4) 419 tx->owner_counter = 1; 420 tx->increment_owner_counter--; 421 if (tx->increment_owner_counter < 0) 422 tx->increment_owner_counter = TSNEP_RING_SIZE - 1; 423 } 424 entry->properties |= 425 (tx->owner_counter << TSNEP_DESC_OWNER_COUNTER_SHIFT) & 426 TSNEP_DESC_OWNER_COUNTER_MASK; 427 if (entry->owner_user_flag) 428 entry->properties |= TSNEP_TX_DESC_OWNER_USER_FLAG; 429 entry->desc->more_properties = 430 __cpu_to_le32(entry->len & TSNEP_DESC_LENGTH_MASK); 431 if (entry->type & TSNEP_TX_TYPE_INLINE) 432 entry->properties |= TSNEP_TX_DESC_DATA_AFTER_DESC_FLAG; 433 434 /* descriptor properties shall be written last, because valid data is 435 * signaled there 436 */ 437 dma_wmb(); 438 439 entry->desc->properties = __cpu_to_le32(entry->properties); 440 } 441 442 static int tsnep_tx_desc_available(struct tsnep_tx *tx) 443 { 444 if (tx->read <= tx->write) 445 return TSNEP_RING_SIZE - tx->write + tx->read - 1; 446 else 447 return tx->read - tx->write - 1; 448 } 449 450 static int tsnep_tx_map_frag(skb_frag_t *frag, struct tsnep_tx_entry *entry, 451 struct device *dmadev, dma_addr_t *dma) 452 { 453 unsigned int len; 454 int mapped; 455 456 len = skb_frag_size(frag); 457 if (likely(len > TSNEP_DESC_SIZE_DATA_AFTER_INLINE)) { 458 *dma = skb_frag_dma_map(dmadev, frag, 0, len, DMA_TO_DEVICE); 459 if (dma_mapping_error(dmadev, *dma)) 460 return -ENOMEM; 461 entry->type = TSNEP_TX_TYPE_SKB_FRAG_MAP_PAGE; 462 mapped = 1; 463 } else { 464 void *fragdata = skb_frag_address_safe(frag); 465 466 if (likely(fragdata)) { 467 memcpy(&entry->desc->tx, fragdata, len); 468 } else { 469 struct page *page = skb_frag_page(frag); 470 471 fragdata = kmap_local_page(page); 472 memcpy(&entry->desc->tx, fragdata + skb_frag_off(frag), 473 len); 474 kunmap_local(fragdata); 475 } 476 entry->type = TSNEP_TX_TYPE_SKB_FRAG_INLINE; 477 mapped = 0; 478 } 479 480 return mapped; 481 } 482 483 static int tsnep_tx_map(struct sk_buff *skb, struct tsnep_tx *tx, int count) 484 { 485 struct device *dmadev = tx->adapter->dmadev; 486 struct tsnep_tx_entry *entry; 487 unsigned int len; 488 int map_len = 0; 489 dma_addr_t dma; 490 int i, mapped; 491 492 for (i = 0; i < count; i++) { 493 entry = &tx->entry[(tx->write + i) & TSNEP_RING_MASK]; 494 495 if (!i) { 496 len = skb_headlen(skb); 497 if (likely(len > TSNEP_DESC_SIZE_DATA_AFTER_INLINE)) { 498 dma = dma_map_single(dmadev, skb->data, len, 499 DMA_TO_DEVICE); 500 if (dma_mapping_error(dmadev, dma)) 501 return -ENOMEM; 502 entry->type = TSNEP_TX_TYPE_SKB_MAP; 503 mapped = 1; 504 } else { 505 memcpy(&entry->desc->tx, skb->data, len); 506 entry->type = TSNEP_TX_TYPE_SKB_INLINE; 507 mapped = 0; 508 } 509 } else { 510 skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1]; 511 512 len = skb_frag_size(frag); 513 mapped = tsnep_tx_map_frag(frag, entry, dmadev, &dma); 514 if (mapped < 0) 515 return mapped; 516 } 517 518 entry->len = len; 519 if (likely(mapped)) { 520 dma_unmap_addr_set(entry, dma, dma); 521 entry->desc->tx = __cpu_to_le64(dma); 522 } 523 524 map_len += len; 525 } 526 527 return map_len; 528 } 529 530 static int tsnep_tx_unmap(struct tsnep_tx *tx, int index, int count) 531 { 532 struct device *dmadev = tx->adapter->dmadev; 533 struct tsnep_tx_entry *entry; 534 int map_len = 0; 535 int i; 536 537 for (i = 0; i < count; i++) { 538 entry = &tx->entry[(index + i) & TSNEP_RING_MASK]; 539 540 if (entry->len) { 541 if (entry->type & TSNEP_TX_TYPE_MAP) 542 dma_unmap_single(dmadev, 543 dma_unmap_addr(entry, dma), 544 dma_unmap_len(entry, len), 545 DMA_TO_DEVICE); 546 else if (entry->type & TSNEP_TX_TYPE_MAP_PAGE) 547 dma_unmap_page(dmadev, 548 dma_unmap_addr(entry, dma), 549 dma_unmap_len(entry, len), 550 DMA_TO_DEVICE); 551 map_len += entry->len; 552 entry->len = 0; 553 } 554 } 555 556 return map_len; 557 } 558 559 static netdev_tx_t tsnep_xmit_frame_ring(struct sk_buff *skb, 560 struct tsnep_tx *tx) 561 { 562 int count = 1; 563 struct tsnep_tx_entry *entry; 564 int length; 565 int i; 566 int retval; 567 568 if (skb_shinfo(skb)->nr_frags > 0) 569 count += skb_shinfo(skb)->nr_frags; 570 571 if (tsnep_tx_desc_available(tx) < count) { 572 /* ring full, shall not happen because queue is stopped if full 573 * below 574 */ 575 netif_stop_subqueue(tx->adapter->netdev, tx->queue_index); 576 577 return NETDEV_TX_BUSY; 578 } 579 580 entry = &tx->entry[tx->write]; 581 entry->skb = skb; 582 583 retval = tsnep_tx_map(skb, tx, count); 584 if (retval < 0) { 585 tsnep_tx_unmap(tx, tx->write, count); 586 dev_kfree_skb_any(entry->skb); 587 entry->skb = NULL; 588 589 tx->dropped++; 590 591 return NETDEV_TX_OK; 592 } 593 length = retval; 594 595 if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) 596 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 597 598 for (i = 0; i < count; i++) 599 tsnep_tx_activate(tx, (tx->write + i) & TSNEP_RING_MASK, length, 600 i == count - 1); 601 tx->write = (tx->write + count) & TSNEP_RING_MASK; 602 603 skb_tx_timestamp(skb); 604 605 /* descriptor properties shall be valid before hardware is notified */ 606 dma_wmb(); 607 608 iowrite32(TSNEP_CONTROL_TX_ENABLE, tx->addr + TSNEP_CONTROL); 609 610 if (tsnep_tx_desc_available(tx) < (MAX_SKB_FRAGS + 1)) { 611 /* ring can get full with next frame */ 612 netif_stop_subqueue(tx->adapter->netdev, tx->queue_index); 613 } 614 615 return NETDEV_TX_OK; 616 } 617 618 static int tsnep_xdp_tx_map(struct xdp_frame *xdpf, struct tsnep_tx *tx, 619 struct skb_shared_info *shinfo, int count, u32 type) 620 { 621 struct device *dmadev = tx->adapter->dmadev; 622 struct tsnep_tx_entry *entry; 623 struct page *page; 624 skb_frag_t *frag; 625 unsigned int len; 626 int map_len = 0; 627 dma_addr_t dma; 628 void *data; 629 int i; 630 631 frag = NULL; 632 len = xdpf->len; 633 for (i = 0; i < count; i++) { 634 entry = &tx->entry[(tx->write + i) & TSNEP_RING_MASK]; 635 if (type & TSNEP_TX_TYPE_XDP_NDO) { 636 data = unlikely(frag) ? skb_frag_address(frag) : 637 xdpf->data; 638 dma = dma_map_single(dmadev, data, len, DMA_TO_DEVICE); 639 if (dma_mapping_error(dmadev, dma)) 640 return -ENOMEM; 641 642 entry->type = TSNEP_TX_TYPE_XDP_NDO_MAP_PAGE; 643 } else { 644 page = unlikely(frag) ? skb_frag_page(frag) : 645 virt_to_page(xdpf->data); 646 dma = page_pool_get_dma_addr(page); 647 if (unlikely(frag)) 648 dma += skb_frag_off(frag); 649 else 650 dma += sizeof(*xdpf) + xdpf->headroom; 651 dma_sync_single_for_device(dmadev, dma, len, 652 DMA_BIDIRECTIONAL); 653 654 entry->type = TSNEP_TX_TYPE_XDP_TX; 655 } 656 657 entry->len = len; 658 dma_unmap_addr_set(entry, dma, dma); 659 660 entry->desc->tx = __cpu_to_le64(dma); 661 662 map_len += len; 663 664 if (i + 1 < count) { 665 frag = &shinfo->frags[i]; 666 len = skb_frag_size(frag); 667 } 668 } 669 670 return map_len; 671 } 672 673 /* This function requires __netif_tx_lock is held by the caller. */ 674 static bool tsnep_xdp_xmit_frame_ring(struct xdp_frame *xdpf, 675 struct tsnep_tx *tx, u32 type) 676 { 677 struct skb_shared_info *shinfo = xdp_get_shared_info_from_frame(xdpf); 678 struct tsnep_tx_entry *entry; 679 int count, length, retval, i; 680 681 count = 1; 682 if (unlikely(xdp_frame_has_frags(xdpf))) 683 count += shinfo->nr_frags; 684 685 /* ensure that TX ring is not filled up by XDP, always MAX_SKB_FRAGS 686 * will be available for normal TX path and queue is stopped there if 687 * necessary 688 */ 689 if (tsnep_tx_desc_available(tx) < (MAX_SKB_FRAGS + 1 + count)) 690 return false; 691 692 entry = &tx->entry[tx->write]; 693 entry->xdpf = xdpf; 694 695 retval = tsnep_xdp_tx_map(xdpf, tx, shinfo, count, type); 696 if (retval < 0) { 697 tsnep_tx_unmap(tx, tx->write, count); 698 entry->xdpf = NULL; 699 700 tx->dropped++; 701 702 return false; 703 } 704 length = retval; 705 706 for (i = 0; i < count; i++) 707 tsnep_tx_activate(tx, (tx->write + i) & TSNEP_RING_MASK, length, 708 i == count - 1); 709 tx->write = (tx->write + count) & TSNEP_RING_MASK; 710 711 /* descriptor properties shall be valid before hardware is notified */ 712 dma_wmb(); 713 714 return true; 715 } 716 717 static void tsnep_xdp_xmit_flush(struct tsnep_tx *tx) 718 { 719 iowrite32(TSNEP_CONTROL_TX_ENABLE, tx->addr + TSNEP_CONTROL); 720 } 721 722 static bool tsnep_xdp_xmit_back(struct tsnep_adapter *adapter, 723 struct xdp_buff *xdp, 724 struct netdev_queue *tx_nq, struct tsnep_tx *tx, 725 bool zc) 726 { 727 struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); 728 bool xmit; 729 u32 type; 730 731 if (unlikely(!xdpf)) 732 return false; 733 734 /* no page pool for zero copy */ 735 if (zc) 736 type = TSNEP_TX_TYPE_XDP_NDO; 737 else 738 type = TSNEP_TX_TYPE_XDP_TX; 739 740 __netif_tx_lock(tx_nq, smp_processor_id()); 741 742 xmit = tsnep_xdp_xmit_frame_ring(xdpf, tx, type); 743 744 /* Avoid transmit queue timeout since we share it with the slow path */ 745 if (xmit) 746 txq_trans_cond_update(tx_nq); 747 748 __netif_tx_unlock(tx_nq); 749 750 return xmit; 751 } 752 753 static int tsnep_xdp_tx_map_zc(struct xdp_desc *xdpd, struct tsnep_tx *tx) 754 { 755 struct tsnep_tx_entry *entry; 756 dma_addr_t dma; 757 758 entry = &tx->entry[tx->write]; 759 entry->zc = true; 760 761 dma = xsk_buff_raw_get_dma(tx->xsk_pool, xdpd->addr); 762 xsk_buff_raw_dma_sync_for_device(tx->xsk_pool, dma, xdpd->len); 763 764 entry->type = TSNEP_TX_TYPE_XSK; 765 entry->len = xdpd->len; 766 767 entry->desc->tx = __cpu_to_le64(dma); 768 769 return xdpd->len; 770 } 771 772 static void tsnep_xdp_xmit_frame_ring_zc(struct xdp_desc *xdpd, 773 struct tsnep_tx *tx) 774 { 775 int length; 776 777 length = tsnep_xdp_tx_map_zc(xdpd, tx); 778 779 tsnep_tx_activate(tx, tx->write, length, true); 780 tx->write = (tx->write + 1) & TSNEP_RING_MASK; 781 } 782 783 static void tsnep_xdp_xmit_zc(struct tsnep_tx *tx) 784 { 785 int desc_available = tsnep_tx_desc_available(tx); 786 struct xdp_desc *descs = tx->xsk_pool->tx_descs; 787 int batch, i; 788 789 /* ensure that TX ring is not filled up by XDP, always MAX_SKB_FRAGS 790 * will be available for normal TX path and queue is stopped there if 791 * necessary 792 */ 793 if (desc_available <= (MAX_SKB_FRAGS + 1)) 794 return; 795 desc_available -= MAX_SKB_FRAGS + 1; 796 797 batch = xsk_tx_peek_release_desc_batch(tx->xsk_pool, desc_available); 798 for (i = 0; i < batch; i++) 799 tsnep_xdp_xmit_frame_ring_zc(&descs[i], tx); 800 801 if (batch) { 802 /* descriptor properties shall be valid before hardware is 803 * notified 804 */ 805 dma_wmb(); 806 807 tsnep_xdp_xmit_flush(tx); 808 } 809 } 810 811 static bool tsnep_tx_poll(struct tsnep_tx *tx, int napi_budget) 812 { 813 struct tsnep_tx_entry *entry; 814 struct netdev_queue *nq; 815 int xsk_frames = 0; 816 int budget = 128; 817 int length; 818 int count; 819 820 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index); 821 __netif_tx_lock(nq, smp_processor_id()); 822 823 do { 824 if (tx->read == tx->write) 825 break; 826 827 entry = &tx->entry[tx->read]; 828 if ((__le32_to_cpu(entry->desc_wb->properties) & 829 TSNEP_TX_DESC_OWNER_MASK) != 830 (entry->properties & TSNEP_TX_DESC_OWNER_MASK)) 831 break; 832 833 /* descriptor properties shall be read first, because valid data 834 * is signaled there 835 */ 836 dma_rmb(); 837 838 count = 1; 839 if ((entry->type & TSNEP_TX_TYPE_SKB) && 840 skb_shinfo(entry->skb)->nr_frags > 0) 841 count += skb_shinfo(entry->skb)->nr_frags; 842 else if ((entry->type & TSNEP_TX_TYPE_XDP) && 843 xdp_frame_has_frags(entry->xdpf)) 844 count += xdp_get_shared_info_from_frame(entry->xdpf)->nr_frags; 845 846 length = tsnep_tx_unmap(tx, tx->read, count); 847 848 if ((entry->type & TSNEP_TX_TYPE_SKB) && 849 (skb_shinfo(entry->skb)->tx_flags & SKBTX_IN_PROGRESS) && 850 (__le32_to_cpu(entry->desc_wb->properties) & 851 TSNEP_DESC_EXTENDED_WRITEBACK_FLAG)) { 852 struct skb_shared_hwtstamps hwtstamps; 853 u64 timestamp; 854 855 if (skb_shinfo(entry->skb)->tx_flags & 856 SKBTX_HW_TSTAMP_USE_CYCLES) 857 timestamp = 858 __le64_to_cpu(entry->desc_wb->counter); 859 else 860 timestamp = 861 __le64_to_cpu(entry->desc_wb->timestamp); 862 863 memset(&hwtstamps, 0, sizeof(hwtstamps)); 864 hwtstamps.hwtstamp = ns_to_ktime(timestamp); 865 866 skb_tstamp_tx(entry->skb, &hwtstamps); 867 } 868 869 if (entry->type & TSNEP_TX_TYPE_SKB) 870 napi_consume_skb(entry->skb, napi_budget); 871 else if (entry->type & TSNEP_TX_TYPE_XDP) 872 xdp_return_frame_rx_napi(entry->xdpf); 873 else 874 xsk_frames++; 875 /* xdpf and zc are union with skb */ 876 entry->skb = NULL; 877 878 tx->read = (tx->read + count) & TSNEP_RING_MASK; 879 880 tx->packets++; 881 tx->bytes += length + ETH_FCS_LEN; 882 883 budget--; 884 } while (likely(budget)); 885 886 if (tx->xsk_pool) { 887 if (xsk_frames) 888 xsk_tx_completed(tx->xsk_pool, xsk_frames); 889 if (xsk_uses_need_wakeup(tx->xsk_pool)) 890 xsk_set_tx_need_wakeup(tx->xsk_pool); 891 tsnep_xdp_xmit_zc(tx); 892 } 893 894 if ((tsnep_tx_desc_available(tx) >= ((MAX_SKB_FRAGS + 1) * 2)) && 895 netif_tx_queue_stopped(nq)) { 896 netif_tx_wake_queue(nq); 897 } 898 899 __netif_tx_unlock(nq); 900 901 return budget != 0; 902 } 903 904 static bool tsnep_tx_pending(struct tsnep_tx *tx) 905 { 906 struct tsnep_tx_entry *entry; 907 struct netdev_queue *nq; 908 bool pending = false; 909 910 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index); 911 __netif_tx_lock(nq, smp_processor_id()); 912 913 if (tx->read != tx->write) { 914 entry = &tx->entry[tx->read]; 915 if ((__le32_to_cpu(entry->desc_wb->properties) & 916 TSNEP_TX_DESC_OWNER_MASK) == 917 (entry->properties & TSNEP_TX_DESC_OWNER_MASK)) 918 pending = true; 919 } 920 921 __netif_tx_unlock(nq); 922 923 return pending; 924 } 925 926 static int tsnep_tx_open(struct tsnep_tx *tx) 927 { 928 int retval; 929 930 retval = tsnep_tx_ring_create(tx); 931 if (retval) 932 return retval; 933 934 tsnep_tx_init(tx); 935 936 return 0; 937 } 938 939 static void tsnep_tx_close(struct tsnep_tx *tx) 940 { 941 tsnep_tx_ring_cleanup(tx); 942 } 943 944 static void tsnep_rx_ring_cleanup(struct tsnep_rx *rx) 945 { 946 struct device *dmadev = rx->adapter->dmadev; 947 struct tsnep_rx_entry *entry; 948 int i; 949 950 for (i = 0; i < TSNEP_RING_SIZE; i++) { 951 entry = &rx->entry[i]; 952 if (!rx->xsk_pool && entry->page) 953 page_pool_put_full_page(rx->page_pool, entry->page, 954 false); 955 if (rx->xsk_pool && entry->xdp) 956 xsk_buff_free(entry->xdp); 957 /* xdp is union with page */ 958 entry->page = NULL; 959 } 960 961 if (rx->page_pool) 962 page_pool_destroy(rx->page_pool); 963 964 memset(rx->entry, 0, sizeof(rx->entry)); 965 966 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) { 967 if (rx->page[i]) { 968 dma_free_coherent(dmadev, PAGE_SIZE, rx->page[i], 969 rx->page_dma[i]); 970 rx->page[i] = NULL; 971 rx->page_dma[i] = 0; 972 } 973 } 974 } 975 976 static int tsnep_rx_ring_create(struct tsnep_rx *rx) 977 { 978 struct device *dmadev = rx->adapter->dmadev; 979 struct tsnep_rx_entry *entry; 980 struct page_pool_params pp_params = { 0 }; 981 struct tsnep_rx_entry *next_entry; 982 int i, j; 983 int retval; 984 985 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) { 986 rx->page[i] = 987 dma_alloc_coherent(dmadev, PAGE_SIZE, &rx->page_dma[i], 988 GFP_KERNEL); 989 if (!rx->page[i]) { 990 retval = -ENOMEM; 991 goto failed; 992 } 993 for (j = 0; j < TSNEP_RING_ENTRIES_PER_PAGE; j++) { 994 entry = &rx->entry[TSNEP_RING_ENTRIES_PER_PAGE * i + j]; 995 entry->desc_wb = (struct tsnep_rx_desc_wb *) 996 (((u8 *)rx->page[i]) + TSNEP_DESC_SIZE * j); 997 entry->desc = (struct tsnep_rx_desc *) 998 (((u8 *)entry->desc_wb) + TSNEP_DESC_OFFSET); 999 entry->desc_dma = rx->page_dma[i] + TSNEP_DESC_SIZE * j; 1000 } 1001 } 1002 1003 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 1004 pp_params.order = 0; 1005 pp_params.pool_size = TSNEP_RING_SIZE; 1006 pp_params.nid = dev_to_node(dmadev); 1007 pp_params.dev = dmadev; 1008 pp_params.dma_dir = DMA_BIDIRECTIONAL; 1009 pp_params.max_len = TSNEP_MAX_RX_BUF_SIZE; 1010 pp_params.offset = TSNEP_RX_OFFSET; 1011 rx->page_pool = page_pool_create(&pp_params); 1012 if (IS_ERR(rx->page_pool)) { 1013 retval = PTR_ERR(rx->page_pool); 1014 rx->page_pool = NULL; 1015 goto failed; 1016 } 1017 1018 for (i = 0; i < TSNEP_RING_SIZE; i++) { 1019 entry = &rx->entry[i]; 1020 next_entry = &rx->entry[(i + 1) & TSNEP_RING_MASK]; 1021 entry->desc->next = __cpu_to_le64(next_entry->desc_dma); 1022 } 1023 1024 return 0; 1025 1026 failed: 1027 tsnep_rx_ring_cleanup(rx); 1028 return retval; 1029 } 1030 1031 static void tsnep_rx_init(struct tsnep_rx *rx) 1032 { 1033 dma_addr_t dma; 1034 1035 dma = rx->entry[0].desc_dma | TSNEP_RESET_OWNER_COUNTER; 1036 iowrite32(DMA_ADDR_LOW(dma), rx->addr + TSNEP_RX_DESC_ADDR_LOW); 1037 iowrite32(DMA_ADDR_HIGH(dma), rx->addr + TSNEP_RX_DESC_ADDR_HIGH); 1038 rx->write = 0; 1039 rx->read = 0; 1040 rx->owner_counter = 1; 1041 rx->increment_owner_counter = TSNEP_RING_SIZE - 1; 1042 } 1043 1044 static void tsnep_rx_enable(struct tsnep_rx *rx) 1045 { 1046 /* descriptor properties shall be valid before hardware is notified */ 1047 dma_wmb(); 1048 1049 iowrite32(TSNEP_CONTROL_RX_ENABLE, rx->addr + TSNEP_CONTROL); 1050 } 1051 1052 static void tsnep_rx_disable(struct tsnep_rx *rx) 1053 { 1054 u32 val; 1055 1056 iowrite32(TSNEP_CONTROL_RX_DISABLE, rx->addr + TSNEP_CONTROL); 1057 readx_poll_timeout(ioread32, rx->addr + TSNEP_CONTROL, val, 1058 ((val & TSNEP_CONTROL_RX_ENABLE) == 0), 10000, 1059 1000000); 1060 } 1061 1062 static int tsnep_rx_desc_available(struct tsnep_rx *rx) 1063 { 1064 if (rx->read <= rx->write) 1065 return TSNEP_RING_SIZE - rx->write + rx->read - 1; 1066 else 1067 return rx->read - rx->write - 1; 1068 } 1069 1070 static void tsnep_rx_free_page_buffer(struct tsnep_rx *rx) 1071 { 1072 struct page **page; 1073 1074 /* last entry of page_buffer is always zero, because ring cannot be 1075 * filled completely 1076 */ 1077 page = rx->page_buffer; 1078 while (*page) { 1079 page_pool_put_full_page(rx->page_pool, *page, false); 1080 *page = NULL; 1081 page++; 1082 } 1083 } 1084 1085 static int tsnep_rx_alloc_page_buffer(struct tsnep_rx *rx) 1086 { 1087 int i; 1088 1089 /* alloc for all ring entries except the last one, because ring cannot 1090 * be filled completely 1091 */ 1092 for (i = 0; i < TSNEP_RING_SIZE - 1; i++) { 1093 rx->page_buffer[i] = page_pool_dev_alloc_pages(rx->page_pool); 1094 if (!rx->page_buffer[i]) { 1095 tsnep_rx_free_page_buffer(rx); 1096 1097 return -ENOMEM; 1098 } 1099 } 1100 1101 return 0; 1102 } 1103 1104 static void tsnep_rx_set_page(struct tsnep_rx *rx, struct tsnep_rx_entry *entry, 1105 struct page *page) 1106 { 1107 entry->page = page; 1108 entry->len = TSNEP_MAX_RX_BUF_SIZE; 1109 entry->dma = page_pool_get_dma_addr(entry->page); 1110 entry->desc->rx = __cpu_to_le64(entry->dma + TSNEP_RX_OFFSET); 1111 } 1112 1113 static int tsnep_rx_alloc_buffer(struct tsnep_rx *rx, int index) 1114 { 1115 struct tsnep_rx_entry *entry = &rx->entry[index]; 1116 struct page *page; 1117 1118 page = page_pool_dev_alloc_pages(rx->page_pool); 1119 if (unlikely(!page)) 1120 return -ENOMEM; 1121 tsnep_rx_set_page(rx, entry, page); 1122 1123 return 0; 1124 } 1125 1126 static void tsnep_rx_reuse_buffer(struct tsnep_rx *rx, int index) 1127 { 1128 struct tsnep_rx_entry *entry = &rx->entry[index]; 1129 struct tsnep_rx_entry *read = &rx->entry[rx->read]; 1130 1131 tsnep_rx_set_page(rx, entry, read->page); 1132 read->page = NULL; 1133 } 1134 1135 static void tsnep_rx_activate(struct tsnep_rx *rx, int index) 1136 { 1137 struct tsnep_rx_entry *entry = &rx->entry[index]; 1138 1139 /* TSNEP_MAX_RX_BUF_SIZE and TSNEP_XSK_RX_BUF_SIZE are multiple of 4 */ 1140 entry->properties = entry->len & TSNEP_DESC_LENGTH_MASK; 1141 entry->properties |= TSNEP_DESC_INTERRUPT_FLAG; 1142 if (index == rx->increment_owner_counter) { 1143 rx->owner_counter++; 1144 if (rx->owner_counter == 4) 1145 rx->owner_counter = 1; 1146 rx->increment_owner_counter--; 1147 if (rx->increment_owner_counter < 0) 1148 rx->increment_owner_counter = TSNEP_RING_SIZE - 1; 1149 } 1150 entry->properties |= 1151 (rx->owner_counter << TSNEP_DESC_OWNER_COUNTER_SHIFT) & 1152 TSNEP_DESC_OWNER_COUNTER_MASK; 1153 1154 /* descriptor properties shall be written last, because valid data is 1155 * signaled there 1156 */ 1157 dma_wmb(); 1158 1159 entry->desc->properties = __cpu_to_le32(entry->properties); 1160 } 1161 1162 static int tsnep_rx_alloc(struct tsnep_rx *rx, int count, bool reuse) 1163 { 1164 bool alloc_failed = false; 1165 int i, index; 1166 1167 for (i = 0; i < count && !alloc_failed; i++) { 1168 index = (rx->write + i) & TSNEP_RING_MASK; 1169 1170 if (unlikely(tsnep_rx_alloc_buffer(rx, index))) { 1171 rx->alloc_failed++; 1172 alloc_failed = true; 1173 1174 /* reuse only if no other allocation was successful */ 1175 if (i == 0 && reuse) 1176 tsnep_rx_reuse_buffer(rx, index); 1177 else 1178 break; 1179 } 1180 1181 tsnep_rx_activate(rx, index); 1182 } 1183 1184 if (i) 1185 rx->write = (rx->write + i) & TSNEP_RING_MASK; 1186 1187 return i; 1188 } 1189 1190 static int tsnep_rx_refill(struct tsnep_rx *rx, int count, bool reuse) 1191 { 1192 int desc_refilled; 1193 1194 desc_refilled = tsnep_rx_alloc(rx, count, reuse); 1195 if (desc_refilled) 1196 tsnep_rx_enable(rx); 1197 1198 return desc_refilled; 1199 } 1200 1201 static void tsnep_rx_set_xdp(struct tsnep_rx *rx, struct tsnep_rx_entry *entry, 1202 struct xdp_buff *xdp) 1203 { 1204 entry->xdp = xdp; 1205 entry->len = TSNEP_XSK_RX_BUF_SIZE; 1206 entry->dma = xsk_buff_xdp_get_dma(entry->xdp); 1207 entry->desc->rx = __cpu_to_le64(entry->dma); 1208 } 1209 1210 static void tsnep_rx_reuse_buffer_zc(struct tsnep_rx *rx, int index) 1211 { 1212 struct tsnep_rx_entry *entry = &rx->entry[index]; 1213 struct tsnep_rx_entry *read = &rx->entry[rx->read]; 1214 1215 tsnep_rx_set_xdp(rx, entry, read->xdp); 1216 read->xdp = NULL; 1217 } 1218 1219 static int tsnep_rx_alloc_zc(struct tsnep_rx *rx, int count, bool reuse) 1220 { 1221 u32 allocated; 1222 int i; 1223 1224 allocated = xsk_buff_alloc_batch(rx->xsk_pool, rx->xdp_batch, count); 1225 for (i = 0; i < allocated; i++) { 1226 int index = (rx->write + i) & TSNEP_RING_MASK; 1227 struct tsnep_rx_entry *entry = &rx->entry[index]; 1228 1229 tsnep_rx_set_xdp(rx, entry, rx->xdp_batch[i]); 1230 tsnep_rx_activate(rx, index); 1231 } 1232 if (i == 0) { 1233 rx->alloc_failed++; 1234 1235 if (reuse) { 1236 tsnep_rx_reuse_buffer_zc(rx, rx->write); 1237 tsnep_rx_activate(rx, rx->write); 1238 } 1239 } 1240 1241 if (i) 1242 rx->write = (rx->write + i) & TSNEP_RING_MASK; 1243 1244 return i; 1245 } 1246 1247 static void tsnep_rx_free_zc(struct tsnep_rx *rx) 1248 { 1249 int i; 1250 1251 for (i = 0; i < TSNEP_RING_SIZE; i++) { 1252 struct tsnep_rx_entry *entry = &rx->entry[i]; 1253 1254 if (entry->xdp) 1255 xsk_buff_free(entry->xdp); 1256 entry->xdp = NULL; 1257 } 1258 } 1259 1260 static int tsnep_rx_refill_zc(struct tsnep_rx *rx, int count, bool reuse) 1261 { 1262 int desc_refilled; 1263 1264 desc_refilled = tsnep_rx_alloc_zc(rx, count, reuse); 1265 if (desc_refilled) 1266 tsnep_rx_enable(rx); 1267 1268 return desc_refilled; 1269 } 1270 1271 static void tsnep_xsk_rx_need_wakeup(struct tsnep_rx *rx, int desc_available) 1272 { 1273 if (desc_available) 1274 xsk_set_rx_need_wakeup(rx->xsk_pool); 1275 else 1276 xsk_clear_rx_need_wakeup(rx->xsk_pool); 1277 } 1278 1279 static bool tsnep_xdp_run_prog(struct tsnep_rx *rx, struct bpf_prog *prog, 1280 struct xdp_buff *xdp, int *status, 1281 struct netdev_queue *tx_nq, struct tsnep_tx *tx) 1282 { 1283 unsigned int length; 1284 unsigned int sync; 1285 u32 act; 1286 1287 length = xdp->data_end - xdp->data_hard_start - XDP_PACKET_HEADROOM; 1288 1289 act = bpf_prog_run_xdp(prog, xdp); 1290 switch (act) { 1291 case XDP_PASS: 1292 return false; 1293 case XDP_TX: 1294 if (!tsnep_xdp_xmit_back(rx->adapter, xdp, tx_nq, tx, false)) 1295 goto out_failure; 1296 *status |= TSNEP_XDP_TX; 1297 return true; 1298 case XDP_REDIRECT: 1299 if (xdp_do_redirect(rx->adapter->netdev, xdp, prog) < 0) 1300 goto out_failure; 1301 *status |= TSNEP_XDP_REDIRECT; 1302 return true; 1303 default: 1304 bpf_warn_invalid_xdp_action(rx->adapter->netdev, prog, act); 1305 fallthrough; 1306 case XDP_ABORTED: 1307 out_failure: 1308 trace_xdp_exception(rx->adapter->netdev, prog, act); 1309 fallthrough; 1310 case XDP_DROP: 1311 /* Due xdp_adjust_tail: DMA sync for_device cover max len CPU 1312 * touch 1313 */ 1314 sync = xdp->data_end - xdp->data_hard_start - 1315 XDP_PACKET_HEADROOM; 1316 sync = max(sync, length); 1317 page_pool_put_page(rx->page_pool, virt_to_head_page(xdp->data), 1318 sync, true); 1319 return true; 1320 } 1321 } 1322 1323 static bool tsnep_xdp_run_prog_zc(struct tsnep_rx *rx, struct bpf_prog *prog, 1324 struct xdp_buff *xdp, int *status, 1325 struct netdev_queue *tx_nq, 1326 struct tsnep_tx *tx) 1327 { 1328 u32 act; 1329 1330 act = bpf_prog_run_xdp(prog, xdp); 1331 1332 /* XDP_REDIRECT is the main action for zero-copy */ 1333 if (likely(act == XDP_REDIRECT)) { 1334 if (xdp_do_redirect(rx->adapter->netdev, xdp, prog) < 0) 1335 goto out_failure; 1336 *status |= TSNEP_XDP_REDIRECT; 1337 return true; 1338 } 1339 1340 switch (act) { 1341 case XDP_PASS: 1342 return false; 1343 case XDP_TX: 1344 if (!tsnep_xdp_xmit_back(rx->adapter, xdp, tx_nq, tx, true)) 1345 goto out_failure; 1346 *status |= TSNEP_XDP_TX; 1347 return true; 1348 default: 1349 bpf_warn_invalid_xdp_action(rx->adapter->netdev, prog, act); 1350 fallthrough; 1351 case XDP_ABORTED: 1352 out_failure: 1353 trace_xdp_exception(rx->adapter->netdev, prog, act); 1354 fallthrough; 1355 case XDP_DROP: 1356 xsk_buff_free(xdp); 1357 return true; 1358 } 1359 } 1360 1361 static void tsnep_finalize_xdp(struct tsnep_adapter *adapter, int status, 1362 struct netdev_queue *tx_nq, struct tsnep_tx *tx) 1363 { 1364 if (status & TSNEP_XDP_TX) { 1365 __netif_tx_lock(tx_nq, smp_processor_id()); 1366 tsnep_xdp_xmit_flush(tx); 1367 __netif_tx_unlock(tx_nq); 1368 } 1369 1370 if (status & TSNEP_XDP_REDIRECT) 1371 xdp_do_flush(); 1372 } 1373 1374 static struct sk_buff *tsnep_build_skb(struct tsnep_rx *rx, struct page *page, 1375 int length) 1376 { 1377 struct sk_buff *skb; 1378 1379 skb = napi_build_skb(page_address(page), PAGE_SIZE); 1380 if (unlikely(!skb)) 1381 return NULL; 1382 1383 /* update pointers within the skb to store the data */ 1384 skb_reserve(skb, TSNEP_RX_OFFSET + TSNEP_RX_INLINE_METADATA_SIZE); 1385 __skb_put(skb, length - ETH_FCS_LEN); 1386 1387 if (rx->adapter->hwtstamp_config.rx_filter == HWTSTAMP_FILTER_ALL) { 1388 struct skb_shared_hwtstamps *hwtstamps = skb_hwtstamps(skb); 1389 struct tsnep_rx_inline *rx_inline = 1390 (struct tsnep_rx_inline *)(page_address(page) + 1391 TSNEP_RX_OFFSET); 1392 1393 skb_shinfo(skb)->tx_flags |= 1394 SKBTX_HW_TSTAMP_NETDEV; 1395 memset(hwtstamps, 0, sizeof(*hwtstamps)); 1396 hwtstamps->netdev_data = rx_inline; 1397 } 1398 1399 skb_record_rx_queue(skb, rx->queue_index); 1400 skb->protocol = eth_type_trans(skb, rx->adapter->netdev); 1401 1402 return skb; 1403 } 1404 1405 static void tsnep_rx_page(struct tsnep_rx *rx, struct napi_struct *napi, 1406 struct page *page, int length) 1407 { 1408 struct sk_buff *skb; 1409 1410 skb = tsnep_build_skb(rx, page, length); 1411 if (skb) { 1412 skb_mark_for_recycle(skb); 1413 1414 rx->packets++; 1415 rx->bytes += length; 1416 if (skb->pkt_type == PACKET_MULTICAST) 1417 rx->multicast++; 1418 1419 napi_gro_receive(napi, skb); 1420 } else { 1421 page_pool_recycle_direct(rx->page_pool, page); 1422 1423 rx->dropped++; 1424 } 1425 } 1426 1427 static int tsnep_rx_poll(struct tsnep_rx *rx, struct napi_struct *napi, 1428 int budget) 1429 { 1430 struct device *dmadev = rx->adapter->dmadev; 1431 enum dma_data_direction dma_dir; 1432 struct tsnep_rx_entry *entry; 1433 struct netdev_queue *tx_nq; 1434 struct bpf_prog *prog; 1435 struct xdp_buff xdp; 1436 struct tsnep_tx *tx; 1437 int desc_available; 1438 int xdp_status = 0; 1439 int done = 0; 1440 int length; 1441 1442 desc_available = tsnep_rx_desc_available(rx); 1443 dma_dir = page_pool_get_dma_dir(rx->page_pool); 1444 prog = READ_ONCE(rx->adapter->xdp_prog); 1445 if (prog) { 1446 tx_nq = netdev_get_tx_queue(rx->adapter->netdev, 1447 rx->tx_queue_index); 1448 tx = &rx->adapter->tx[rx->tx_queue_index]; 1449 1450 xdp_init_buff(&xdp, PAGE_SIZE, &rx->xdp_rxq); 1451 } 1452 1453 while (likely(done < budget) && (rx->read != rx->write)) { 1454 entry = &rx->entry[rx->read]; 1455 if ((__le32_to_cpu(entry->desc_wb->properties) & 1456 TSNEP_DESC_OWNER_COUNTER_MASK) != 1457 (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK)) 1458 break; 1459 done++; 1460 1461 if (desc_available >= TSNEP_RING_RX_REFILL) { 1462 bool reuse = desc_available >= TSNEP_RING_RX_REUSE; 1463 1464 desc_available -= tsnep_rx_refill(rx, desc_available, 1465 reuse); 1466 if (!entry->page) { 1467 /* buffer has been reused for refill to prevent 1468 * empty RX ring, thus buffer cannot be used for 1469 * RX processing 1470 */ 1471 rx->read = (rx->read + 1) & TSNEP_RING_MASK; 1472 desc_available++; 1473 1474 rx->dropped++; 1475 1476 continue; 1477 } 1478 } 1479 1480 /* descriptor properties shall be read first, because valid data 1481 * is signaled there 1482 */ 1483 dma_rmb(); 1484 1485 prefetch(page_address(entry->page) + TSNEP_RX_OFFSET); 1486 length = __le32_to_cpu(entry->desc_wb->properties) & 1487 TSNEP_DESC_LENGTH_MASK; 1488 dma_sync_single_range_for_cpu(dmadev, entry->dma, 1489 TSNEP_RX_OFFSET, length, dma_dir); 1490 1491 /* RX metadata with timestamps is in front of actual data, 1492 * subtract metadata size to get length of actual data and 1493 * consider metadata size as offset of actual data during RX 1494 * processing 1495 */ 1496 length -= TSNEP_RX_INLINE_METADATA_SIZE; 1497 1498 rx->read = (rx->read + 1) & TSNEP_RING_MASK; 1499 desc_available++; 1500 1501 if (prog) { 1502 bool consume; 1503 1504 xdp_prepare_buff(&xdp, page_address(entry->page), 1505 XDP_PACKET_HEADROOM + TSNEP_RX_INLINE_METADATA_SIZE, 1506 length - ETH_FCS_LEN, false); 1507 1508 consume = tsnep_xdp_run_prog(rx, prog, &xdp, 1509 &xdp_status, tx_nq, tx); 1510 if (consume) { 1511 rx->packets++; 1512 rx->bytes += length; 1513 1514 entry->page = NULL; 1515 1516 continue; 1517 } 1518 } 1519 1520 tsnep_rx_page(rx, napi, entry->page, length); 1521 entry->page = NULL; 1522 } 1523 1524 if (xdp_status) 1525 tsnep_finalize_xdp(rx->adapter, xdp_status, tx_nq, tx); 1526 1527 if (desc_available) 1528 tsnep_rx_refill(rx, desc_available, false); 1529 1530 return done; 1531 } 1532 1533 static int tsnep_rx_poll_zc(struct tsnep_rx *rx, struct napi_struct *napi, 1534 int budget) 1535 { 1536 struct tsnep_rx_entry *entry; 1537 struct netdev_queue *tx_nq; 1538 struct bpf_prog *prog; 1539 struct tsnep_tx *tx; 1540 int desc_available; 1541 int xdp_status = 0; 1542 struct page *page; 1543 int done = 0; 1544 int length; 1545 1546 desc_available = tsnep_rx_desc_available(rx); 1547 prog = READ_ONCE(rx->adapter->xdp_prog); 1548 if (prog) { 1549 tx_nq = netdev_get_tx_queue(rx->adapter->netdev, 1550 rx->tx_queue_index); 1551 tx = &rx->adapter->tx[rx->tx_queue_index]; 1552 } 1553 1554 while (likely(done < budget) && (rx->read != rx->write)) { 1555 entry = &rx->entry[rx->read]; 1556 if ((__le32_to_cpu(entry->desc_wb->properties) & 1557 TSNEP_DESC_OWNER_COUNTER_MASK) != 1558 (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK)) 1559 break; 1560 done++; 1561 1562 if (desc_available >= TSNEP_RING_RX_REFILL) { 1563 bool reuse = desc_available >= TSNEP_RING_RX_REUSE; 1564 1565 desc_available -= tsnep_rx_refill_zc(rx, desc_available, 1566 reuse); 1567 if (!entry->xdp) { 1568 /* buffer has been reused for refill to prevent 1569 * empty RX ring, thus buffer cannot be used for 1570 * RX processing 1571 */ 1572 rx->read = (rx->read + 1) & TSNEP_RING_MASK; 1573 desc_available++; 1574 1575 rx->dropped++; 1576 1577 continue; 1578 } 1579 } 1580 1581 /* descriptor properties shall be read first, because valid data 1582 * is signaled there 1583 */ 1584 dma_rmb(); 1585 1586 prefetch(entry->xdp->data); 1587 length = __le32_to_cpu(entry->desc_wb->properties) & 1588 TSNEP_DESC_LENGTH_MASK; 1589 xsk_buff_set_size(entry->xdp, length - ETH_FCS_LEN); 1590 xsk_buff_dma_sync_for_cpu(entry->xdp, rx->xsk_pool); 1591 1592 /* RX metadata with timestamps is in front of actual data, 1593 * subtract metadata size to get length of actual data and 1594 * consider metadata size as offset of actual data during RX 1595 * processing 1596 */ 1597 length -= TSNEP_RX_INLINE_METADATA_SIZE; 1598 1599 rx->read = (rx->read + 1) & TSNEP_RING_MASK; 1600 desc_available++; 1601 1602 if (prog) { 1603 bool consume; 1604 1605 entry->xdp->data += TSNEP_RX_INLINE_METADATA_SIZE; 1606 entry->xdp->data_meta += TSNEP_RX_INLINE_METADATA_SIZE; 1607 1608 consume = tsnep_xdp_run_prog_zc(rx, prog, entry->xdp, 1609 &xdp_status, tx_nq, tx); 1610 if (consume) { 1611 rx->packets++; 1612 rx->bytes += length; 1613 1614 entry->xdp = NULL; 1615 1616 continue; 1617 } 1618 } 1619 1620 page = page_pool_dev_alloc_pages(rx->page_pool); 1621 if (page) { 1622 memcpy(page_address(page) + TSNEP_RX_OFFSET, 1623 entry->xdp->data - TSNEP_RX_INLINE_METADATA_SIZE, 1624 length + TSNEP_RX_INLINE_METADATA_SIZE); 1625 tsnep_rx_page(rx, napi, page, length); 1626 } else { 1627 rx->dropped++; 1628 } 1629 xsk_buff_free(entry->xdp); 1630 entry->xdp = NULL; 1631 } 1632 1633 if (xdp_status) 1634 tsnep_finalize_xdp(rx->adapter, xdp_status, tx_nq, tx); 1635 1636 if (desc_available) 1637 desc_available -= tsnep_rx_refill_zc(rx, desc_available, false); 1638 1639 if (xsk_uses_need_wakeup(rx->xsk_pool)) { 1640 tsnep_xsk_rx_need_wakeup(rx, desc_available); 1641 1642 return done; 1643 } 1644 1645 return desc_available ? budget : done; 1646 } 1647 1648 static bool tsnep_rx_pending(struct tsnep_rx *rx) 1649 { 1650 struct tsnep_rx_entry *entry; 1651 1652 if (rx->read != rx->write) { 1653 entry = &rx->entry[rx->read]; 1654 if ((__le32_to_cpu(entry->desc_wb->properties) & 1655 TSNEP_DESC_OWNER_COUNTER_MASK) == 1656 (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK)) 1657 return true; 1658 } 1659 1660 return false; 1661 } 1662 1663 static int tsnep_rx_open(struct tsnep_rx *rx) 1664 { 1665 int desc_available; 1666 int retval; 1667 1668 retval = tsnep_rx_ring_create(rx); 1669 if (retval) 1670 return retval; 1671 1672 tsnep_rx_init(rx); 1673 1674 desc_available = tsnep_rx_desc_available(rx); 1675 if (rx->xsk_pool) 1676 retval = tsnep_rx_alloc_zc(rx, desc_available, false); 1677 else 1678 retval = tsnep_rx_alloc(rx, desc_available, false); 1679 if (retval != desc_available) { 1680 retval = -ENOMEM; 1681 1682 goto alloc_failed; 1683 } 1684 1685 /* prealloc pages to prevent allocation failures when XSK pool is 1686 * disabled at runtime 1687 */ 1688 if (rx->xsk_pool) { 1689 retval = tsnep_rx_alloc_page_buffer(rx); 1690 if (retval) 1691 goto alloc_failed; 1692 } 1693 1694 return 0; 1695 1696 alloc_failed: 1697 tsnep_rx_ring_cleanup(rx); 1698 return retval; 1699 } 1700 1701 static void tsnep_rx_close(struct tsnep_rx *rx) 1702 { 1703 if (rx->xsk_pool) 1704 tsnep_rx_free_page_buffer(rx); 1705 1706 tsnep_rx_ring_cleanup(rx); 1707 } 1708 1709 static void tsnep_rx_reopen(struct tsnep_rx *rx) 1710 { 1711 struct page **page = rx->page_buffer; 1712 int i; 1713 1714 tsnep_rx_init(rx); 1715 1716 for (i = 0; i < TSNEP_RING_SIZE; i++) { 1717 struct tsnep_rx_entry *entry = &rx->entry[i]; 1718 1719 /* defined initial values for properties are required for 1720 * correct owner counter checking 1721 */ 1722 entry->desc->properties = 0; 1723 entry->desc_wb->properties = 0; 1724 1725 /* prevent allocation failures by reusing kept pages */ 1726 if (*page) { 1727 tsnep_rx_set_page(rx, entry, *page); 1728 tsnep_rx_activate(rx, rx->write); 1729 rx->write++; 1730 1731 *page = NULL; 1732 page++; 1733 } 1734 } 1735 } 1736 1737 static void tsnep_rx_reopen_xsk(struct tsnep_rx *rx) 1738 { 1739 struct page **page = rx->page_buffer; 1740 u32 allocated; 1741 int i; 1742 1743 tsnep_rx_init(rx); 1744 1745 /* alloc all ring entries except the last one, because ring cannot be 1746 * filled completely, as many buffers as possible is enough as wakeup is 1747 * done if new buffers are available 1748 */ 1749 allocated = xsk_buff_alloc_batch(rx->xsk_pool, rx->xdp_batch, 1750 TSNEP_RING_SIZE - 1); 1751 1752 for (i = 0; i < TSNEP_RING_SIZE; i++) { 1753 struct tsnep_rx_entry *entry = &rx->entry[i]; 1754 1755 /* keep pages to prevent allocation failures when xsk is 1756 * disabled 1757 */ 1758 if (entry->page) { 1759 *page = entry->page; 1760 entry->page = NULL; 1761 1762 page++; 1763 } 1764 1765 /* defined initial values for properties are required for 1766 * correct owner counter checking 1767 */ 1768 entry->desc->properties = 0; 1769 entry->desc_wb->properties = 0; 1770 1771 if (allocated) { 1772 tsnep_rx_set_xdp(rx, entry, 1773 rx->xdp_batch[allocated - 1]); 1774 tsnep_rx_activate(rx, rx->write); 1775 rx->write++; 1776 1777 allocated--; 1778 } 1779 } 1780 1781 /* set need wakeup flag immediately if ring is not filled completely, 1782 * first polling would be too late as need wakeup signalisation would 1783 * be delayed for an indefinite time 1784 */ 1785 if (xsk_uses_need_wakeup(rx->xsk_pool)) 1786 tsnep_xsk_rx_need_wakeup(rx, tsnep_rx_desc_available(rx)); 1787 } 1788 1789 static bool tsnep_pending(struct tsnep_queue *queue) 1790 { 1791 if (queue->tx && tsnep_tx_pending(queue->tx)) 1792 return true; 1793 1794 if (queue->rx && tsnep_rx_pending(queue->rx)) 1795 return true; 1796 1797 return false; 1798 } 1799 1800 static int tsnep_poll(struct napi_struct *napi, int budget) 1801 { 1802 struct tsnep_queue *queue = container_of(napi, struct tsnep_queue, 1803 napi); 1804 bool complete = true; 1805 int done = 0; 1806 1807 if (queue->tx) 1808 complete = tsnep_tx_poll(queue->tx, budget); 1809 1810 /* handle case where we are called by netpoll with a budget of 0 */ 1811 if (unlikely(budget <= 0)) 1812 return budget; 1813 1814 if (queue->rx) { 1815 done = queue->rx->xsk_pool ? 1816 tsnep_rx_poll_zc(queue->rx, napi, budget) : 1817 tsnep_rx_poll(queue->rx, napi, budget); 1818 if (done >= budget) 1819 complete = false; 1820 } 1821 1822 /* if all work not completed, return budget and keep polling */ 1823 if (!complete) 1824 return budget; 1825 1826 if (likely(napi_complete_done(napi, done))) { 1827 tsnep_enable_irq(queue->adapter, queue->irq_mask); 1828 1829 /* reschedule if work is already pending, prevent rotten packets 1830 * which are transmitted or received after polling but before 1831 * interrupt enable 1832 */ 1833 if (tsnep_pending(queue)) { 1834 tsnep_disable_irq(queue->adapter, queue->irq_mask); 1835 napi_schedule(napi); 1836 } 1837 } 1838 1839 return min(done, budget - 1); 1840 } 1841 1842 static int tsnep_request_irq(struct tsnep_queue *queue, bool first) 1843 { 1844 const char *name = netdev_name(queue->adapter->netdev); 1845 irq_handler_t handler; 1846 void *dev; 1847 int retval; 1848 1849 if (first) { 1850 sprintf(queue->name, "%s-mac", name); 1851 handler = tsnep_irq; 1852 dev = queue->adapter; 1853 } else { 1854 if (queue->tx && queue->rx) 1855 snprintf(queue->name, sizeof(queue->name), "%s-txrx-%d", 1856 name, queue->rx->queue_index); 1857 else if (queue->tx) 1858 snprintf(queue->name, sizeof(queue->name), "%s-tx-%d", 1859 name, queue->tx->queue_index); 1860 else 1861 snprintf(queue->name, sizeof(queue->name), "%s-rx-%d", 1862 name, queue->rx->queue_index); 1863 handler = tsnep_irq_txrx; 1864 dev = queue; 1865 } 1866 1867 retval = request_irq(queue->irq, handler, 0, queue->name, dev); 1868 if (retval) { 1869 /* if name is empty, then interrupt won't be freed */ 1870 memset(queue->name, 0, sizeof(queue->name)); 1871 } 1872 1873 return retval; 1874 } 1875 1876 static void tsnep_free_irq(struct tsnep_queue *queue, bool first) 1877 { 1878 void *dev; 1879 1880 if (!strlen(queue->name)) 1881 return; 1882 1883 if (first) 1884 dev = queue->adapter; 1885 else 1886 dev = queue; 1887 1888 free_irq(queue->irq, dev); 1889 memset(queue->name, 0, sizeof(queue->name)); 1890 } 1891 1892 static void tsnep_queue_close(struct tsnep_queue *queue, bool first) 1893 { 1894 struct tsnep_rx *rx = queue->rx; 1895 1896 tsnep_free_irq(queue, first); 1897 1898 if (rx) { 1899 if (xdp_rxq_info_is_reg(&rx->xdp_rxq)) 1900 xdp_rxq_info_unreg(&rx->xdp_rxq); 1901 if (xdp_rxq_info_is_reg(&rx->xdp_rxq_zc)) 1902 xdp_rxq_info_unreg(&rx->xdp_rxq_zc); 1903 } 1904 1905 netif_napi_del(&queue->napi); 1906 } 1907 1908 static int tsnep_queue_open(struct tsnep_adapter *adapter, 1909 struct tsnep_queue *queue, bool first) 1910 { 1911 struct tsnep_rx *rx = queue->rx; 1912 struct tsnep_tx *tx = queue->tx; 1913 int retval; 1914 1915 netif_napi_add(adapter->netdev, &queue->napi, tsnep_poll); 1916 1917 if (rx) { 1918 /* choose TX queue for XDP_TX */ 1919 if (tx) 1920 rx->tx_queue_index = tx->queue_index; 1921 else if (rx->queue_index < adapter->num_tx_queues) 1922 rx->tx_queue_index = rx->queue_index; 1923 else 1924 rx->tx_queue_index = 0; 1925 1926 /* prepare both memory models to eliminate possible registration 1927 * errors when memory model is switched between page pool and 1928 * XSK pool during runtime 1929 */ 1930 retval = xdp_rxq_info_reg(&rx->xdp_rxq, adapter->netdev, 1931 rx->queue_index, queue->napi.napi_id); 1932 if (retval) 1933 goto failed; 1934 retval = xdp_rxq_info_reg_mem_model(&rx->xdp_rxq, 1935 MEM_TYPE_PAGE_POOL, 1936 rx->page_pool); 1937 if (retval) 1938 goto failed; 1939 retval = xdp_rxq_info_reg(&rx->xdp_rxq_zc, adapter->netdev, 1940 rx->queue_index, queue->napi.napi_id); 1941 if (retval) 1942 goto failed; 1943 retval = xdp_rxq_info_reg_mem_model(&rx->xdp_rxq_zc, 1944 MEM_TYPE_XSK_BUFF_POOL, 1945 NULL); 1946 if (retval) 1947 goto failed; 1948 if (rx->xsk_pool) 1949 xsk_pool_set_rxq_info(rx->xsk_pool, &rx->xdp_rxq_zc); 1950 } 1951 1952 retval = tsnep_request_irq(queue, first); 1953 if (retval) { 1954 netif_err(adapter, drv, adapter->netdev, 1955 "can't get assigned irq %d.\n", queue->irq); 1956 goto failed; 1957 } 1958 1959 return 0; 1960 1961 failed: 1962 tsnep_queue_close(queue, first); 1963 1964 return retval; 1965 } 1966 1967 static void tsnep_queue_enable(struct tsnep_queue *queue) 1968 { 1969 napi_enable(&queue->napi); 1970 tsnep_enable_irq(queue->adapter, queue->irq_mask); 1971 1972 if (queue->tx) 1973 tsnep_tx_enable(queue->tx); 1974 1975 if (queue->rx) 1976 tsnep_rx_enable(queue->rx); 1977 } 1978 1979 static void tsnep_queue_disable(struct tsnep_queue *queue) 1980 { 1981 if (queue->tx) 1982 tsnep_tx_disable(queue->tx, &queue->napi); 1983 1984 napi_disable(&queue->napi); 1985 tsnep_disable_irq(queue->adapter, queue->irq_mask); 1986 1987 /* disable RX after NAPI polling has been disabled, because RX can be 1988 * enabled during NAPI polling 1989 */ 1990 if (queue->rx) 1991 tsnep_rx_disable(queue->rx); 1992 } 1993 1994 static int tsnep_netdev_open(struct net_device *netdev) 1995 { 1996 struct tsnep_adapter *adapter = netdev_priv(netdev); 1997 int i, retval; 1998 1999 for (i = 0; i < adapter->num_queues; i++) { 2000 if (adapter->queue[i].tx) { 2001 retval = tsnep_tx_open(adapter->queue[i].tx); 2002 if (retval) 2003 goto failed; 2004 } 2005 if (adapter->queue[i].rx) { 2006 retval = tsnep_rx_open(adapter->queue[i].rx); 2007 if (retval) 2008 goto failed; 2009 } 2010 2011 retval = tsnep_queue_open(adapter, &adapter->queue[i], i == 0); 2012 if (retval) 2013 goto failed; 2014 } 2015 2016 retval = netif_set_real_num_tx_queues(adapter->netdev, 2017 adapter->num_tx_queues); 2018 if (retval) 2019 goto failed; 2020 retval = netif_set_real_num_rx_queues(adapter->netdev, 2021 adapter->num_rx_queues); 2022 if (retval) 2023 goto failed; 2024 2025 tsnep_enable_irq(adapter, ECM_INT_LINK); 2026 retval = tsnep_phy_open(adapter); 2027 if (retval) 2028 goto phy_failed; 2029 2030 for (i = 0; i < adapter->num_queues; i++) 2031 tsnep_queue_enable(&adapter->queue[i]); 2032 2033 return 0; 2034 2035 phy_failed: 2036 tsnep_disable_irq(adapter, ECM_INT_LINK); 2037 failed: 2038 for (i = 0; i < adapter->num_queues; i++) { 2039 tsnep_queue_close(&adapter->queue[i], i == 0); 2040 2041 if (adapter->queue[i].rx) 2042 tsnep_rx_close(adapter->queue[i].rx); 2043 if (adapter->queue[i].tx) 2044 tsnep_tx_close(adapter->queue[i].tx); 2045 } 2046 return retval; 2047 } 2048 2049 static int tsnep_netdev_close(struct net_device *netdev) 2050 { 2051 struct tsnep_adapter *adapter = netdev_priv(netdev); 2052 int i; 2053 2054 tsnep_disable_irq(adapter, ECM_INT_LINK); 2055 tsnep_phy_close(adapter); 2056 2057 for (i = 0; i < adapter->num_queues; i++) { 2058 tsnep_queue_disable(&adapter->queue[i]); 2059 2060 tsnep_queue_close(&adapter->queue[i], i == 0); 2061 2062 if (adapter->queue[i].rx) 2063 tsnep_rx_close(adapter->queue[i].rx); 2064 if (adapter->queue[i].tx) 2065 tsnep_tx_close(adapter->queue[i].tx); 2066 } 2067 2068 return 0; 2069 } 2070 2071 int tsnep_enable_xsk(struct tsnep_queue *queue, struct xsk_buff_pool *pool) 2072 { 2073 bool running = netif_running(queue->adapter->netdev); 2074 u32 frame_size; 2075 2076 frame_size = xsk_pool_get_rx_frame_size(pool); 2077 if (frame_size < TSNEP_XSK_RX_BUF_SIZE) 2078 return -EOPNOTSUPP; 2079 2080 queue->rx->page_buffer = kcalloc(TSNEP_RING_SIZE, 2081 sizeof(*queue->rx->page_buffer), 2082 GFP_KERNEL); 2083 if (!queue->rx->page_buffer) 2084 return -ENOMEM; 2085 queue->rx->xdp_batch = kcalloc(TSNEP_RING_SIZE, 2086 sizeof(*queue->rx->xdp_batch), 2087 GFP_KERNEL); 2088 if (!queue->rx->xdp_batch) { 2089 kfree(queue->rx->page_buffer); 2090 queue->rx->page_buffer = NULL; 2091 2092 return -ENOMEM; 2093 } 2094 2095 xsk_pool_set_rxq_info(pool, &queue->rx->xdp_rxq_zc); 2096 2097 if (running) 2098 tsnep_queue_disable(queue); 2099 2100 queue->tx->xsk_pool = pool; 2101 queue->rx->xsk_pool = pool; 2102 2103 if (running) { 2104 tsnep_rx_reopen_xsk(queue->rx); 2105 tsnep_queue_enable(queue); 2106 } 2107 2108 return 0; 2109 } 2110 2111 void tsnep_disable_xsk(struct tsnep_queue *queue) 2112 { 2113 bool running = netif_running(queue->adapter->netdev); 2114 2115 if (running) 2116 tsnep_queue_disable(queue); 2117 2118 tsnep_rx_free_zc(queue->rx); 2119 2120 queue->rx->xsk_pool = NULL; 2121 queue->tx->xsk_pool = NULL; 2122 2123 if (running) { 2124 tsnep_rx_reopen(queue->rx); 2125 tsnep_queue_enable(queue); 2126 } 2127 2128 kfree(queue->rx->xdp_batch); 2129 queue->rx->xdp_batch = NULL; 2130 kfree(queue->rx->page_buffer); 2131 queue->rx->page_buffer = NULL; 2132 } 2133 2134 static netdev_tx_t tsnep_netdev_xmit_frame(struct sk_buff *skb, 2135 struct net_device *netdev) 2136 { 2137 struct tsnep_adapter *adapter = netdev_priv(netdev); 2138 u16 queue_mapping = skb_get_queue_mapping(skb); 2139 2140 if (queue_mapping >= adapter->num_tx_queues) 2141 queue_mapping = 0; 2142 2143 return tsnep_xmit_frame_ring(skb, &adapter->tx[queue_mapping]); 2144 } 2145 2146 static int tsnep_netdev_ioctl(struct net_device *netdev, struct ifreq *ifr, 2147 int cmd) 2148 { 2149 if (!netif_running(netdev)) 2150 return -EINVAL; 2151 if (cmd == SIOCSHWTSTAMP || cmd == SIOCGHWTSTAMP) 2152 return tsnep_ptp_ioctl(netdev, ifr, cmd); 2153 return phy_mii_ioctl(netdev->phydev, ifr, cmd); 2154 } 2155 2156 static void tsnep_netdev_set_multicast(struct net_device *netdev) 2157 { 2158 struct tsnep_adapter *adapter = netdev_priv(netdev); 2159 2160 u16 rx_filter = 0; 2161 2162 /* configured MAC address and broadcasts are never filtered */ 2163 if (netdev->flags & IFF_PROMISC) { 2164 rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_MULTICASTS; 2165 rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_UNICASTS; 2166 } else if (!netdev_mc_empty(netdev) || (netdev->flags & IFF_ALLMULTI)) { 2167 rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_MULTICASTS; 2168 } 2169 iowrite16(rx_filter, adapter->addr + TSNEP_RX_FILTER); 2170 } 2171 2172 static void tsnep_netdev_get_stats64(struct net_device *netdev, 2173 struct rtnl_link_stats64 *stats) 2174 { 2175 struct tsnep_adapter *adapter = netdev_priv(netdev); 2176 u32 reg; 2177 u32 val; 2178 int i; 2179 2180 for (i = 0; i < adapter->num_tx_queues; i++) { 2181 stats->tx_packets += adapter->tx[i].packets; 2182 stats->tx_bytes += adapter->tx[i].bytes; 2183 stats->tx_dropped += adapter->tx[i].dropped; 2184 } 2185 for (i = 0; i < adapter->num_rx_queues; i++) { 2186 stats->rx_packets += adapter->rx[i].packets; 2187 stats->rx_bytes += adapter->rx[i].bytes; 2188 stats->rx_dropped += adapter->rx[i].dropped; 2189 stats->multicast += adapter->rx[i].multicast; 2190 2191 reg = ioread32(adapter->addr + TSNEP_QUEUE(i) + 2192 TSNEP_RX_STATISTIC); 2193 val = (reg & TSNEP_RX_STATISTIC_NO_DESC_MASK) >> 2194 TSNEP_RX_STATISTIC_NO_DESC_SHIFT; 2195 stats->rx_dropped += val; 2196 val = (reg & TSNEP_RX_STATISTIC_BUFFER_TOO_SMALL_MASK) >> 2197 TSNEP_RX_STATISTIC_BUFFER_TOO_SMALL_SHIFT; 2198 stats->rx_dropped += val; 2199 val = (reg & TSNEP_RX_STATISTIC_FIFO_OVERFLOW_MASK) >> 2200 TSNEP_RX_STATISTIC_FIFO_OVERFLOW_SHIFT; 2201 stats->rx_errors += val; 2202 stats->rx_fifo_errors += val; 2203 val = (reg & TSNEP_RX_STATISTIC_INVALID_FRAME_MASK) >> 2204 TSNEP_RX_STATISTIC_INVALID_FRAME_SHIFT; 2205 stats->rx_errors += val; 2206 stats->rx_frame_errors += val; 2207 } 2208 2209 reg = ioread32(adapter->addr + ECM_STAT); 2210 val = (reg & ECM_STAT_RX_ERR_MASK) >> ECM_STAT_RX_ERR_SHIFT; 2211 stats->rx_errors += val; 2212 val = (reg & ECM_STAT_INV_FRM_MASK) >> ECM_STAT_INV_FRM_SHIFT; 2213 stats->rx_errors += val; 2214 stats->rx_crc_errors += val; 2215 val = (reg & ECM_STAT_FWD_RX_ERR_MASK) >> ECM_STAT_FWD_RX_ERR_SHIFT; 2216 stats->rx_errors += val; 2217 } 2218 2219 static void tsnep_mac_set_address(struct tsnep_adapter *adapter, u8 *addr) 2220 { 2221 iowrite32(*(u32 *)addr, adapter->addr + TSNEP_MAC_ADDRESS_LOW); 2222 iowrite16(*(u16 *)(addr + sizeof(u32)), 2223 adapter->addr + TSNEP_MAC_ADDRESS_HIGH); 2224 2225 ether_addr_copy(adapter->mac_address, addr); 2226 netif_info(adapter, drv, adapter->netdev, "MAC address set to %pM\n", 2227 addr); 2228 } 2229 2230 static int tsnep_netdev_set_mac_address(struct net_device *netdev, void *addr) 2231 { 2232 struct tsnep_adapter *adapter = netdev_priv(netdev); 2233 struct sockaddr *sock_addr = addr; 2234 int retval; 2235 2236 retval = eth_prepare_mac_addr_change(netdev, sock_addr); 2237 if (retval) 2238 return retval; 2239 eth_hw_addr_set(netdev, sock_addr->sa_data); 2240 tsnep_mac_set_address(adapter, sock_addr->sa_data); 2241 2242 return 0; 2243 } 2244 2245 static int tsnep_netdev_set_features(struct net_device *netdev, 2246 netdev_features_t features) 2247 { 2248 struct tsnep_adapter *adapter = netdev_priv(netdev); 2249 netdev_features_t changed = netdev->features ^ features; 2250 bool enable; 2251 int retval = 0; 2252 2253 if (changed & NETIF_F_LOOPBACK) { 2254 enable = !!(features & NETIF_F_LOOPBACK); 2255 retval = tsnep_phy_loopback(adapter, enable); 2256 } 2257 2258 return retval; 2259 } 2260 2261 static ktime_t tsnep_netdev_get_tstamp(struct net_device *netdev, 2262 const struct skb_shared_hwtstamps *hwtstamps, 2263 bool cycles) 2264 { 2265 struct tsnep_rx_inline *rx_inline = hwtstamps->netdev_data; 2266 u64 timestamp; 2267 2268 if (cycles) 2269 timestamp = __le64_to_cpu(rx_inline->counter); 2270 else 2271 timestamp = __le64_to_cpu(rx_inline->timestamp); 2272 2273 return ns_to_ktime(timestamp); 2274 } 2275 2276 static int tsnep_netdev_bpf(struct net_device *dev, struct netdev_bpf *bpf) 2277 { 2278 struct tsnep_adapter *adapter = netdev_priv(dev); 2279 2280 switch (bpf->command) { 2281 case XDP_SETUP_PROG: 2282 return tsnep_xdp_setup_prog(adapter, bpf->prog, bpf->extack); 2283 case XDP_SETUP_XSK_POOL: 2284 return tsnep_xdp_setup_pool(adapter, bpf->xsk.pool, 2285 bpf->xsk.queue_id); 2286 default: 2287 return -EOPNOTSUPP; 2288 } 2289 } 2290 2291 static struct tsnep_tx *tsnep_xdp_get_tx(struct tsnep_adapter *adapter, u32 cpu) 2292 { 2293 if (cpu >= TSNEP_MAX_QUEUES) 2294 cpu &= TSNEP_MAX_QUEUES - 1; 2295 2296 while (cpu >= adapter->num_tx_queues) 2297 cpu -= adapter->num_tx_queues; 2298 2299 return &adapter->tx[cpu]; 2300 } 2301 2302 static int tsnep_netdev_xdp_xmit(struct net_device *dev, int n, 2303 struct xdp_frame **xdp, u32 flags) 2304 { 2305 struct tsnep_adapter *adapter = netdev_priv(dev); 2306 u32 cpu = smp_processor_id(); 2307 struct netdev_queue *nq; 2308 struct tsnep_tx *tx; 2309 int nxmit; 2310 bool xmit; 2311 2312 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 2313 return -EINVAL; 2314 2315 tx = tsnep_xdp_get_tx(adapter, cpu); 2316 nq = netdev_get_tx_queue(adapter->netdev, tx->queue_index); 2317 2318 __netif_tx_lock(nq, cpu); 2319 2320 for (nxmit = 0; nxmit < n; nxmit++) { 2321 xmit = tsnep_xdp_xmit_frame_ring(xdp[nxmit], tx, 2322 TSNEP_TX_TYPE_XDP_NDO); 2323 if (!xmit) 2324 break; 2325 2326 /* avoid transmit queue timeout since we share it with the slow 2327 * path 2328 */ 2329 txq_trans_cond_update(nq); 2330 } 2331 2332 if (flags & XDP_XMIT_FLUSH) 2333 tsnep_xdp_xmit_flush(tx); 2334 2335 __netif_tx_unlock(nq); 2336 2337 return nxmit; 2338 } 2339 2340 static int tsnep_netdev_xsk_wakeup(struct net_device *dev, u32 queue_id, 2341 u32 flags) 2342 { 2343 struct tsnep_adapter *adapter = netdev_priv(dev); 2344 struct tsnep_queue *queue; 2345 2346 if (queue_id >= adapter->num_rx_queues || 2347 queue_id >= adapter->num_tx_queues) 2348 return -EINVAL; 2349 2350 queue = &adapter->queue[queue_id]; 2351 2352 if (!napi_if_scheduled_mark_missed(&queue->napi)) 2353 napi_schedule(&queue->napi); 2354 2355 return 0; 2356 } 2357 2358 static const struct net_device_ops tsnep_netdev_ops = { 2359 .ndo_open = tsnep_netdev_open, 2360 .ndo_stop = tsnep_netdev_close, 2361 .ndo_start_xmit = tsnep_netdev_xmit_frame, 2362 .ndo_eth_ioctl = tsnep_netdev_ioctl, 2363 .ndo_set_rx_mode = tsnep_netdev_set_multicast, 2364 .ndo_get_stats64 = tsnep_netdev_get_stats64, 2365 .ndo_set_mac_address = tsnep_netdev_set_mac_address, 2366 .ndo_set_features = tsnep_netdev_set_features, 2367 .ndo_get_tstamp = tsnep_netdev_get_tstamp, 2368 .ndo_setup_tc = tsnep_tc_setup, 2369 .ndo_bpf = tsnep_netdev_bpf, 2370 .ndo_xdp_xmit = tsnep_netdev_xdp_xmit, 2371 .ndo_xsk_wakeup = tsnep_netdev_xsk_wakeup, 2372 }; 2373 2374 static int tsnep_mac_init(struct tsnep_adapter *adapter) 2375 { 2376 int retval; 2377 2378 /* initialize RX filtering, at least configured MAC address and 2379 * broadcast are not filtered 2380 */ 2381 iowrite16(0, adapter->addr + TSNEP_RX_FILTER); 2382 2383 /* try to get MAC address in the following order: 2384 * - device tree 2385 * - valid MAC address already set 2386 * - MAC address register if valid 2387 * - random MAC address 2388 */ 2389 retval = of_get_mac_address(adapter->pdev->dev.of_node, 2390 adapter->mac_address); 2391 if (retval == -EPROBE_DEFER) 2392 return retval; 2393 if (retval && !is_valid_ether_addr(adapter->mac_address)) { 2394 *(u32 *)adapter->mac_address = 2395 ioread32(adapter->addr + TSNEP_MAC_ADDRESS_LOW); 2396 *(u16 *)(adapter->mac_address + sizeof(u32)) = 2397 ioread16(adapter->addr + TSNEP_MAC_ADDRESS_HIGH); 2398 if (!is_valid_ether_addr(adapter->mac_address)) 2399 eth_random_addr(adapter->mac_address); 2400 } 2401 2402 tsnep_mac_set_address(adapter, adapter->mac_address); 2403 eth_hw_addr_set(adapter->netdev, adapter->mac_address); 2404 2405 return 0; 2406 } 2407 2408 static int tsnep_mdio_init(struct tsnep_adapter *adapter) 2409 { 2410 struct device_node *np = adapter->pdev->dev.of_node; 2411 int retval; 2412 2413 if (np) { 2414 np = of_get_child_by_name(np, "mdio"); 2415 if (!np) 2416 return 0; 2417 2418 adapter->suppress_preamble = 2419 of_property_read_bool(np, "suppress-preamble"); 2420 } 2421 2422 adapter->mdiobus = devm_mdiobus_alloc(&adapter->pdev->dev); 2423 if (!adapter->mdiobus) { 2424 retval = -ENOMEM; 2425 2426 goto out; 2427 } 2428 2429 adapter->mdiobus->priv = (void *)adapter; 2430 adapter->mdiobus->parent = &adapter->pdev->dev; 2431 adapter->mdiobus->read = tsnep_mdiobus_read; 2432 adapter->mdiobus->write = tsnep_mdiobus_write; 2433 adapter->mdiobus->name = TSNEP "-mdiobus"; 2434 snprintf(adapter->mdiobus->id, MII_BUS_ID_SIZE, "%s", 2435 adapter->pdev->name); 2436 2437 /* do not scan broadcast address */ 2438 adapter->mdiobus->phy_mask = 0x0000001; 2439 2440 retval = of_mdiobus_register(adapter->mdiobus, np); 2441 2442 out: 2443 of_node_put(np); 2444 2445 return retval; 2446 } 2447 2448 static int tsnep_phy_init(struct tsnep_adapter *adapter) 2449 { 2450 struct device_node *phy_node; 2451 int retval; 2452 2453 retval = of_get_phy_mode(adapter->pdev->dev.of_node, 2454 &adapter->phy_mode); 2455 if (retval) 2456 adapter->phy_mode = PHY_INTERFACE_MODE_GMII; 2457 2458 phy_node = of_parse_phandle(adapter->pdev->dev.of_node, "phy-handle", 2459 0); 2460 adapter->phydev = of_phy_find_device(phy_node); 2461 of_node_put(phy_node); 2462 if (!adapter->phydev && adapter->mdiobus) 2463 adapter->phydev = phy_find_first(adapter->mdiobus); 2464 if (!adapter->phydev) 2465 return -EIO; 2466 2467 return 0; 2468 } 2469 2470 static int tsnep_queue_init(struct tsnep_adapter *adapter, int queue_count) 2471 { 2472 u32 irq_mask = ECM_INT_TX_0 | ECM_INT_RX_0; 2473 char name[8]; 2474 int i; 2475 int retval; 2476 2477 /* one TX/RX queue pair for netdev is mandatory */ 2478 if (platform_irq_count(adapter->pdev) == 1) 2479 retval = platform_get_irq(adapter->pdev, 0); 2480 else 2481 retval = platform_get_irq_byname(adapter->pdev, "mac"); 2482 if (retval < 0) 2483 return retval; 2484 adapter->num_tx_queues = 1; 2485 adapter->num_rx_queues = 1; 2486 adapter->num_queues = 1; 2487 adapter->queue[0].adapter = adapter; 2488 adapter->queue[0].irq = retval; 2489 adapter->queue[0].tx = &adapter->tx[0]; 2490 adapter->queue[0].tx->adapter = adapter; 2491 adapter->queue[0].tx->addr = adapter->addr + TSNEP_QUEUE(0); 2492 adapter->queue[0].tx->queue_index = 0; 2493 adapter->queue[0].rx = &adapter->rx[0]; 2494 adapter->queue[0].rx->adapter = adapter; 2495 adapter->queue[0].rx->addr = adapter->addr + TSNEP_QUEUE(0); 2496 adapter->queue[0].rx->queue_index = 0; 2497 adapter->queue[0].irq_mask = irq_mask; 2498 adapter->queue[0].irq_delay_addr = adapter->addr + ECM_INT_DELAY; 2499 retval = tsnep_set_irq_coalesce(&adapter->queue[0], 2500 TSNEP_COALESCE_USECS_DEFAULT); 2501 if (retval < 0) 2502 return retval; 2503 2504 adapter->netdev->irq = adapter->queue[0].irq; 2505 2506 /* add additional TX/RX queue pairs only if dedicated interrupt is 2507 * available 2508 */ 2509 for (i = 1; i < queue_count; i++) { 2510 sprintf(name, "txrx-%d", i); 2511 retval = platform_get_irq_byname_optional(adapter->pdev, name); 2512 if (retval < 0) 2513 break; 2514 2515 adapter->num_tx_queues++; 2516 adapter->num_rx_queues++; 2517 adapter->num_queues++; 2518 adapter->queue[i].adapter = adapter; 2519 adapter->queue[i].irq = retval; 2520 adapter->queue[i].tx = &adapter->tx[i]; 2521 adapter->queue[i].tx->adapter = adapter; 2522 adapter->queue[i].tx->addr = adapter->addr + TSNEP_QUEUE(i); 2523 adapter->queue[i].tx->queue_index = i; 2524 adapter->queue[i].rx = &adapter->rx[i]; 2525 adapter->queue[i].rx->adapter = adapter; 2526 adapter->queue[i].rx->addr = adapter->addr + TSNEP_QUEUE(i); 2527 adapter->queue[i].rx->queue_index = i; 2528 adapter->queue[i].irq_mask = 2529 irq_mask << (ECM_INT_TXRX_SHIFT * i); 2530 adapter->queue[i].irq_delay_addr = 2531 adapter->addr + ECM_INT_DELAY + ECM_INT_DELAY_OFFSET * i; 2532 retval = tsnep_set_irq_coalesce(&adapter->queue[i], 2533 TSNEP_COALESCE_USECS_DEFAULT); 2534 if (retval < 0) 2535 return retval; 2536 } 2537 2538 return 0; 2539 } 2540 2541 static int tsnep_probe(struct platform_device *pdev) 2542 { 2543 struct tsnep_adapter *adapter; 2544 struct net_device *netdev; 2545 struct resource *io; 2546 u32 type; 2547 int revision; 2548 int version; 2549 int queue_count; 2550 int retval; 2551 2552 netdev = devm_alloc_etherdev_mqs(&pdev->dev, 2553 sizeof(struct tsnep_adapter), 2554 TSNEP_MAX_QUEUES, TSNEP_MAX_QUEUES); 2555 if (!netdev) 2556 return -ENODEV; 2557 SET_NETDEV_DEV(netdev, &pdev->dev); 2558 adapter = netdev_priv(netdev); 2559 platform_set_drvdata(pdev, adapter); 2560 adapter->pdev = pdev; 2561 adapter->dmadev = &pdev->dev; 2562 adapter->netdev = netdev; 2563 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE | 2564 NETIF_MSG_LINK | NETIF_MSG_IFUP | 2565 NETIF_MSG_IFDOWN | NETIF_MSG_TX_QUEUED; 2566 2567 netdev->min_mtu = ETH_MIN_MTU; 2568 netdev->max_mtu = TSNEP_MAX_FRAME_SIZE; 2569 2570 mutex_init(&adapter->gate_control_lock); 2571 mutex_init(&adapter->rxnfc_lock); 2572 INIT_LIST_HEAD(&adapter->rxnfc_rules); 2573 2574 adapter->addr = devm_platform_get_and_ioremap_resource(pdev, 0, &io); 2575 if (IS_ERR(adapter->addr)) 2576 return PTR_ERR(adapter->addr); 2577 netdev->mem_start = io->start; 2578 netdev->mem_end = io->end; 2579 2580 type = ioread32(adapter->addr + ECM_TYPE); 2581 revision = (type & ECM_REVISION_MASK) >> ECM_REVISION_SHIFT; 2582 version = (type & ECM_VERSION_MASK) >> ECM_VERSION_SHIFT; 2583 queue_count = (type & ECM_QUEUE_COUNT_MASK) >> ECM_QUEUE_COUNT_SHIFT; 2584 adapter->gate_control = type & ECM_GATE_CONTROL; 2585 adapter->rxnfc_max = TSNEP_RX_ASSIGN_ETHER_TYPE_COUNT; 2586 2587 tsnep_disable_irq(adapter, ECM_INT_ALL); 2588 2589 retval = tsnep_queue_init(adapter, queue_count); 2590 if (retval) 2591 return retval; 2592 2593 retval = dma_set_mask_and_coherent(&adapter->pdev->dev, 2594 DMA_BIT_MASK(64)); 2595 if (retval) { 2596 dev_err(&adapter->pdev->dev, "no usable DMA configuration.\n"); 2597 return retval; 2598 } 2599 2600 retval = tsnep_mac_init(adapter); 2601 if (retval) 2602 return retval; 2603 2604 retval = tsnep_mdio_init(adapter); 2605 if (retval) 2606 goto mdio_init_failed; 2607 2608 retval = tsnep_phy_init(adapter); 2609 if (retval) 2610 goto phy_init_failed; 2611 2612 retval = tsnep_ptp_init(adapter); 2613 if (retval) 2614 goto ptp_init_failed; 2615 2616 retval = tsnep_tc_init(adapter); 2617 if (retval) 2618 goto tc_init_failed; 2619 2620 retval = tsnep_rxnfc_init(adapter); 2621 if (retval) 2622 goto rxnfc_init_failed; 2623 2624 netdev->netdev_ops = &tsnep_netdev_ops; 2625 netdev->ethtool_ops = &tsnep_ethtool_ops; 2626 netdev->features = NETIF_F_SG; 2627 netdev->hw_features = netdev->features | NETIF_F_LOOPBACK; 2628 2629 netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | 2630 NETDEV_XDP_ACT_NDO_XMIT | 2631 NETDEV_XDP_ACT_NDO_XMIT_SG | 2632 NETDEV_XDP_ACT_XSK_ZEROCOPY; 2633 2634 /* carrier off reporting is important to ethtool even BEFORE open */ 2635 netif_carrier_off(netdev); 2636 2637 retval = register_netdev(netdev); 2638 if (retval) 2639 goto register_failed; 2640 2641 dev_info(&adapter->pdev->dev, "device version %d.%02d\n", version, 2642 revision); 2643 if (adapter->gate_control) 2644 dev_info(&adapter->pdev->dev, "gate control detected\n"); 2645 2646 return 0; 2647 2648 register_failed: 2649 tsnep_rxnfc_cleanup(adapter); 2650 rxnfc_init_failed: 2651 tsnep_tc_cleanup(adapter); 2652 tc_init_failed: 2653 tsnep_ptp_cleanup(adapter); 2654 ptp_init_failed: 2655 phy_init_failed: 2656 if (adapter->mdiobus) 2657 mdiobus_unregister(adapter->mdiobus); 2658 mdio_init_failed: 2659 return retval; 2660 } 2661 2662 static void tsnep_remove(struct platform_device *pdev) 2663 { 2664 struct tsnep_adapter *adapter = platform_get_drvdata(pdev); 2665 2666 unregister_netdev(adapter->netdev); 2667 2668 tsnep_rxnfc_cleanup(adapter); 2669 2670 tsnep_tc_cleanup(adapter); 2671 2672 tsnep_ptp_cleanup(adapter); 2673 2674 if (adapter->mdiobus) 2675 mdiobus_unregister(adapter->mdiobus); 2676 2677 tsnep_disable_irq(adapter, ECM_INT_ALL); 2678 } 2679 2680 static const struct of_device_id tsnep_of_match[] = { 2681 { .compatible = "engleder,tsnep", }, 2682 { }, 2683 }; 2684 MODULE_DEVICE_TABLE(of, tsnep_of_match); 2685 2686 static struct platform_driver tsnep_driver = { 2687 .driver = { 2688 .name = TSNEP, 2689 .of_match_table = tsnep_of_match, 2690 }, 2691 .probe = tsnep_probe, 2692 .remove_new = tsnep_remove, 2693 }; 2694 module_platform_driver(tsnep_driver); 2695 2696 MODULE_AUTHOR("Gerhard Engleder <gerhard@engleder-embedded.com>"); 2697 MODULE_DESCRIPTION("TSN endpoint Ethernet MAC driver"); 2698 MODULE_LICENSE("GPL"); 2699