1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 2 /* Copyright 2017-2019 NXP */ 3 4 #include "enetc.h" 5 #include <linux/bpf_trace.h> 6 #include <linux/tcp.h> 7 #include <linux/udp.h> 8 #include <linux/vmalloc.h> 9 #include <linux/ptp_classify.h> 10 #include <net/ip6_checksum.h> 11 #include <net/pkt_sched.h> 12 #include <net/tso.h> 13 14 u32 enetc_port_mac_rd(struct enetc_si *si, u32 reg) 15 { 16 return enetc_port_rd(&si->hw, reg); 17 } 18 EXPORT_SYMBOL_GPL(enetc_port_mac_rd); 19 20 void enetc_port_mac_wr(struct enetc_si *si, u32 reg, u32 val) 21 { 22 enetc_port_wr(&si->hw, reg, val); 23 if (si->hw_features & ENETC_SI_F_QBU) 24 enetc_port_wr(&si->hw, reg + ENETC_PMAC_OFFSET, val); 25 } 26 EXPORT_SYMBOL_GPL(enetc_port_mac_wr); 27 28 static int enetc_num_stack_tx_queues(struct enetc_ndev_priv *priv) 29 { 30 int num_tx_rings = priv->num_tx_rings; 31 32 if (priv->xdp_prog) 33 return num_tx_rings - num_possible_cpus(); 34 35 return num_tx_rings; 36 } 37 38 static struct enetc_bdr *enetc_rx_ring_from_xdp_tx_ring(struct enetc_ndev_priv *priv, 39 struct enetc_bdr *tx_ring) 40 { 41 int index = &priv->tx_ring[tx_ring->index] - priv->xdp_tx_ring; 42 43 return priv->rx_ring[index]; 44 } 45 46 static struct sk_buff *enetc_tx_swbd_get_skb(struct enetc_tx_swbd *tx_swbd) 47 { 48 if (tx_swbd->is_xdp_tx || tx_swbd->is_xdp_redirect) 49 return NULL; 50 51 return tx_swbd->skb; 52 } 53 54 static struct xdp_frame * 55 enetc_tx_swbd_get_xdp_frame(struct enetc_tx_swbd *tx_swbd) 56 { 57 if (tx_swbd->is_xdp_redirect) 58 return tx_swbd->xdp_frame; 59 60 return NULL; 61 } 62 63 static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring, 64 struct enetc_tx_swbd *tx_swbd) 65 { 66 /* For XDP_TX, pages come from RX, whereas for the other contexts where 67 * we have is_dma_page_set, those come from skb_frag_dma_map. We need 68 * to match the DMA mapping length, so we need to differentiate those. 69 */ 70 if (tx_swbd->is_dma_page) 71 dma_unmap_page(tx_ring->dev, tx_swbd->dma, 72 tx_swbd->is_xdp_tx ? PAGE_SIZE : tx_swbd->len, 73 tx_swbd->dir); 74 else 75 dma_unmap_single(tx_ring->dev, tx_swbd->dma, 76 tx_swbd->len, tx_swbd->dir); 77 tx_swbd->dma = 0; 78 } 79 80 static void enetc_free_tx_frame(struct enetc_bdr *tx_ring, 81 struct enetc_tx_swbd *tx_swbd) 82 { 83 struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd); 84 struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd); 85 86 if (tx_swbd->dma) 87 enetc_unmap_tx_buff(tx_ring, tx_swbd); 88 89 if (xdp_frame) { 90 xdp_return_frame(tx_swbd->xdp_frame); 91 tx_swbd->xdp_frame = NULL; 92 } else if (skb) { 93 dev_kfree_skb_any(skb); 94 tx_swbd->skb = NULL; 95 } 96 } 97 98 /* Let H/W know BD ring has been updated */ 99 static void enetc_update_tx_ring_tail(struct enetc_bdr *tx_ring) 100 { 101 /* includes wmb() */ 102 enetc_wr_reg_hot(tx_ring->tpir, tx_ring->next_to_use); 103 } 104 105 static int enetc_ptp_parse(struct sk_buff *skb, u8 *udp, 106 u8 *msgtype, u8 *twostep, 107 u16 *correction_offset, u16 *body_offset) 108 { 109 unsigned int ptp_class; 110 struct ptp_header *hdr; 111 unsigned int type; 112 u8 *base; 113 114 ptp_class = ptp_classify_raw(skb); 115 if (ptp_class == PTP_CLASS_NONE) 116 return -EINVAL; 117 118 hdr = ptp_parse_header(skb, ptp_class); 119 if (!hdr) 120 return -EINVAL; 121 122 type = ptp_class & PTP_CLASS_PMASK; 123 if (type == PTP_CLASS_IPV4 || type == PTP_CLASS_IPV6) 124 *udp = 1; 125 else 126 *udp = 0; 127 128 *msgtype = ptp_get_msgtype(hdr, ptp_class); 129 *twostep = hdr->flag_field[0] & 0x2; 130 131 base = skb_mac_header(skb); 132 *correction_offset = (u8 *)&hdr->correction - base; 133 *body_offset = (u8 *)hdr + sizeof(struct ptp_header) - base; 134 135 return 0; 136 } 137 138 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb) 139 { 140 bool do_vlan, do_onestep_tstamp = false, do_twostep_tstamp = false; 141 struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev); 142 struct enetc_hw *hw = &priv->si->hw; 143 struct enetc_tx_swbd *tx_swbd; 144 int len = skb_headlen(skb); 145 union enetc_tx_bd temp_bd; 146 u8 msgtype, twostep, udp; 147 union enetc_tx_bd *txbd; 148 u16 offset1, offset2; 149 int i, count = 0; 150 skb_frag_t *frag; 151 unsigned int f; 152 dma_addr_t dma; 153 u8 flags = 0; 154 155 i = tx_ring->next_to_use; 156 txbd = ENETC_TXBD(*tx_ring, i); 157 prefetchw(txbd); 158 159 dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE); 160 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) 161 goto dma_err; 162 163 temp_bd.addr = cpu_to_le64(dma); 164 temp_bd.buf_len = cpu_to_le16(len); 165 temp_bd.lstatus = 0; 166 167 tx_swbd = &tx_ring->tx_swbd[i]; 168 tx_swbd->dma = dma; 169 tx_swbd->len = len; 170 tx_swbd->is_dma_page = 0; 171 tx_swbd->dir = DMA_TO_DEVICE; 172 count++; 173 174 do_vlan = skb_vlan_tag_present(skb); 175 if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) { 176 if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep, &offset1, 177 &offset2) || 178 msgtype != PTP_MSGTYPE_SYNC || twostep) 179 WARN_ONCE(1, "Bad packet for one-step timestamping\n"); 180 else 181 do_onestep_tstamp = true; 182 } else if (skb->cb[0] & ENETC_F_TX_TSTAMP) { 183 do_twostep_tstamp = true; 184 } 185 186 tx_swbd->do_twostep_tstamp = do_twostep_tstamp; 187 tx_swbd->qbv_en = !!(priv->active_offloads & ENETC_F_QBV); 188 tx_swbd->check_wb = tx_swbd->do_twostep_tstamp || tx_swbd->qbv_en; 189 190 if (do_vlan || do_onestep_tstamp || do_twostep_tstamp) 191 flags |= ENETC_TXBD_FLAGS_EX; 192 193 if (tx_ring->tsd_enable) 194 flags |= ENETC_TXBD_FLAGS_TSE | ENETC_TXBD_FLAGS_TXSTART; 195 196 /* first BD needs frm_len and offload flags set */ 197 temp_bd.frm_len = cpu_to_le16(skb->len); 198 temp_bd.flags = flags; 199 200 if (flags & ENETC_TXBD_FLAGS_TSE) 201 temp_bd.txstart = enetc_txbd_set_tx_start(skb->skb_mstamp_ns, 202 flags); 203 204 if (flags & ENETC_TXBD_FLAGS_EX) { 205 u8 e_flags = 0; 206 *txbd = temp_bd; 207 enetc_clear_tx_bd(&temp_bd); 208 209 /* add extension BD for VLAN and/or timestamping */ 210 flags = 0; 211 tx_swbd++; 212 txbd++; 213 i++; 214 if (unlikely(i == tx_ring->bd_count)) { 215 i = 0; 216 tx_swbd = tx_ring->tx_swbd; 217 txbd = ENETC_TXBD(*tx_ring, 0); 218 } 219 prefetchw(txbd); 220 221 if (do_vlan) { 222 temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb)); 223 temp_bd.ext.tpid = 0; /* < C-TAG */ 224 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS; 225 } 226 227 if (do_onestep_tstamp) { 228 u32 lo, hi, val; 229 u64 sec, nsec; 230 u8 *data; 231 232 lo = enetc_rd_hot(hw, ENETC_SICTR0); 233 hi = enetc_rd_hot(hw, ENETC_SICTR1); 234 sec = (u64)hi << 32 | lo; 235 nsec = do_div(sec, 1000000000); 236 237 /* Configure extension BD */ 238 temp_bd.ext.tstamp = cpu_to_le32(lo & 0x3fffffff); 239 e_flags |= ENETC_TXBD_E_FLAGS_ONE_STEP_PTP; 240 241 /* Update originTimestamp field of Sync packet 242 * - 48 bits seconds field 243 * - 32 bits nanseconds field 244 */ 245 data = skb_mac_header(skb); 246 *(__be16 *)(data + offset2) = 247 htons((sec >> 32) & 0xffff); 248 *(__be32 *)(data + offset2 + 2) = 249 htonl(sec & 0xffffffff); 250 *(__be32 *)(data + offset2 + 6) = htonl(nsec); 251 252 /* Configure single-step register */ 253 val = ENETC_PM0_SINGLE_STEP_EN; 254 val |= ENETC_SET_SINGLE_STEP_OFFSET(offset1); 255 if (udp) 256 val |= ENETC_PM0_SINGLE_STEP_CH; 257 258 enetc_port_mac_wr(priv->si, ENETC_PM0_SINGLE_STEP, 259 val); 260 } else if (do_twostep_tstamp) { 261 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 262 e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP; 263 } 264 265 temp_bd.ext.e_flags = e_flags; 266 count++; 267 } 268 269 frag = &skb_shinfo(skb)->frags[0]; 270 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) { 271 len = skb_frag_size(frag); 272 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len, 273 DMA_TO_DEVICE); 274 if (dma_mapping_error(tx_ring->dev, dma)) 275 goto dma_err; 276 277 *txbd = temp_bd; 278 enetc_clear_tx_bd(&temp_bd); 279 280 flags = 0; 281 tx_swbd++; 282 txbd++; 283 i++; 284 if (unlikely(i == tx_ring->bd_count)) { 285 i = 0; 286 tx_swbd = tx_ring->tx_swbd; 287 txbd = ENETC_TXBD(*tx_ring, 0); 288 } 289 prefetchw(txbd); 290 291 temp_bd.addr = cpu_to_le64(dma); 292 temp_bd.buf_len = cpu_to_le16(len); 293 294 tx_swbd->dma = dma; 295 tx_swbd->len = len; 296 tx_swbd->is_dma_page = 1; 297 tx_swbd->dir = DMA_TO_DEVICE; 298 count++; 299 } 300 301 /* last BD needs 'F' bit set */ 302 flags |= ENETC_TXBD_FLAGS_F; 303 temp_bd.flags = flags; 304 *txbd = temp_bd; 305 306 tx_ring->tx_swbd[i].is_eof = true; 307 tx_ring->tx_swbd[i].skb = skb; 308 309 enetc_bdr_idx_inc(tx_ring, &i); 310 tx_ring->next_to_use = i; 311 312 skb_tx_timestamp(skb); 313 314 enetc_update_tx_ring_tail(tx_ring); 315 316 return count; 317 318 dma_err: 319 dev_err(tx_ring->dev, "DMA map error"); 320 321 do { 322 tx_swbd = &tx_ring->tx_swbd[i]; 323 enetc_free_tx_frame(tx_ring, tx_swbd); 324 if (i == 0) 325 i = tx_ring->bd_count; 326 i--; 327 } while (count--); 328 329 return 0; 330 } 331 332 static void enetc_map_tx_tso_hdr(struct enetc_bdr *tx_ring, struct sk_buff *skb, 333 struct enetc_tx_swbd *tx_swbd, 334 union enetc_tx_bd *txbd, int *i, int hdr_len, 335 int data_len) 336 { 337 union enetc_tx_bd txbd_tmp; 338 u8 flags = 0, e_flags = 0; 339 dma_addr_t addr; 340 341 enetc_clear_tx_bd(&txbd_tmp); 342 addr = tx_ring->tso_headers_dma + *i * TSO_HEADER_SIZE; 343 344 if (skb_vlan_tag_present(skb)) 345 flags |= ENETC_TXBD_FLAGS_EX; 346 347 txbd_tmp.addr = cpu_to_le64(addr); 348 txbd_tmp.buf_len = cpu_to_le16(hdr_len); 349 350 /* first BD needs frm_len and offload flags set */ 351 txbd_tmp.frm_len = cpu_to_le16(hdr_len + data_len); 352 txbd_tmp.flags = flags; 353 354 /* For the TSO header we do not set the dma address since we do not 355 * want it unmapped when we do cleanup. We still set len so that we 356 * count the bytes sent. 357 */ 358 tx_swbd->len = hdr_len; 359 tx_swbd->do_twostep_tstamp = false; 360 tx_swbd->check_wb = false; 361 362 /* Actually write the header in the BD */ 363 *txbd = txbd_tmp; 364 365 /* Add extension BD for VLAN */ 366 if (flags & ENETC_TXBD_FLAGS_EX) { 367 /* Get the next BD */ 368 enetc_bdr_idx_inc(tx_ring, i); 369 txbd = ENETC_TXBD(*tx_ring, *i); 370 tx_swbd = &tx_ring->tx_swbd[*i]; 371 prefetchw(txbd); 372 373 /* Setup the VLAN fields */ 374 enetc_clear_tx_bd(&txbd_tmp); 375 txbd_tmp.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb)); 376 txbd_tmp.ext.tpid = 0; /* < C-TAG */ 377 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS; 378 379 /* Write the BD */ 380 txbd_tmp.ext.e_flags = e_flags; 381 *txbd = txbd_tmp; 382 } 383 } 384 385 static int enetc_map_tx_tso_data(struct enetc_bdr *tx_ring, struct sk_buff *skb, 386 struct enetc_tx_swbd *tx_swbd, 387 union enetc_tx_bd *txbd, char *data, 388 int size, bool last_bd) 389 { 390 union enetc_tx_bd txbd_tmp; 391 dma_addr_t addr; 392 u8 flags = 0; 393 394 enetc_clear_tx_bd(&txbd_tmp); 395 396 addr = dma_map_single(tx_ring->dev, data, size, DMA_TO_DEVICE); 397 if (unlikely(dma_mapping_error(tx_ring->dev, addr))) { 398 netdev_err(tx_ring->ndev, "DMA map error\n"); 399 return -ENOMEM; 400 } 401 402 if (last_bd) { 403 flags |= ENETC_TXBD_FLAGS_F; 404 tx_swbd->is_eof = 1; 405 } 406 407 txbd_tmp.addr = cpu_to_le64(addr); 408 txbd_tmp.buf_len = cpu_to_le16(size); 409 txbd_tmp.flags = flags; 410 411 tx_swbd->dma = addr; 412 tx_swbd->len = size; 413 tx_swbd->dir = DMA_TO_DEVICE; 414 415 *txbd = txbd_tmp; 416 417 return 0; 418 } 419 420 static __wsum enetc_tso_hdr_csum(struct tso_t *tso, struct sk_buff *skb, 421 char *hdr, int hdr_len, int *l4_hdr_len) 422 { 423 char *l4_hdr = hdr + skb_transport_offset(skb); 424 int mac_hdr_len = skb_network_offset(skb); 425 426 if (tso->tlen != sizeof(struct udphdr)) { 427 struct tcphdr *tcph = (struct tcphdr *)(l4_hdr); 428 429 tcph->check = 0; 430 } else { 431 struct udphdr *udph = (struct udphdr *)(l4_hdr); 432 433 udph->check = 0; 434 } 435 436 /* Compute the IP checksum. This is necessary since tso_build_hdr() 437 * already incremented the IP ID field. 438 */ 439 if (!tso->ipv6) { 440 struct iphdr *iph = (void *)(hdr + mac_hdr_len); 441 442 iph->check = 0; 443 iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl); 444 } 445 446 /* Compute the checksum over the L4 header. */ 447 *l4_hdr_len = hdr_len - skb_transport_offset(skb); 448 return csum_partial(l4_hdr, *l4_hdr_len, 0); 449 } 450 451 static void enetc_tso_complete_csum(struct enetc_bdr *tx_ring, struct tso_t *tso, 452 struct sk_buff *skb, char *hdr, int len, 453 __wsum sum) 454 { 455 char *l4_hdr = hdr + skb_transport_offset(skb); 456 __sum16 csum_final; 457 458 /* Complete the L4 checksum by appending the pseudo-header to the 459 * already computed checksum. 460 */ 461 if (!tso->ipv6) 462 csum_final = csum_tcpudp_magic(ip_hdr(skb)->saddr, 463 ip_hdr(skb)->daddr, 464 len, ip_hdr(skb)->protocol, sum); 465 else 466 csum_final = csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 467 &ipv6_hdr(skb)->daddr, 468 len, ipv6_hdr(skb)->nexthdr, sum); 469 470 if (tso->tlen != sizeof(struct udphdr)) { 471 struct tcphdr *tcph = (struct tcphdr *)(l4_hdr); 472 473 tcph->check = csum_final; 474 } else { 475 struct udphdr *udph = (struct udphdr *)(l4_hdr); 476 477 udph->check = csum_final; 478 } 479 } 480 481 static int enetc_map_tx_tso_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb) 482 { 483 int hdr_len, total_len, data_len; 484 struct enetc_tx_swbd *tx_swbd; 485 union enetc_tx_bd *txbd; 486 struct tso_t tso; 487 __wsum csum, csum2; 488 int count = 0, pos; 489 int err, i, bd_data_num; 490 491 /* Initialize the TSO handler, and prepare the first payload */ 492 hdr_len = tso_start(skb, &tso); 493 total_len = skb->len - hdr_len; 494 i = tx_ring->next_to_use; 495 496 while (total_len > 0) { 497 char *hdr; 498 499 /* Get the BD */ 500 txbd = ENETC_TXBD(*tx_ring, i); 501 tx_swbd = &tx_ring->tx_swbd[i]; 502 prefetchw(txbd); 503 504 /* Determine the length of this packet */ 505 data_len = min_t(int, skb_shinfo(skb)->gso_size, total_len); 506 total_len -= data_len; 507 508 /* prepare packet headers: MAC + IP + TCP */ 509 hdr = tx_ring->tso_headers + i * TSO_HEADER_SIZE; 510 tso_build_hdr(skb, hdr, &tso, data_len, total_len == 0); 511 512 /* compute the csum over the L4 header */ 513 csum = enetc_tso_hdr_csum(&tso, skb, hdr, hdr_len, &pos); 514 enetc_map_tx_tso_hdr(tx_ring, skb, tx_swbd, txbd, &i, hdr_len, data_len); 515 bd_data_num = 0; 516 count++; 517 518 while (data_len > 0) { 519 int size; 520 521 size = min_t(int, tso.size, data_len); 522 523 /* Advance the index in the BDR */ 524 enetc_bdr_idx_inc(tx_ring, &i); 525 txbd = ENETC_TXBD(*tx_ring, i); 526 tx_swbd = &tx_ring->tx_swbd[i]; 527 prefetchw(txbd); 528 529 /* Compute the checksum over this segment of data and 530 * add it to the csum already computed (over the L4 531 * header and possible other data segments). 532 */ 533 csum2 = csum_partial(tso.data, size, 0); 534 csum = csum_block_add(csum, csum2, pos); 535 pos += size; 536 537 err = enetc_map_tx_tso_data(tx_ring, skb, tx_swbd, txbd, 538 tso.data, size, 539 size == data_len); 540 if (err) 541 goto err_map_data; 542 543 data_len -= size; 544 count++; 545 bd_data_num++; 546 tso_build_data(skb, &tso, size); 547 548 if (unlikely(bd_data_num >= ENETC_MAX_SKB_FRAGS && data_len)) 549 goto err_chained_bd; 550 } 551 552 enetc_tso_complete_csum(tx_ring, &tso, skb, hdr, pos, csum); 553 554 if (total_len == 0) 555 tx_swbd->skb = skb; 556 557 /* Go to the next BD */ 558 enetc_bdr_idx_inc(tx_ring, &i); 559 } 560 561 tx_ring->next_to_use = i; 562 enetc_update_tx_ring_tail(tx_ring); 563 564 return count; 565 566 err_map_data: 567 dev_err(tx_ring->dev, "DMA map error"); 568 569 err_chained_bd: 570 do { 571 tx_swbd = &tx_ring->tx_swbd[i]; 572 enetc_free_tx_frame(tx_ring, tx_swbd); 573 if (i == 0) 574 i = tx_ring->bd_count; 575 i--; 576 } while (count--); 577 578 return 0; 579 } 580 581 static netdev_tx_t enetc_start_xmit(struct sk_buff *skb, 582 struct net_device *ndev) 583 { 584 struct enetc_ndev_priv *priv = netdev_priv(ndev); 585 struct enetc_bdr *tx_ring; 586 int count, err; 587 588 /* Queue one-step Sync packet if already locked */ 589 if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) { 590 if (test_and_set_bit_lock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, 591 &priv->flags)) { 592 skb_queue_tail(&priv->tx_skbs, skb); 593 return NETDEV_TX_OK; 594 } 595 } 596 597 tx_ring = priv->tx_ring[skb->queue_mapping]; 598 599 if (skb_is_gso(skb)) { 600 if (enetc_bd_unused(tx_ring) < tso_count_descs(skb)) { 601 netif_stop_subqueue(ndev, tx_ring->index); 602 return NETDEV_TX_BUSY; 603 } 604 605 enetc_lock_mdio(); 606 count = enetc_map_tx_tso_buffs(tx_ring, skb); 607 enetc_unlock_mdio(); 608 } else { 609 if (unlikely(skb_shinfo(skb)->nr_frags > ENETC_MAX_SKB_FRAGS)) 610 if (unlikely(skb_linearize(skb))) 611 goto drop_packet_err; 612 613 count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */ 614 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) { 615 netif_stop_subqueue(ndev, tx_ring->index); 616 return NETDEV_TX_BUSY; 617 } 618 619 if (skb->ip_summed == CHECKSUM_PARTIAL) { 620 err = skb_checksum_help(skb); 621 if (err) 622 goto drop_packet_err; 623 } 624 enetc_lock_mdio(); 625 count = enetc_map_tx_buffs(tx_ring, skb); 626 enetc_unlock_mdio(); 627 } 628 629 if (unlikely(!count)) 630 goto drop_packet_err; 631 632 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED) 633 netif_stop_subqueue(ndev, tx_ring->index); 634 635 return NETDEV_TX_OK; 636 637 drop_packet_err: 638 dev_kfree_skb_any(skb); 639 return NETDEV_TX_OK; 640 } 641 642 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev) 643 { 644 struct enetc_ndev_priv *priv = netdev_priv(ndev); 645 u8 udp, msgtype, twostep; 646 u16 offset1, offset2; 647 648 /* Mark tx timestamp type on skb->cb[0] if requires */ 649 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 650 (priv->active_offloads & ENETC_F_TX_TSTAMP_MASK)) { 651 skb->cb[0] = priv->active_offloads & ENETC_F_TX_TSTAMP_MASK; 652 } else { 653 skb->cb[0] = 0; 654 } 655 656 /* Fall back to two-step timestamp if not one-step Sync packet */ 657 if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) { 658 if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep, 659 &offset1, &offset2) || 660 msgtype != PTP_MSGTYPE_SYNC || twostep != 0) 661 skb->cb[0] = ENETC_F_TX_TSTAMP; 662 } 663 664 return enetc_start_xmit(skb, ndev); 665 } 666 EXPORT_SYMBOL_GPL(enetc_xmit); 667 668 static irqreturn_t enetc_msix(int irq, void *data) 669 { 670 struct enetc_int_vector *v = data; 671 int i; 672 673 enetc_lock_mdio(); 674 675 /* disable interrupts */ 676 enetc_wr_reg_hot(v->rbier, 0); 677 enetc_wr_reg_hot(v->ricr1, v->rx_ictt); 678 679 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS) 680 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 0); 681 682 enetc_unlock_mdio(); 683 684 napi_schedule(&v->napi); 685 686 return IRQ_HANDLED; 687 } 688 689 static void enetc_rx_dim_work(struct work_struct *w) 690 { 691 struct dim *dim = container_of(w, struct dim, work); 692 struct dim_cq_moder moder = 693 net_dim_get_rx_moderation(dim->mode, dim->profile_ix); 694 struct enetc_int_vector *v = 695 container_of(dim, struct enetc_int_vector, rx_dim); 696 697 v->rx_ictt = enetc_usecs_to_cycles(moder.usec); 698 dim->state = DIM_START_MEASURE; 699 } 700 701 static void enetc_rx_net_dim(struct enetc_int_vector *v) 702 { 703 struct dim_sample dim_sample = {}; 704 705 v->comp_cnt++; 706 707 if (!v->rx_napi_work) 708 return; 709 710 dim_update_sample(v->comp_cnt, 711 v->rx_ring.stats.packets, 712 v->rx_ring.stats.bytes, 713 &dim_sample); 714 net_dim(&v->rx_dim, dim_sample); 715 } 716 717 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci) 718 { 719 int pi = enetc_rd_reg_hot(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK; 720 721 return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi; 722 } 723 724 static bool enetc_page_reusable(struct page *page) 725 { 726 return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1); 727 } 728 729 static void enetc_reuse_page(struct enetc_bdr *rx_ring, 730 struct enetc_rx_swbd *old) 731 { 732 struct enetc_rx_swbd *new; 733 734 new = &rx_ring->rx_swbd[rx_ring->next_to_alloc]; 735 736 /* next buf that may reuse a page */ 737 enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc); 738 739 /* copy page reference */ 740 *new = *old; 741 } 742 743 static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd, 744 u64 *tstamp) 745 { 746 u32 lo, hi, tstamp_lo; 747 748 lo = enetc_rd_hot(hw, ENETC_SICTR0); 749 hi = enetc_rd_hot(hw, ENETC_SICTR1); 750 tstamp_lo = le32_to_cpu(txbd->wb.tstamp); 751 if (lo <= tstamp_lo) 752 hi -= 1; 753 *tstamp = (u64)hi << 32 | tstamp_lo; 754 } 755 756 static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp) 757 { 758 struct skb_shared_hwtstamps shhwtstamps; 759 760 if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) { 761 memset(&shhwtstamps, 0, sizeof(shhwtstamps)); 762 shhwtstamps.hwtstamp = ns_to_ktime(tstamp); 763 skb_txtime_consumed(skb); 764 skb_tstamp_tx(skb, &shhwtstamps); 765 } 766 } 767 768 static void enetc_recycle_xdp_tx_buff(struct enetc_bdr *tx_ring, 769 struct enetc_tx_swbd *tx_swbd) 770 { 771 struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev); 772 struct enetc_rx_swbd rx_swbd = { 773 .dma = tx_swbd->dma, 774 .page = tx_swbd->page, 775 .page_offset = tx_swbd->page_offset, 776 .dir = tx_swbd->dir, 777 .len = tx_swbd->len, 778 }; 779 struct enetc_bdr *rx_ring; 780 781 rx_ring = enetc_rx_ring_from_xdp_tx_ring(priv, tx_ring); 782 783 if (likely(enetc_swbd_unused(rx_ring))) { 784 enetc_reuse_page(rx_ring, &rx_swbd); 785 786 /* sync for use by the device */ 787 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd.dma, 788 rx_swbd.page_offset, 789 ENETC_RXB_DMA_SIZE_XDP, 790 rx_swbd.dir); 791 792 rx_ring->stats.recycles++; 793 } else { 794 /* RX ring is already full, we need to unmap and free the 795 * page, since there's nothing useful we can do with it. 796 */ 797 rx_ring->stats.recycle_failures++; 798 799 dma_unmap_page(rx_ring->dev, rx_swbd.dma, PAGE_SIZE, 800 rx_swbd.dir); 801 __free_page(rx_swbd.page); 802 } 803 804 rx_ring->xdp.xdp_tx_in_flight--; 805 } 806 807 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget) 808 { 809 int tx_frm_cnt = 0, tx_byte_cnt = 0, tx_win_drop = 0; 810 struct net_device *ndev = tx_ring->ndev; 811 struct enetc_ndev_priv *priv = netdev_priv(ndev); 812 struct enetc_tx_swbd *tx_swbd; 813 int i, bds_to_clean; 814 bool do_twostep_tstamp; 815 u64 tstamp = 0; 816 817 i = tx_ring->next_to_clean; 818 tx_swbd = &tx_ring->tx_swbd[i]; 819 820 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 821 822 do_twostep_tstamp = false; 823 824 while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) { 825 struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd); 826 struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd); 827 bool is_eof = tx_swbd->is_eof; 828 829 if (unlikely(tx_swbd->check_wb)) { 830 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i); 831 832 if (txbd->flags & ENETC_TXBD_FLAGS_W && 833 tx_swbd->do_twostep_tstamp) { 834 enetc_get_tx_tstamp(&priv->si->hw, txbd, 835 &tstamp); 836 do_twostep_tstamp = true; 837 } 838 839 if (tx_swbd->qbv_en && 840 txbd->wb.status & ENETC_TXBD_STATS_WIN) 841 tx_win_drop++; 842 } 843 844 if (tx_swbd->is_xdp_tx) 845 enetc_recycle_xdp_tx_buff(tx_ring, tx_swbd); 846 else if (likely(tx_swbd->dma)) 847 enetc_unmap_tx_buff(tx_ring, tx_swbd); 848 849 if (xdp_frame) { 850 xdp_return_frame(xdp_frame); 851 } else if (skb) { 852 if (unlikely(skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)) { 853 /* Start work to release lock for next one-step 854 * timestamping packet. And send one skb in 855 * tx_skbs queue if has. 856 */ 857 schedule_work(&priv->tx_onestep_tstamp); 858 } else if (unlikely(do_twostep_tstamp)) { 859 enetc_tstamp_tx(skb, tstamp); 860 do_twostep_tstamp = false; 861 } 862 napi_consume_skb(skb, napi_budget); 863 } 864 865 tx_byte_cnt += tx_swbd->len; 866 /* Scrub the swbd here so we don't have to do that 867 * when we reuse it during xmit 868 */ 869 memset(tx_swbd, 0, sizeof(*tx_swbd)); 870 871 bds_to_clean--; 872 tx_swbd++; 873 i++; 874 if (unlikely(i == tx_ring->bd_count)) { 875 i = 0; 876 tx_swbd = tx_ring->tx_swbd; 877 } 878 879 /* BD iteration loop end */ 880 if (is_eof) { 881 tx_frm_cnt++; 882 /* re-arm interrupt source */ 883 enetc_wr_reg_hot(tx_ring->idr, BIT(tx_ring->index) | 884 BIT(16 + tx_ring->index)); 885 } 886 887 if (unlikely(!bds_to_clean)) 888 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 889 } 890 891 tx_ring->next_to_clean = i; 892 tx_ring->stats.packets += tx_frm_cnt; 893 tx_ring->stats.bytes += tx_byte_cnt; 894 tx_ring->stats.win_drop += tx_win_drop; 895 896 if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) && 897 __netif_subqueue_stopped(ndev, tx_ring->index) && 898 (enetc_bd_unused(tx_ring) >= ENETC_TXBDS_MAX_NEEDED))) { 899 netif_wake_subqueue(ndev, tx_ring->index); 900 } 901 902 return tx_frm_cnt != ENETC_DEFAULT_TX_WORK; 903 } 904 905 static bool enetc_new_page(struct enetc_bdr *rx_ring, 906 struct enetc_rx_swbd *rx_swbd) 907 { 908 bool xdp = !!(rx_ring->xdp.prog); 909 struct page *page; 910 dma_addr_t addr; 911 912 page = dev_alloc_page(); 913 if (unlikely(!page)) 914 return false; 915 916 /* For XDP_TX, we forgo dma_unmap -> dma_map */ 917 rx_swbd->dir = xdp ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; 918 919 addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, rx_swbd->dir); 920 if (unlikely(dma_mapping_error(rx_ring->dev, addr))) { 921 __free_page(page); 922 923 return false; 924 } 925 926 rx_swbd->dma = addr; 927 rx_swbd->page = page; 928 rx_swbd->page_offset = rx_ring->buffer_offset; 929 930 return true; 931 } 932 933 static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt) 934 { 935 struct enetc_rx_swbd *rx_swbd; 936 union enetc_rx_bd *rxbd; 937 int i, j; 938 939 i = rx_ring->next_to_use; 940 rx_swbd = &rx_ring->rx_swbd[i]; 941 rxbd = enetc_rxbd(rx_ring, i); 942 943 for (j = 0; j < buff_cnt; j++) { 944 /* try reuse page */ 945 if (unlikely(!rx_swbd->page)) { 946 if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) { 947 rx_ring->stats.rx_alloc_errs++; 948 break; 949 } 950 } 951 952 /* update RxBD */ 953 rxbd->w.addr = cpu_to_le64(rx_swbd->dma + 954 rx_swbd->page_offset); 955 /* clear 'R" as well */ 956 rxbd->r.lstatus = 0; 957 958 enetc_rxbd_next(rx_ring, &rxbd, &i); 959 rx_swbd = &rx_ring->rx_swbd[i]; 960 } 961 962 if (likely(j)) { 963 rx_ring->next_to_alloc = i; /* keep track from page reuse */ 964 rx_ring->next_to_use = i; 965 966 /* update ENETC's consumer index */ 967 enetc_wr_reg_hot(rx_ring->rcir, rx_ring->next_to_use); 968 } 969 970 return j; 971 } 972 973 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 974 static void enetc_get_rx_tstamp(struct net_device *ndev, 975 union enetc_rx_bd *rxbd, 976 struct sk_buff *skb) 977 { 978 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb); 979 struct enetc_ndev_priv *priv = netdev_priv(ndev); 980 struct enetc_hw *hw = &priv->si->hw; 981 u32 lo, hi, tstamp_lo; 982 u64 tstamp; 983 984 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) { 985 lo = enetc_rd_reg_hot(hw->reg + ENETC_SICTR0); 986 hi = enetc_rd_reg_hot(hw->reg + ENETC_SICTR1); 987 rxbd = enetc_rxbd_ext(rxbd); 988 tstamp_lo = le32_to_cpu(rxbd->ext.tstamp); 989 if (lo <= tstamp_lo) 990 hi -= 1; 991 992 tstamp = (u64)hi << 32 | tstamp_lo; 993 memset(shhwtstamps, 0, sizeof(*shhwtstamps)); 994 shhwtstamps->hwtstamp = ns_to_ktime(tstamp); 995 } 996 } 997 #endif 998 999 static void enetc_get_offloads(struct enetc_bdr *rx_ring, 1000 union enetc_rx_bd *rxbd, struct sk_buff *skb) 1001 { 1002 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev); 1003 1004 /* TODO: hashing */ 1005 if (rx_ring->ndev->features & NETIF_F_RXCSUM) { 1006 u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum); 1007 1008 skb->csum = csum_unfold((__force __sum16)~htons(inet_csum)); 1009 skb->ip_summed = CHECKSUM_COMPLETE; 1010 } 1011 1012 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN) { 1013 __be16 tpid = 0; 1014 1015 switch (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TPID) { 1016 case 0: 1017 tpid = htons(ETH_P_8021Q); 1018 break; 1019 case 1: 1020 tpid = htons(ETH_P_8021AD); 1021 break; 1022 case 2: 1023 tpid = htons(enetc_port_rd(&priv->si->hw, 1024 ENETC_PCVLANR1)); 1025 break; 1026 case 3: 1027 tpid = htons(enetc_port_rd(&priv->si->hw, 1028 ENETC_PCVLANR2)); 1029 break; 1030 default: 1031 break; 1032 } 1033 1034 __vlan_hwaccel_put_tag(skb, tpid, le16_to_cpu(rxbd->r.vlan_opt)); 1035 } 1036 1037 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 1038 if (priv->active_offloads & ENETC_F_RX_TSTAMP) 1039 enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb); 1040 #endif 1041 } 1042 1043 /* This gets called during the non-XDP NAPI poll cycle as well as on XDP_PASS, 1044 * so it needs to work with both DMA_FROM_DEVICE as well as DMA_BIDIRECTIONAL 1045 * mapped buffers. 1046 */ 1047 static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring, 1048 int i, u16 size) 1049 { 1050 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 1051 1052 dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma, 1053 rx_swbd->page_offset, 1054 size, rx_swbd->dir); 1055 return rx_swbd; 1056 } 1057 1058 /* Reuse the current page without performing half-page buffer flipping */ 1059 static void enetc_put_rx_buff(struct enetc_bdr *rx_ring, 1060 struct enetc_rx_swbd *rx_swbd) 1061 { 1062 size_t buffer_size = ENETC_RXB_TRUESIZE - rx_ring->buffer_offset; 1063 1064 enetc_reuse_page(rx_ring, rx_swbd); 1065 1066 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma, 1067 rx_swbd->page_offset, 1068 buffer_size, rx_swbd->dir); 1069 1070 rx_swbd->page = NULL; 1071 } 1072 1073 /* Reuse the current page by performing half-page buffer flipping */ 1074 static void enetc_flip_rx_buff(struct enetc_bdr *rx_ring, 1075 struct enetc_rx_swbd *rx_swbd) 1076 { 1077 if (likely(enetc_page_reusable(rx_swbd->page))) { 1078 rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE; 1079 page_ref_inc(rx_swbd->page); 1080 1081 enetc_put_rx_buff(rx_ring, rx_swbd); 1082 } else { 1083 dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE, 1084 rx_swbd->dir); 1085 rx_swbd->page = NULL; 1086 } 1087 } 1088 1089 static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring, 1090 int i, u16 size) 1091 { 1092 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1093 struct sk_buff *skb; 1094 void *ba; 1095 1096 ba = page_address(rx_swbd->page) + rx_swbd->page_offset; 1097 skb = build_skb(ba - rx_ring->buffer_offset, ENETC_RXB_TRUESIZE); 1098 if (unlikely(!skb)) { 1099 rx_ring->stats.rx_alloc_errs++; 1100 return NULL; 1101 } 1102 1103 skb_reserve(skb, rx_ring->buffer_offset); 1104 __skb_put(skb, size); 1105 1106 enetc_flip_rx_buff(rx_ring, rx_swbd); 1107 1108 return skb; 1109 } 1110 1111 static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i, 1112 u16 size, struct sk_buff *skb) 1113 { 1114 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1115 1116 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page, 1117 rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE); 1118 1119 enetc_flip_rx_buff(rx_ring, rx_swbd); 1120 } 1121 1122 static bool enetc_check_bd_errors_and_consume(struct enetc_bdr *rx_ring, 1123 u32 bd_status, 1124 union enetc_rx_bd **rxbd, int *i) 1125 { 1126 if (likely(!(bd_status & ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK)))) 1127 return false; 1128 1129 enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]); 1130 enetc_rxbd_next(rx_ring, rxbd, i); 1131 1132 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 1133 dma_rmb(); 1134 bd_status = le32_to_cpu((*rxbd)->r.lstatus); 1135 1136 enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]); 1137 enetc_rxbd_next(rx_ring, rxbd, i); 1138 } 1139 1140 rx_ring->ndev->stats.rx_dropped++; 1141 rx_ring->ndev->stats.rx_errors++; 1142 1143 return true; 1144 } 1145 1146 static struct sk_buff *enetc_build_skb(struct enetc_bdr *rx_ring, 1147 u32 bd_status, union enetc_rx_bd **rxbd, 1148 int *i, int *cleaned_cnt, int buffer_size) 1149 { 1150 struct sk_buff *skb; 1151 u16 size; 1152 1153 size = le16_to_cpu((*rxbd)->r.buf_len); 1154 skb = enetc_map_rx_buff_to_skb(rx_ring, *i, size); 1155 if (!skb) 1156 return NULL; 1157 1158 enetc_get_offloads(rx_ring, *rxbd, skb); 1159 1160 (*cleaned_cnt)++; 1161 1162 enetc_rxbd_next(rx_ring, rxbd, i); 1163 1164 /* not last BD in frame? */ 1165 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 1166 bd_status = le32_to_cpu((*rxbd)->r.lstatus); 1167 size = buffer_size; 1168 1169 if (bd_status & ENETC_RXBD_LSTATUS_F) { 1170 dma_rmb(); 1171 size = le16_to_cpu((*rxbd)->r.buf_len); 1172 } 1173 1174 enetc_add_rx_buff_to_skb(rx_ring, *i, size, skb); 1175 1176 (*cleaned_cnt)++; 1177 1178 enetc_rxbd_next(rx_ring, rxbd, i); 1179 } 1180 1181 skb_record_rx_queue(skb, rx_ring->index); 1182 skb->protocol = eth_type_trans(skb, rx_ring->ndev); 1183 1184 return skb; 1185 } 1186 1187 #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */ 1188 1189 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring, 1190 struct napi_struct *napi, int work_limit) 1191 { 1192 int rx_frm_cnt = 0, rx_byte_cnt = 0; 1193 int cleaned_cnt, i; 1194 1195 cleaned_cnt = enetc_bd_unused(rx_ring); 1196 /* next descriptor to process */ 1197 i = rx_ring->next_to_clean; 1198 1199 while (likely(rx_frm_cnt < work_limit)) { 1200 union enetc_rx_bd *rxbd; 1201 struct sk_buff *skb; 1202 u32 bd_status; 1203 1204 if (cleaned_cnt >= ENETC_RXBD_BUNDLE) 1205 cleaned_cnt -= enetc_refill_rx_ring(rx_ring, 1206 cleaned_cnt); 1207 1208 rxbd = enetc_rxbd(rx_ring, i); 1209 bd_status = le32_to_cpu(rxbd->r.lstatus); 1210 if (!bd_status) 1211 break; 1212 1213 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index)); 1214 dma_rmb(); /* for reading other rxbd fields */ 1215 1216 if (enetc_check_bd_errors_and_consume(rx_ring, bd_status, 1217 &rxbd, &i)) 1218 break; 1219 1220 skb = enetc_build_skb(rx_ring, bd_status, &rxbd, &i, 1221 &cleaned_cnt, ENETC_RXB_DMA_SIZE); 1222 if (!skb) 1223 break; 1224 1225 rx_byte_cnt += skb->len; 1226 rx_frm_cnt++; 1227 1228 napi_gro_receive(napi, skb); 1229 } 1230 1231 rx_ring->next_to_clean = i; 1232 1233 rx_ring->stats.packets += rx_frm_cnt; 1234 rx_ring->stats.bytes += rx_byte_cnt; 1235 1236 return rx_frm_cnt; 1237 } 1238 1239 static void enetc_xdp_map_tx_buff(struct enetc_bdr *tx_ring, int i, 1240 struct enetc_tx_swbd *tx_swbd, 1241 int frm_len) 1242 { 1243 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i); 1244 1245 prefetchw(txbd); 1246 1247 enetc_clear_tx_bd(txbd); 1248 txbd->addr = cpu_to_le64(tx_swbd->dma + tx_swbd->page_offset); 1249 txbd->buf_len = cpu_to_le16(tx_swbd->len); 1250 txbd->frm_len = cpu_to_le16(frm_len); 1251 1252 memcpy(&tx_ring->tx_swbd[i], tx_swbd, sizeof(*tx_swbd)); 1253 } 1254 1255 /* Puts in the TX ring one XDP frame, mapped as an array of TX software buffer 1256 * descriptors. 1257 */ 1258 static bool enetc_xdp_tx(struct enetc_bdr *tx_ring, 1259 struct enetc_tx_swbd *xdp_tx_arr, int num_tx_swbd) 1260 { 1261 struct enetc_tx_swbd *tmp_tx_swbd = xdp_tx_arr; 1262 int i, k, frm_len = tmp_tx_swbd->len; 1263 1264 if (unlikely(enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(num_tx_swbd))) 1265 return false; 1266 1267 while (unlikely(!tmp_tx_swbd->is_eof)) { 1268 tmp_tx_swbd++; 1269 frm_len += tmp_tx_swbd->len; 1270 } 1271 1272 i = tx_ring->next_to_use; 1273 1274 for (k = 0; k < num_tx_swbd; k++) { 1275 struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[k]; 1276 1277 enetc_xdp_map_tx_buff(tx_ring, i, xdp_tx_swbd, frm_len); 1278 1279 /* last BD needs 'F' bit set */ 1280 if (xdp_tx_swbd->is_eof) { 1281 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i); 1282 1283 txbd->flags = ENETC_TXBD_FLAGS_F; 1284 } 1285 1286 enetc_bdr_idx_inc(tx_ring, &i); 1287 } 1288 1289 tx_ring->next_to_use = i; 1290 1291 return true; 1292 } 1293 1294 static int enetc_xdp_frame_to_xdp_tx_swbd(struct enetc_bdr *tx_ring, 1295 struct enetc_tx_swbd *xdp_tx_arr, 1296 struct xdp_frame *xdp_frame) 1297 { 1298 struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[0]; 1299 struct skb_shared_info *shinfo; 1300 void *data = xdp_frame->data; 1301 int len = xdp_frame->len; 1302 skb_frag_t *frag; 1303 dma_addr_t dma; 1304 unsigned int f; 1305 int n = 0; 1306 1307 dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE); 1308 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) { 1309 netdev_err(tx_ring->ndev, "DMA map error\n"); 1310 return -1; 1311 } 1312 1313 xdp_tx_swbd->dma = dma; 1314 xdp_tx_swbd->dir = DMA_TO_DEVICE; 1315 xdp_tx_swbd->len = len; 1316 xdp_tx_swbd->is_xdp_redirect = true; 1317 xdp_tx_swbd->is_eof = false; 1318 xdp_tx_swbd->xdp_frame = NULL; 1319 1320 n++; 1321 1322 if (!xdp_frame_has_frags(xdp_frame)) 1323 goto out; 1324 1325 xdp_tx_swbd = &xdp_tx_arr[n]; 1326 1327 shinfo = xdp_get_shared_info_from_frame(xdp_frame); 1328 1329 for (f = 0, frag = &shinfo->frags[0]; f < shinfo->nr_frags; 1330 f++, frag++) { 1331 data = skb_frag_address(frag); 1332 len = skb_frag_size(frag); 1333 1334 dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE); 1335 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) { 1336 /* Undo the DMA mapping for all fragments */ 1337 while (--n >= 0) 1338 enetc_unmap_tx_buff(tx_ring, &xdp_tx_arr[n]); 1339 1340 netdev_err(tx_ring->ndev, "DMA map error\n"); 1341 return -1; 1342 } 1343 1344 xdp_tx_swbd->dma = dma; 1345 xdp_tx_swbd->dir = DMA_TO_DEVICE; 1346 xdp_tx_swbd->len = len; 1347 xdp_tx_swbd->is_xdp_redirect = true; 1348 xdp_tx_swbd->is_eof = false; 1349 xdp_tx_swbd->xdp_frame = NULL; 1350 1351 n++; 1352 xdp_tx_swbd = &xdp_tx_arr[n]; 1353 } 1354 out: 1355 xdp_tx_arr[n - 1].is_eof = true; 1356 xdp_tx_arr[n - 1].xdp_frame = xdp_frame; 1357 1358 return n; 1359 } 1360 1361 int enetc_xdp_xmit(struct net_device *ndev, int num_frames, 1362 struct xdp_frame **frames, u32 flags) 1363 { 1364 struct enetc_tx_swbd xdp_redirect_arr[ENETC_MAX_SKB_FRAGS] = {0}; 1365 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1366 struct enetc_bdr *tx_ring; 1367 int xdp_tx_bd_cnt, i, k; 1368 int xdp_tx_frm_cnt = 0; 1369 1370 enetc_lock_mdio(); 1371 1372 tx_ring = priv->xdp_tx_ring[smp_processor_id()]; 1373 1374 prefetchw(ENETC_TXBD(*tx_ring, tx_ring->next_to_use)); 1375 1376 for (k = 0; k < num_frames; k++) { 1377 xdp_tx_bd_cnt = enetc_xdp_frame_to_xdp_tx_swbd(tx_ring, 1378 xdp_redirect_arr, 1379 frames[k]); 1380 if (unlikely(xdp_tx_bd_cnt < 0)) 1381 break; 1382 1383 if (unlikely(!enetc_xdp_tx(tx_ring, xdp_redirect_arr, 1384 xdp_tx_bd_cnt))) { 1385 for (i = 0; i < xdp_tx_bd_cnt; i++) 1386 enetc_unmap_tx_buff(tx_ring, 1387 &xdp_redirect_arr[i]); 1388 tx_ring->stats.xdp_tx_drops++; 1389 break; 1390 } 1391 1392 xdp_tx_frm_cnt++; 1393 } 1394 1395 if (unlikely((flags & XDP_XMIT_FLUSH) || k != xdp_tx_frm_cnt)) 1396 enetc_update_tx_ring_tail(tx_ring); 1397 1398 tx_ring->stats.xdp_tx += xdp_tx_frm_cnt; 1399 1400 enetc_unlock_mdio(); 1401 1402 return xdp_tx_frm_cnt; 1403 } 1404 EXPORT_SYMBOL_GPL(enetc_xdp_xmit); 1405 1406 static void enetc_map_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i, 1407 struct xdp_buff *xdp_buff, u16 size) 1408 { 1409 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1410 void *hard_start = page_address(rx_swbd->page) + rx_swbd->page_offset; 1411 1412 /* To be used for XDP_TX */ 1413 rx_swbd->len = size; 1414 1415 xdp_prepare_buff(xdp_buff, hard_start - rx_ring->buffer_offset, 1416 rx_ring->buffer_offset, size, false); 1417 } 1418 1419 static void enetc_add_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i, 1420 u16 size, struct xdp_buff *xdp_buff) 1421 { 1422 struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp_buff); 1423 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1424 skb_frag_t *frag; 1425 1426 /* To be used for XDP_TX */ 1427 rx_swbd->len = size; 1428 1429 if (!xdp_buff_has_frags(xdp_buff)) { 1430 xdp_buff_set_frags_flag(xdp_buff); 1431 shinfo->xdp_frags_size = size; 1432 shinfo->nr_frags = 0; 1433 } else { 1434 shinfo->xdp_frags_size += size; 1435 } 1436 1437 if (page_is_pfmemalloc(rx_swbd->page)) 1438 xdp_buff_set_frag_pfmemalloc(xdp_buff); 1439 1440 frag = &shinfo->frags[shinfo->nr_frags]; 1441 skb_frag_off_set(frag, rx_swbd->page_offset); 1442 skb_frag_size_set(frag, size); 1443 __skb_frag_set_page(frag, rx_swbd->page); 1444 1445 shinfo->nr_frags++; 1446 } 1447 1448 static void enetc_build_xdp_buff(struct enetc_bdr *rx_ring, u32 bd_status, 1449 union enetc_rx_bd **rxbd, int *i, 1450 int *cleaned_cnt, struct xdp_buff *xdp_buff) 1451 { 1452 u16 size = le16_to_cpu((*rxbd)->r.buf_len); 1453 1454 xdp_init_buff(xdp_buff, ENETC_RXB_TRUESIZE, &rx_ring->xdp.rxq); 1455 1456 enetc_map_rx_buff_to_xdp(rx_ring, *i, xdp_buff, size); 1457 (*cleaned_cnt)++; 1458 enetc_rxbd_next(rx_ring, rxbd, i); 1459 1460 /* not last BD in frame? */ 1461 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 1462 bd_status = le32_to_cpu((*rxbd)->r.lstatus); 1463 size = ENETC_RXB_DMA_SIZE_XDP; 1464 1465 if (bd_status & ENETC_RXBD_LSTATUS_F) { 1466 dma_rmb(); 1467 size = le16_to_cpu((*rxbd)->r.buf_len); 1468 } 1469 1470 enetc_add_rx_buff_to_xdp(rx_ring, *i, size, xdp_buff); 1471 (*cleaned_cnt)++; 1472 enetc_rxbd_next(rx_ring, rxbd, i); 1473 } 1474 } 1475 1476 /* Convert RX buffer descriptors to TX buffer descriptors. These will be 1477 * recycled back into the RX ring in enetc_clean_tx_ring. 1478 */ 1479 static int enetc_rx_swbd_to_xdp_tx_swbd(struct enetc_tx_swbd *xdp_tx_arr, 1480 struct enetc_bdr *rx_ring, 1481 int rx_ring_first, int rx_ring_last) 1482 { 1483 int n = 0; 1484 1485 for (; rx_ring_first != rx_ring_last; 1486 n++, enetc_bdr_idx_inc(rx_ring, &rx_ring_first)) { 1487 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[rx_ring_first]; 1488 struct enetc_tx_swbd *tx_swbd = &xdp_tx_arr[n]; 1489 1490 /* No need to dma_map, we already have DMA_BIDIRECTIONAL */ 1491 tx_swbd->dma = rx_swbd->dma; 1492 tx_swbd->dir = rx_swbd->dir; 1493 tx_swbd->page = rx_swbd->page; 1494 tx_swbd->page_offset = rx_swbd->page_offset; 1495 tx_swbd->len = rx_swbd->len; 1496 tx_swbd->is_dma_page = true; 1497 tx_swbd->is_xdp_tx = true; 1498 tx_swbd->is_eof = false; 1499 } 1500 1501 /* We rely on caller providing an rx_ring_last > rx_ring_first */ 1502 xdp_tx_arr[n - 1].is_eof = true; 1503 1504 return n; 1505 } 1506 1507 static void enetc_xdp_drop(struct enetc_bdr *rx_ring, int rx_ring_first, 1508 int rx_ring_last) 1509 { 1510 while (rx_ring_first != rx_ring_last) { 1511 enetc_put_rx_buff(rx_ring, 1512 &rx_ring->rx_swbd[rx_ring_first]); 1513 enetc_bdr_idx_inc(rx_ring, &rx_ring_first); 1514 } 1515 rx_ring->stats.xdp_drops++; 1516 } 1517 1518 static int enetc_clean_rx_ring_xdp(struct enetc_bdr *rx_ring, 1519 struct napi_struct *napi, int work_limit, 1520 struct bpf_prog *prog) 1521 { 1522 int xdp_tx_bd_cnt, xdp_tx_frm_cnt = 0, xdp_redirect_frm_cnt = 0; 1523 struct enetc_tx_swbd xdp_tx_arr[ENETC_MAX_SKB_FRAGS] = {0}; 1524 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev); 1525 int rx_frm_cnt = 0, rx_byte_cnt = 0; 1526 struct enetc_bdr *tx_ring; 1527 int cleaned_cnt, i; 1528 u32 xdp_act; 1529 1530 cleaned_cnt = enetc_bd_unused(rx_ring); 1531 /* next descriptor to process */ 1532 i = rx_ring->next_to_clean; 1533 1534 while (likely(rx_frm_cnt < work_limit)) { 1535 union enetc_rx_bd *rxbd, *orig_rxbd; 1536 int orig_i, orig_cleaned_cnt; 1537 struct xdp_buff xdp_buff; 1538 struct sk_buff *skb; 1539 u32 bd_status; 1540 int err; 1541 1542 rxbd = enetc_rxbd(rx_ring, i); 1543 bd_status = le32_to_cpu(rxbd->r.lstatus); 1544 if (!bd_status) 1545 break; 1546 1547 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index)); 1548 dma_rmb(); /* for reading other rxbd fields */ 1549 1550 if (enetc_check_bd_errors_and_consume(rx_ring, bd_status, 1551 &rxbd, &i)) 1552 break; 1553 1554 orig_rxbd = rxbd; 1555 orig_cleaned_cnt = cleaned_cnt; 1556 orig_i = i; 1557 1558 enetc_build_xdp_buff(rx_ring, bd_status, &rxbd, &i, 1559 &cleaned_cnt, &xdp_buff); 1560 1561 xdp_act = bpf_prog_run_xdp(prog, &xdp_buff); 1562 1563 switch (xdp_act) { 1564 default: 1565 bpf_warn_invalid_xdp_action(rx_ring->ndev, prog, xdp_act); 1566 fallthrough; 1567 case XDP_ABORTED: 1568 trace_xdp_exception(rx_ring->ndev, prog, xdp_act); 1569 fallthrough; 1570 case XDP_DROP: 1571 enetc_xdp_drop(rx_ring, orig_i, i); 1572 break; 1573 case XDP_PASS: 1574 rxbd = orig_rxbd; 1575 cleaned_cnt = orig_cleaned_cnt; 1576 i = orig_i; 1577 1578 skb = enetc_build_skb(rx_ring, bd_status, &rxbd, 1579 &i, &cleaned_cnt, 1580 ENETC_RXB_DMA_SIZE_XDP); 1581 if (unlikely(!skb)) 1582 goto out; 1583 1584 napi_gro_receive(napi, skb); 1585 break; 1586 case XDP_TX: 1587 tx_ring = priv->xdp_tx_ring[rx_ring->index]; 1588 xdp_tx_bd_cnt = enetc_rx_swbd_to_xdp_tx_swbd(xdp_tx_arr, 1589 rx_ring, 1590 orig_i, i); 1591 1592 if (!enetc_xdp_tx(tx_ring, xdp_tx_arr, xdp_tx_bd_cnt)) { 1593 enetc_xdp_drop(rx_ring, orig_i, i); 1594 tx_ring->stats.xdp_tx_drops++; 1595 } else { 1596 tx_ring->stats.xdp_tx += xdp_tx_bd_cnt; 1597 rx_ring->xdp.xdp_tx_in_flight += xdp_tx_bd_cnt; 1598 xdp_tx_frm_cnt++; 1599 /* The XDP_TX enqueue was successful, so we 1600 * need to scrub the RX software BDs because 1601 * the ownership of the buffers no longer 1602 * belongs to the RX ring, and we must prevent 1603 * enetc_refill_rx_ring() from reusing 1604 * rx_swbd->page. 1605 */ 1606 while (orig_i != i) { 1607 rx_ring->rx_swbd[orig_i].page = NULL; 1608 enetc_bdr_idx_inc(rx_ring, &orig_i); 1609 } 1610 } 1611 break; 1612 case XDP_REDIRECT: 1613 err = xdp_do_redirect(rx_ring->ndev, &xdp_buff, prog); 1614 if (unlikely(err)) { 1615 enetc_xdp_drop(rx_ring, orig_i, i); 1616 rx_ring->stats.xdp_redirect_failures++; 1617 } else { 1618 while (orig_i != i) { 1619 enetc_flip_rx_buff(rx_ring, 1620 &rx_ring->rx_swbd[orig_i]); 1621 enetc_bdr_idx_inc(rx_ring, &orig_i); 1622 } 1623 xdp_redirect_frm_cnt++; 1624 rx_ring->stats.xdp_redirect++; 1625 } 1626 } 1627 1628 rx_frm_cnt++; 1629 } 1630 1631 out: 1632 rx_ring->next_to_clean = i; 1633 1634 rx_ring->stats.packets += rx_frm_cnt; 1635 rx_ring->stats.bytes += rx_byte_cnt; 1636 1637 if (xdp_redirect_frm_cnt) 1638 xdp_do_flush_map(); 1639 1640 if (xdp_tx_frm_cnt) 1641 enetc_update_tx_ring_tail(tx_ring); 1642 1643 if (cleaned_cnt > rx_ring->xdp.xdp_tx_in_flight) 1644 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring) - 1645 rx_ring->xdp.xdp_tx_in_flight); 1646 1647 return rx_frm_cnt; 1648 } 1649 1650 static int enetc_poll(struct napi_struct *napi, int budget) 1651 { 1652 struct enetc_int_vector 1653 *v = container_of(napi, struct enetc_int_vector, napi); 1654 struct enetc_bdr *rx_ring = &v->rx_ring; 1655 struct bpf_prog *prog; 1656 bool complete = true; 1657 int work_done; 1658 int i; 1659 1660 enetc_lock_mdio(); 1661 1662 for (i = 0; i < v->count_tx_rings; i++) 1663 if (!enetc_clean_tx_ring(&v->tx_ring[i], budget)) 1664 complete = false; 1665 1666 prog = rx_ring->xdp.prog; 1667 if (prog) 1668 work_done = enetc_clean_rx_ring_xdp(rx_ring, napi, budget, prog); 1669 else 1670 work_done = enetc_clean_rx_ring(rx_ring, napi, budget); 1671 if (work_done == budget) 1672 complete = false; 1673 if (work_done) 1674 v->rx_napi_work = true; 1675 1676 if (!complete) { 1677 enetc_unlock_mdio(); 1678 return budget; 1679 } 1680 1681 napi_complete_done(napi, work_done); 1682 1683 if (likely(v->rx_dim_en)) 1684 enetc_rx_net_dim(v); 1685 1686 v->rx_napi_work = false; 1687 1688 /* enable interrupts */ 1689 enetc_wr_reg_hot(v->rbier, ENETC_RBIER_RXTIE); 1690 1691 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS) 1692 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 1693 ENETC_TBIER_TXTIE); 1694 1695 enetc_unlock_mdio(); 1696 1697 return work_done; 1698 } 1699 1700 /* Probing and Init */ 1701 #define ENETC_MAX_RFS_SIZE 64 1702 void enetc_get_si_caps(struct enetc_si *si) 1703 { 1704 struct enetc_hw *hw = &si->hw; 1705 u32 val; 1706 1707 /* find out how many of various resources we have to work with */ 1708 val = enetc_rd(hw, ENETC_SICAPR0); 1709 si->num_rx_rings = (val >> 16) & 0xff; 1710 si->num_tx_rings = val & 0xff; 1711 1712 val = enetc_rd(hw, ENETC_SIRFSCAPR); 1713 si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val); 1714 si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE); 1715 1716 si->num_rss = 0; 1717 val = enetc_rd(hw, ENETC_SIPCAPR0); 1718 if (val & ENETC_SIPCAPR0_RSS) { 1719 u32 rss; 1720 1721 rss = enetc_rd(hw, ENETC_SIRSSCAPR); 1722 si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss); 1723 } 1724 1725 if (val & ENETC_SIPCAPR0_QBV) 1726 si->hw_features |= ENETC_SI_F_QBV; 1727 1728 if (val & ENETC_SIPCAPR0_QBU) 1729 si->hw_features |= ENETC_SI_F_QBU; 1730 1731 if (val & ENETC_SIPCAPR0_PSFP) 1732 si->hw_features |= ENETC_SI_F_PSFP; 1733 } 1734 EXPORT_SYMBOL_GPL(enetc_get_si_caps); 1735 1736 static int enetc_dma_alloc_bdr(struct enetc_bdr_resource *res) 1737 { 1738 size_t bd_base_size = res->bd_count * res->bd_size; 1739 1740 res->bd_base = dma_alloc_coherent(res->dev, bd_base_size, 1741 &res->bd_dma_base, GFP_KERNEL); 1742 if (!res->bd_base) 1743 return -ENOMEM; 1744 1745 /* h/w requires 128B alignment */ 1746 if (!IS_ALIGNED(res->bd_dma_base, 128)) { 1747 dma_free_coherent(res->dev, bd_base_size, res->bd_base, 1748 res->bd_dma_base); 1749 return -EINVAL; 1750 } 1751 1752 return 0; 1753 } 1754 1755 static void enetc_dma_free_bdr(const struct enetc_bdr_resource *res) 1756 { 1757 size_t bd_base_size = res->bd_count * res->bd_size; 1758 1759 dma_free_coherent(res->dev, bd_base_size, res->bd_base, 1760 res->bd_dma_base); 1761 } 1762 1763 static int enetc_alloc_tx_resource(struct enetc_bdr_resource *res, 1764 struct device *dev, size_t bd_count) 1765 { 1766 int err; 1767 1768 res->dev = dev; 1769 res->bd_count = bd_count; 1770 res->bd_size = sizeof(union enetc_tx_bd); 1771 1772 res->tx_swbd = vzalloc(bd_count * sizeof(*res->tx_swbd)); 1773 if (!res->tx_swbd) 1774 return -ENOMEM; 1775 1776 err = enetc_dma_alloc_bdr(res); 1777 if (err) 1778 goto err_alloc_bdr; 1779 1780 res->tso_headers = dma_alloc_coherent(dev, bd_count * TSO_HEADER_SIZE, 1781 &res->tso_headers_dma, 1782 GFP_KERNEL); 1783 if (!res->tso_headers) { 1784 err = -ENOMEM; 1785 goto err_alloc_tso; 1786 } 1787 1788 return 0; 1789 1790 err_alloc_tso: 1791 enetc_dma_free_bdr(res); 1792 err_alloc_bdr: 1793 vfree(res->tx_swbd); 1794 res->tx_swbd = NULL; 1795 1796 return err; 1797 } 1798 1799 static void enetc_free_tx_resource(const struct enetc_bdr_resource *res) 1800 { 1801 dma_free_coherent(res->dev, res->bd_count * TSO_HEADER_SIZE, 1802 res->tso_headers, res->tso_headers_dma); 1803 enetc_dma_free_bdr(res); 1804 vfree(res->tx_swbd); 1805 } 1806 1807 static struct enetc_bdr_resource * 1808 enetc_alloc_tx_resources(struct enetc_ndev_priv *priv) 1809 { 1810 struct enetc_bdr_resource *tx_res; 1811 int i, err; 1812 1813 tx_res = kcalloc(priv->num_tx_rings, sizeof(*tx_res), GFP_KERNEL); 1814 if (!tx_res) 1815 return ERR_PTR(-ENOMEM); 1816 1817 for (i = 0; i < priv->num_tx_rings; i++) { 1818 struct enetc_bdr *tx_ring = priv->tx_ring[i]; 1819 1820 err = enetc_alloc_tx_resource(&tx_res[i], tx_ring->dev, 1821 tx_ring->bd_count); 1822 if (err) 1823 goto fail; 1824 } 1825 1826 return tx_res; 1827 1828 fail: 1829 while (i-- > 0) 1830 enetc_free_tx_resource(&tx_res[i]); 1831 1832 kfree(tx_res); 1833 1834 return ERR_PTR(err); 1835 } 1836 1837 static void enetc_free_tx_resources(const struct enetc_bdr_resource *tx_res, 1838 size_t num_resources) 1839 { 1840 size_t i; 1841 1842 for (i = 0; i < num_resources; i++) 1843 enetc_free_tx_resource(&tx_res[i]); 1844 1845 kfree(tx_res); 1846 } 1847 1848 static int enetc_alloc_rx_resource(struct enetc_bdr_resource *res, 1849 struct device *dev, size_t bd_count, 1850 bool extended) 1851 { 1852 int err; 1853 1854 res->dev = dev; 1855 res->bd_count = bd_count; 1856 res->bd_size = sizeof(union enetc_rx_bd); 1857 if (extended) 1858 res->bd_size *= 2; 1859 1860 res->rx_swbd = vzalloc(bd_count * sizeof(struct enetc_rx_swbd)); 1861 if (!res->rx_swbd) 1862 return -ENOMEM; 1863 1864 err = enetc_dma_alloc_bdr(res); 1865 if (err) { 1866 vfree(res->rx_swbd); 1867 return err; 1868 } 1869 1870 return 0; 1871 } 1872 1873 static void enetc_free_rx_resource(const struct enetc_bdr_resource *res) 1874 { 1875 enetc_dma_free_bdr(res); 1876 vfree(res->rx_swbd); 1877 } 1878 1879 static struct enetc_bdr_resource * 1880 enetc_alloc_rx_resources(struct enetc_ndev_priv *priv, bool extended) 1881 { 1882 struct enetc_bdr_resource *rx_res; 1883 int i, err; 1884 1885 rx_res = kcalloc(priv->num_rx_rings, sizeof(*rx_res), GFP_KERNEL); 1886 if (!rx_res) 1887 return ERR_PTR(-ENOMEM); 1888 1889 for (i = 0; i < priv->num_rx_rings; i++) { 1890 struct enetc_bdr *rx_ring = priv->rx_ring[i]; 1891 1892 err = enetc_alloc_rx_resource(&rx_res[i], rx_ring->dev, 1893 rx_ring->bd_count, extended); 1894 if (err) 1895 goto fail; 1896 } 1897 1898 return rx_res; 1899 1900 fail: 1901 while (i-- > 0) 1902 enetc_free_rx_resource(&rx_res[i]); 1903 1904 kfree(rx_res); 1905 1906 return ERR_PTR(err); 1907 } 1908 1909 static void enetc_free_rx_resources(const struct enetc_bdr_resource *rx_res, 1910 size_t num_resources) 1911 { 1912 size_t i; 1913 1914 for (i = 0; i < num_resources; i++) 1915 enetc_free_rx_resource(&rx_res[i]); 1916 1917 kfree(rx_res); 1918 } 1919 1920 static void enetc_assign_tx_resource(struct enetc_bdr *tx_ring, 1921 const struct enetc_bdr_resource *res) 1922 { 1923 tx_ring->bd_base = res ? res->bd_base : NULL; 1924 tx_ring->bd_dma_base = res ? res->bd_dma_base : 0; 1925 tx_ring->tx_swbd = res ? res->tx_swbd : NULL; 1926 tx_ring->tso_headers = res ? res->tso_headers : NULL; 1927 tx_ring->tso_headers_dma = res ? res->tso_headers_dma : 0; 1928 } 1929 1930 static void enetc_assign_rx_resource(struct enetc_bdr *rx_ring, 1931 const struct enetc_bdr_resource *res) 1932 { 1933 rx_ring->bd_base = res ? res->bd_base : NULL; 1934 rx_ring->bd_dma_base = res ? res->bd_dma_base : 0; 1935 rx_ring->rx_swbd = res ? res->rx_swbd : NULL; 1936 } 1937 1938 static void enetc_assign_tx_resources(struct enetc_ndev_priv *priv, 1939 const struct enetc_bdr_resource *res) 1940 { 1941 int i; 1942 1943 if (priv->tx_res) 1944 enetc_free_tx_resources(priv->tx_res, priv->num_tx_rings); 1945 1946 for (i = 0; i < priv->num_tx_rings; i++) { 1947 enetc_assign_tx_resource(priv->tx_ring[i], 1948 res ? &res[i] : NULL); 1949 } 1950 1951 priv->tx_res = res; 1952 } 1953 1954 static void enetc_assign_rx_resources(struct enetc_ndev_priv *priv, 1955 const struct enetc_bdr_resource *res) 1956 { 1957 int i; 1958 1959 if (priv->rx_res) 1960 enetc_free_rx_resources(priv->rx_res, priv->num_rx_rings); 1961 1962 for (i = 0; i < priv->num_rx_rings; i++) { 1963 enetc_assign_rx_resource(priv->rx_ring[i], 1964 res ? &res[i] : NULL); 1965 } 1966 1967 priv->rx_res = res; 1968 } 1969 1970 static void enetc_free_tx_ring(struct enetc_bdr *tx_ring) 1971 { 1972 int i; 1973 1974 for (i = 0; i < tx_ring->bd_count; i++) { 1975 struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i]; 1976 1977 enetc_free_tx_frame(tx_ring, tx_swbd); 1978 } 1979 } 1980 1981 static void enetc_free_rx_ring(struct enetc_bdr *rx_ring) 1982 { 1983 int i; 1984 1985 for (i = 0; i < rx_ring->bd_count; i++) { 1986 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 1987 1988 if (!rx_swbd->page) 1989 continue; 1990 1991 dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE, 1992 rx_swbd->dir); 1993 __free_page(rx_swbd->page); 1994 rx_swbd->page = NULL; 1995 } 1996 } 1997 1998 static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv) 1999 { 2000 int i; 2001 2002 for (i = 0; i < priv->num_rx_rings; i++) 2003 enetc_free_rx_ring(priv->rx_ring[i]); 2004 2005 for (i = 0; i < priv->num_tx_rings; i++) 2006 enetc_free_tx_ring(priv->tx_ring[i]); 2007 } 2008 2009 static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups) 2010 { 2011 int *rss_table; 2012 int i; 2013 2014 rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL); 2015 if (!rss_table) 2016 return -ENOMEM; 2017 2018 /* Set up RSS table defaults */ 2019 for (i = 0; i < si->num_rss; i++) 2020 rss_table[i] = i % num_groups; 2021 2022 enetc_set_rss_table(si, rss_table, si->num_rss); 2023 2024 kfree(rss_table); 2025 2026 return 0; 2027 } 2028 2029 int enetc_configure_si(struct enetc_ndev_priv *priv) 2030 { 2031 struct enetc_si *si = priv->si; 2032 struct enetc_hw *hw = &si->hw; 2033 int err; 2034 2035 /* set SI cache attributes */ 2036 enetc_wr(hw, ENETC_SICAR0, 2037 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT); 2038 enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI); 2039 /* enable SI */ 2040 enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN); 2041 2042 if (si->num_rss) { 2043 err = enetc_setup_default_rss_table(si, priv->num_rx_rings); 2044 if (err) 2045 return err; 2046 } 2047 2048 return 0; 2049 } 2050 EXPORT_SYMBOL_GPL(enetc_configure_si); 2051 2052 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv) 2053 { 2054 struct enetc_si *si = priv->si; 2055 int cpus = num_online_cpus(); 2056 2057 priv->tx_bd_count = ENETC_TX_RING_DEFAULT_SIZE; 2058 priv->rx_bd_count = ENETC_RX_RING_DEFAULT_SIZE; 2059 2060 /* Enable all available TX rings in order to configure as many 2061 * priorities as possible, when needed. 2062 * TODO: Make # of TX rings run-time configurable 2063 */ 2064 priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings); 2065 priv->num_tx_rings = si->num_tx_rings; 2066 priv->bdr_int_num = cpus; 2067 priv->ic_mode = ENETC_IC_RX_ADAPTIVE | ENETC_IC_TX_MANUAL; 2068 priv->tx_ictt = ENETC_TXIC_TIMETHR; 2069 } 2070 EXPORT_SYMBOL_GPL(enetc_init_si_rings_params); 2071 2072 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv) 2073 { 2074 struct enetc_si *si = priv->si; 2075 2076 priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules), 2077 GFP_KERNEL); 2078 if (!priv->cls_rules) 2079 return -ENOMEM; 2080 2081 return 0; 2082 } 2083 EXPORT_SYMBOL_GPL(enetc_alloc_si_resources); 2084 2085 void enetc_free_si_resources(struct enetc_ndev_priv *priv) 2086 { 2087 kfree(priv->cls_rules); 2088 } 2089 EXPORT_SYMBOL_GPL(enetc_free_si_resources); 2090 2091 static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 2092 { 2093 int idx = tx_ring->index; 2094 u32 tbmr; 2095 2096 enetc_txbdr_wr(hw, idx, ENETC_TBBAR0, 2097 lower_32_bits(tx_ring->bd_dma_base)); 2098 2099 enetc_txbdr_wr(hw, idx, ENETC_TBBAR1, 2100 upper_32_bits(tx_ring->bd_dma_base)); 2101 2102 WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */ 2103 enetc_txbdr_wr(hw, idx, ENETC_TBLENR, 2104 ENETC_RTBLENR_LEN(tx_ring->bd_count)); 2105 2106 /* clearing PI/CI registers for Tx not supported, adjust sw indexes */ 2107 tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR); 2108 tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR); 2109 2110 /* enable Tx ints by setting pkt thr to 1 */ 2111 enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1); 2112 2113 tbmr = ENETC_TBMR_SET_PRIO(tx_ring->prio); 2114 if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX) 2115 tbmr |= ENETC_TBMR_VIH; 2116 2117 /* enable ring */ 2118 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr); 2119 2120 tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR); 2121 tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR); 2122 tx_ring->idr = hw->reg + ENETC_SITXIDR; 2123 } 2124 2125 static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring, 2126 bool extended) 2127 { 2128 int idx = rx_ring->index; 2129 u32 rbmr = 0; 2130 2131 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0, 2132 lower_32_bits(rx_ring->bd_dma_base)); 2133 2134 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1, 2135 upper_32_bits(rx_ring->bd_dma_base)); 2136 2137 WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */ 2138 enetc_rxbdr_wr(hw, idx, ENETC_RBLENR, 2139 ENETC_RTBLENR_LEN(rx_ring->bd_count)); 2140 2141 if (rx_ring->xdp.prog) 2142 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE_XDP); 2143 else 2144 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE); 2145 2146 /* Also prepare the consumer index in case page allocation never 2147 * succeeds. In that case, hardware will never advance producer index 2148 * to match consumer index, and will drop all frames. 2149 */ 2150 enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0); 2151 enetc_rxbdr_wr(hw, idx, ENETC_RBCIR, 1); 2152 2153 /* enable Rx ints by setting pkt thr to 1 */ 2154 enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1); 2155 2156 rx_ring->ext_en = extended; 2157 if (rx_ring->ext_en) 2158 rbmr |= ENETC_RBMR_BDS; 2159 2160 if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX) 2161 rbmr |= ENETC_RBMR_VTE; 2162 2163 rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR); 2164 rx_ring->idr = hw->reg + ENETC_SIRXIDR; 2165 2166 rx_ring->next_to_clean = 0; 2167 rx_ring->next_to_use = 0; 2168 rx_ring->next_to_alloc = 0; 2169 2170 enetc_lock_mdio(); 2171 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring)); 2172 enetc_unlock_mdio(); 2173 2174 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr); 2175 } 2176 2177 static void enetc_setup_bdrs(struct enetc_ndev_priv *priv, bool extended) 2178 { 2179 struct enetc_hw *hw = &priv->si->hw; 2180 int i; 2181 2182 for (i = 0; i < priv->num_tx_rings; i++) 2183 enetc_setup_txbdr(hw, priv->tx_ring[i]); 2184 2185 for (i = 0; i < priv->num_rx_rings; i++) 2186 enetc_setup_rxbdr(hw, priv->rx_ring[i], extended); 2187 } 2188 2189 static void enetc_enable_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 2190 { 2191 int idx = tx_ring->index; 2192 u32 tbmr; 2193 2194 tbmr = enetc_txbdr_rd(hw, idx, ENETC_TBMR); 2195 tbmr |= ENETC_TBMR_EN; 2196 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr); 2197 } 2198 2199 static void enetc_enable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 2200 { 2201 int idx = rx_ring->index; 2202 u32 rbmr; 2203 2204 rbmr = enetc_rxbdr_rd(hw, idx, ENETC_RBMR); 2205 rbmr |= ENETC_RBMR_EN; 2206 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr); 2207 } 2208 2209 static void enetc_enable_bdrs(struct enetc_ndev_priv *priv) 2210 { 2211 struct enetc_hw *hw = &priv->si->hw; 2212 int i; 2213 2214 for (i = 0; i < priv->num_tx_rings; i++) 2215 enetc_enable_txbdr(hw, priv->tx_ring[i]); 2216 2217 for (i = 0; i < priv->num_rx_rings; i++) 2218 enetc_enable_rxbdr(hw, priv->rx_ring[i]); 2219 } 2220 2221 static void enetc_disable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 2222 { 2223 int idx = rx_ring->index; 2224 2225 /* disable EN bit on ring */ 2226 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0); 2227 } 2228 2229 static void enetc_disable_txbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 2230 { 2231 int idx = rx_ring->index; 2232 2233 /* disable EN bit on ring */ 2234 enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0); 2235 } 2236 2237 static void enetc_disable_bdrs(struct enetc_ndev_priv *priv) 2238 { 2239 struct enetc_hw *hw = &priv->si->hw; 2240 int i; 2241 2242 for (i = 0; i < priv->num_tx_rings; i++) 2243 enetc_disable_txbdr(hw, priv->tx_ring[i]); 2244 2245 for (i = 0; i < priv->num_rx_rings; i++) 2246 enetc_disable_rxbdr(hw, priv->rx_ring[i]); 2247 } 2248 2249 static void enetc_wait_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 2250 { 2251 int delay = 8, timeout = 100; 2252 int idx = tx_ring->index; 2253 2254 /* wait for busy to clear */ 2255 while (delay < timeout && 2256 enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) { 2257 msleep(delay); 2258 delay *= 2; 2259 } 2260 2261 if (delay >= timeout) 2262 netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n", 2263 idx); 2264 } 2265 2266 static void enetc_wait_bdrs(struct enetc_ndev_priv *priv) 2267 { 2268 struct enetc_hw *hw = &priv->si->hw; 2269 int i; 2270 2271 for (i = 0; i < priv->num_tx_rings; i++) 2272 enetc_wait_txbdr(hw, priv->tx_ring[i]); 2273 } 2274 2275 static int enetc_setup_irqs(struct enetc_ndev_priv *priv) 2276 { 2277 struct pci_dev *pdev = priv->si->pdev; 2278 struct enetc_hw *hw = &priv->si->hw; 2279 int i, j, err; 2280 2281 for (i = 0; i < priv->bdr_int_num; i++) { 2282 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 2283 struct enetc_int_vector *v = priv->int_vector[i]; 2284 int entry = ENETC_BDR_INT_BASE_IDX + i; 2285 2286 snprintf(v->name, sizeof(v->name), "%s-rxtx%d", 2287 priv->ndev->name, i); 2288 err = request_irq(irq, enetc_msix, 0, v->name, v); 2289 if (err) { 2290 dev_err(priv->dev, "request_irq() failed!\n"); 2291 goto irq_err; 2292 } 2293 disable_irq(irq); 2294 2295 v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER); 2296 v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER); 2297 v->ricr1 = hw->reg + ENETC_BDR(RX, i, ENETC_RBICR1); 2298 2299 enetc_wr(hw, ENETC_SIMSIRRV(i), entry); 2300 2301 for (j = 0; j < v->count_tx_rings; j++) { 2302 int idx = v->tx_ring[j].index; 2303 2304 enetc_wr(hw, ENETC_SIMSITRV(idx), entry); 2305 } 2306 irq_set_affinity_hint(irq, get_cpu_mask(i % num_online_cpus())); 2307 } 2308 2309 return 0; 2310 2311 irq_err: 2312 while (i--) { 2313 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 2314 2315 irq_set_affinity_hint(irq, NULL); 2316 free_irq(irq, priv->int_vector[i]); 2317 } 2318 2319 return err; 2320 } 2321 2322 static void enetc_free_irqs(struct enetc_ndev_priv *priv) 2323 { 2324 struct pci_dev *pdev = priv->si->pdev; 2325 int i; 2326 2327 for (i = 0; i < priv->bdr_int_num; i++) { 2328 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 2329 2330 irq_set_affinity_hint(irq, NULL); 2331 free_irq(irq, priv->int_vector[i]); 2332 } 2333 } 2334 2335 static void enetc_setup_interrupts(struct enetc_ndev_priv *priv) 2336 { 2337 struct enetc_hw *hw = &priv->si->hw; 2338 u32 icpt, ictt; 2339 int i; 2340 2341 /* enable Tx & Rx event indication */ 2342 if (priv->ic_mode & 2343 (ENETC_IC_RX_MANUAL | ENETC_IC_RX_ADAPTIVE)) { 2344 icpt = ENETC_RBICR0_SET_ICPT(ENETC_RXIC_PKTTHR); 2345 /* init to non-0 minimum, will be adjusted later */ 2346 ictt = 0x1; 2347 } else { 2348 icpt = 0x1; /* enable Rx ints by setting pkt thr to 1 */ 2349 ictt = 0; 2350 } 2351 2352 for (i = 0; i < priv->num_rx_rings; i++) { 2353 enetc_rxbdr_wr(hw, i, ENETC_RBICR1, ictt); 2354 enetc_rxbdr_wr(hw, i, ENETC_RBICR0, ENETC_RBICR0_ICEN | icpt); 2355 enetc_rxbdr_wr(hw, i, ENETC_RBIER, ENETC_RBIER_RXTIE); 2356 } 2357 2358 if (priv->ic_mode & ENETC_IC_TX_MANUAL) 2359 icpt = ENETC_TBICR0_SET_ICPT(ENETC_TXIC_PKTTHR); 2360 else 2361 icpt = 0x1; /* enable Tx ints by setting pkt thr to 1 */ 2362 2363 for (i = 0; i < priv->num_tx_rings; i++) { 2364 enetc_txbdr_wr(hw, i, ENETC_TBICR1, priv->tx_ictt); 2365 enetc_txbdr_wr(hw, i, ENETC_TBICR0, ENETC_TBICR0_ICEN | icpt); 2366 enetc_txbdr_wr(hw, i, ENETC_TBIER, ENETC_TBIER_TXTIE); 2367 } 2368 } 2369 2370 static void enetc_clear_interrupts(struct enetc_ndev_priv *priv) 2371 { 2372 struct enetc_hw *hw = &priv->si->hw; 2373 int i; 2374 2375 for (i = 0; i < priv->num_tx_rings; i++) 2376 enetc_txbdr_wr(hw, i, ENETC_TBIER, 0); 2377 2378 for (i = 0; i < priv->num_rx_rings; i++) 2379 enetc_rxbdr_wr(hw, i, ENETC_RBIER, 0); 2380 } 2381 2382 static int enetc_phylink_connect(struct net_device *ndev) 2383 { 2384 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2385 struct ethtool_eee edata; 2386 int err; 2387 2388 if (!priv->phylink) { 2389 /* phy-less mode */ 2390 netif_carrier_on(ndev); 2391 return 0; 2392 } 2393 2394 err = phylink_of_phy_connect(priv->phylink, priv->dev->of_node, 0); 2395 if (err) { 2396 dev_err(&ndev->dev, "could not attach to PHY\n"); 2397 return err; 2398 } 2399 2400 /* disable EEE autoneg, until ENETC driver supports it */ 2401 memset(&edata, 0, sizeof(struct ethtool_eee)); 2402 phylink_ethtool_set_eee(priv->phylink, &edata); 2403 2404 phylink_start(priv->phylink); 2405 2406 return 0; 2407 } 2408 2409 static void enetc_tx_onestep_tstamp(struct work_struct *work) 2410 { 2411 struct enetc_ndev_priv *priv; 2412 struct sk_buff *skb; 2413 2414 priv = container_of(work, struct enetc_ndev_priv, tx_onestep_tstamp); 2415 2416 netif_tx_lock_bh(priv->ndev); 2417 2418 clear_bit_unlock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, &priv->flags); 2419 skb = skb_dequeue(&priv->tx_skbs); 2420 if (skb) 2421 enetc_start_xmit(skb, priv->ndev); 2422 2423 netif_tx_unlock_bh(priv->ndev); 2424 } 2425 2426 static void enetc_tx_onestep_tstamp_init(struct enetc_ndev_priv *priv) 2427 { 2428 INIT_WORK(&priv->tx_onestep_tstamp, enetc_tx_onestep_tstamp); 2429 skb_queue_head_init(&priv->tx_skbs); 2430 } 2431 2432 void enetc_start(struct net_device *ndev) 2433 { 2434 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2435 int i; 2436 2437 enetc_setup_interrupts(priv); 2438 2439 for (i = 0; i < priv->bdr_int_num; i++) { 2440 int irq = pci_irq_vector(priv->si->pdev, 2441 ENETC_BDR_INT_BASE_IDX + i); 2442 2443 napi_enable(&priv->int_vector[i]->napi); 2444 enable_irq(irq); 2445 } 2446 2447 enetc_enable_bdrs(priv); 2448 2449 netif_tx_start_all_queues(ndev); 2450 } 2451 EXPORT_SYMBOL_GPL(enetc_start); 2452 2453 int enetc_open(struct net_device *ndev) 2454 { 2455 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2456 struct enetc_bdr_resource *tx_res, *rx_res; 2457 bool extended; 2458 int err; 2459 2460 extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP); 2461 2462 err = enetc_setup_irqs(priv); 2463 if (err) 2464 return err; 2465 2466 err = enetc_phylink_connect(ndev); 2467 if (err) 2468 goto err_phy_connect; 2469 2470 tx_res = enetc_alloc_tx_resources(priv); 2471 if (IS_ERR(tx_res)) { 2472 err = PTR_ERR(tx_res); 2473 goto err_alloc_tx; 2474 } 2475 2476 rx_res = enetc_alloc_rx_resources(priv, extended); 2477 if (IS_ERR(rx_res)) { 2478 err = PTR_ERR(rx_res); 2479 goto err_alloc_rx; 2480 } 2481 2482 enetc_tx_onestep_tstamp_init(priv); 2483 enetc_assign_tx_resources(priv, tx_res); 2484 enetc_assign_rx_resources(priv, rx_res); 2485 enetc_setup_bdrs(priv, extended); 2486 enetc_start(ndev); 2487 2488 return 0; 2489 2490 err_alloc_rx: 2491 enetc_free_tx_resources(tx_res, priv->num_tx_rings); 2492 err_alloc_tx: 2493 if (priv->phylink) 2494 phylink_disconnect_phy(priv->phylink); 2495 err_phy_connect: 2496 enetc_free_irqs(priv); 2497 2498 return err; 2499 } 2500 EXPORT_SYMBOL_GPL(enetc_open); 2501 2502 void enetc_stop(struct net_device *ndev) 2503 { 2504 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2505 int i; 2506 2507 netif_tx_stop_all_queues(ndev); 2508 2509 enetc_disable_bdrs(priv); 2510 2511 for (i = 0; i < priv->bdr_int_num; i++) { 2512 int irq = pci_irq_vector(priv->si->pdev, 2513 ENETC_BDR_INT_BASE_IDX + i); 2514 2515 disable_irq(irq); 2516 napi_synchronize(&priv->int_vector[i]->napi); 2517 napi_disable(&priv->int_vector[i]->napi); 2518 } 2519 2520 enetc_wait_bdrs(priv); 2521 2522 enetc_clear_interrupts(priv); 2523 } 2524 EXPORT_SYMBOL_GPL(enetc_stop); 2525 2526 int enetc_close(struct net_device *ndev) 2527 { 2528 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2529 2530 enetc_stop(ndev); 2531 2532 if (priv->phylink) { 2533 phylink_stop(priv->phylink); 2534 phylink_disconnect_phy(priv->phylink); 2535 } else { 2536 netif_carrier_off(ndev); 2537 } 2538 2539 enetc_free_rxtx_rings(priv); 2540 2541 /* Avoids dangling pointers and also frees old resources */ 2542 enetc_assign_rx_resources(priv, NULL); 2543 enetc_assign_tx_resources(priv, NULL); 2544 2545 enetc_free_irqs(priv); 2546 2547 return 0; 2548 } 2549 EXPORT_SYMBOL_GPL(enetc_close); 2550 2551 static int enetc_reconfigure(struct enetc_ndev_priv *priv, bool extended, 2552 int (*cb)(struct enetc_ndev_priv *priv, void *ctx), 2553 void *ctx) 2554 { 2555 struct enetc_bdr_resource *tx_res, *rx_res; 2556 int err; 2557 2558 ASSERT_RTNL(); 2559 2560 /* If the interface is down, run the callback right away, 2561 * without reconfiguration. 2562 */ 2563 if (!netif_running(priv->ndev)) { 2564 if (cb) { 2565 err = cb(priv, ctx); 2566 if (err) 2567 return err; 2568 } 2569 2570 return 0; 2571 } 2572 2573 tx_res = enetc_alloc_tx_resources(priv); 2574 if (IS_ERR(tx_res)) { 2575 err = PTR_ERR(tx_res); 2576 goto out; 2577 } 2578 2579 rx_res = enetc_alloc_rx_resources(priv, extended); 2580 if (IS_ERR(rx_res)) { 2581 err = PTR_ERR(rx_res); 2582 goto out_free_tx_res; 2583 } 2584 2585 enetc_stop(priv->ndev); 2586 enetc_free_rxtx_rings(priv); 2587 2588 /* Interface is down, run optional callback now */ 2589 if (cb) { 2590 err = cb(priv, ctx); 2591 if (err) 2592 goto out_restart; 2593 } 2594 2595 enetc_assign_tx_resources(priv, tx_res); 2596 enetc_assign_rx_resources(priv, rx_res); 2597 enetc_setup_bdrs(priv, extended); 2598 enetc_start(priv->ndev); 2599 2600 return 0; 2601 2602 out_restart: 2603 enetc_setup_bdrs(priv, extended); 2604 enetc_start(priv->ndev); 2605 enetc_free_rx_resources(rx_res, priv->num_rx_rings); 2606 out_free_tx_res: 2607 enetc_free_tx_resources(tx_res, priv->num_tx_rings); 2608 out: 2609 return err; 2610 } 2611 2612 static void enetc_debug_tx_ring_prios(struct enetc_ndev_priv *priv) 2613 { 2614 int i; 2615 2616 for (i = 0; i < priv->num_tx_rings; i++) 2617 netdev_dbg(priv->ndev, "TX ring %d prio %d\n", i, 2618 priv->tx_ring[i]->prio); 2619 } 2620 2621 static void enetc_reset_tc_mqprio(struct net_device *ndev) 2622 { 2623 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2624 struct enetc_hw *hw = &priv->si->hw; 2625 struct enetc_bdr *tx_ring; 2626 int num_stack_tx_queues; 2627 int i; 2628 2629 num_stack_tx_queues = enetc_num_stack_tx_queues(priv); 2630 2631 netdev_reset_tc(ndev); 2632 netif_set_real_num_tx_queues(ndev, num_stack_tx_queues); 2633 priv->min_num_stack_tx_queues = num_possible_cpus(); 2634 2635 /* Reset all ring priorities to 0 */ 2636 for (i = 0; i < priv->num_tx_rings; i++) { 2637 tx_ring = priv->tx_ring[i]; 2638 tx_ring->prio = 0; 2639 enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio); 2640 } 2641 2642 enetc_debug_tx_ring_prios(priv); 2643 } 2644 2645 int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data) 2646 { 2647 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2648 struct tc_mqprio_qopt *mqprio = type_data; 2649 struct enetc_hw *hw = &priv->si->hw; 2650 int num_stack_tx_queues = 0; 2651 u8 num_tc = mqprio->num_tc; 2652 struct enetc_bdr *tx_ring; 2653 int offset, count; 2654 int err, tc, q; 2655 2656 if (!num_tc) { 2657 enetc_reset_tc_mqprio(ndev); 2658 return 0; 2659 } 2660 2661 err = netdev_set_num_tc(ndev, num_tc); 2662 if (err) 2663 return err; 2664 2665 for (tc = 0; tc < num_tc; tc++) { 2666 offset = mqprio->offset[tc]; 2667 count = mqprio->count[tc]; 2668 num_stack_tx_queues += count; 2669 2670 err = netdev_set_tc_queue(ndev, tc, count, offset); 2671 if (err) 2672 goto err_reset_tc; 2673 2674 for (q = offset; q < offset + count; q++) { 2675 tx_ring = priv->tx_ring[q]; 2676 /* The prio_tc_map is skb_tx_hash()'s way of selecting 2677 * between TX queues based on skb->priority. As such, 2678 * there's nothing to offload based on it. 2679 * Make the mqprio "traffic class" be the priority of 2680 * this ring group, and leave the Tx IPV to traffic 2681 * class mapping as its default mapping value of 1:1. 2682 */ 2683 tx_ring->prio = tc; 2684 enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio); 2685 } 2686 } 2687 2688 err = netif_set_real_num_tx_queues(ndev, num_stack_tx_queues); 2689 if (err) 2690 goto err_reset_tc; 2691 2692 priv->min_num_stack_tx_queues = num_stack_tx_queues; 2693 2694 enetc_debug_tx_ring_prios(priv); 2695 2696 return 0; 2697 2698 err_reset_tc: 2699 enetc_reset_tc_mqprio(ndev); 2700 return err; 2701 } 2702 EXPORT_SYMBOL_GPL(enetc_setup_tc_mqprio); 2703 2704 static int enetc_reconfigure_xdp_cb(struct enetc_ndev_priv *priv, void *ctx) 2705 { 2706 struct bpf_prog *old_prog, *prog = ctx; 2707 int num_stack_tx_queues; 2708 int err, i; 2709 2710 old_prog = xchg(&priv->xdp_prog, prog); 2711 2712 num_stack_tx_queues = enetc_num_stack_tx_queues(priv); 2713 err = netif_set_real_num_tx_queues(priv->ndev, num_stack_tx_queues); 2714 if (err) { 2715 xchg(&priv->xdp_prog, old_prog); 2716 return err; 2717 } 2718 2719 if (old_prog) 2720 bpf_prog_put(old_prog); 2721 2722 for (i = 0; i < priv->num_rx_rings; i++) { 2723 struct enetc_bdr *rx_ring = priv->rx_ring[i]; 2724 2725 rx_ring->xdp.prog = prog; 2726 2727 if (prog) 2728 rx_ring->buffer_offset = XDP_PACKET_HEADROOM; 2729 else 2730 rx_ring->buffer_offset = ENETC_RXB_PAD; 2731 } 2732 2733 return 0; 2734 } 2735 2736 static int enetc_setup_xdp_prog(struct net_device *ndev, struct bpf_prog *prog, 2737 struct netlink_ext_ack *extack) 2738 { 2739 int num_xdp_tx_queues = prog ? num_possible_cpus() : 0; 2740 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2741 bool extended; 2742 2743 if (priv->min_num_stack_tx_queues + num_xdp_tx_queues > 2744 priv->num_tx_rings) { 2745 NL_SET_ERR_MSG_FMT_MOD(extack, 2746 "Reserving %d XDP TXQs does not leave a minimum of %d TXQs for network stack (total %d available)", 2747 num_xdp_tx_queues, 2748 priv->min_num_stack_tx_queues, 2749 priv->num_tx_rings); 2750 return -EBUSY; 2751 } 2752 2753 extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP); 2754 2755 /* The buffer layout is changing, so we need to drain the old 2756 * RX buffers and seed new ones. 2757 */ 2758 return enetc_reconfigure(priv, extended, enetc_reconfigure_xdp_cb, prog); 2759 } 2760 2761 int enetc_setup_bpf(struct net_device *ndev, struct netdev_bpf *bpf) 2762 { 2763 switch (bpf->command) { 2764 case XDP_SETUP_PROG: 2765 return enetc_setup_xdp_prog(ndev, bpf->prog, bpf->extack); 2766 default: 2767 return -EINVAL; 2768 } 2769 2770 return 0; 2771 } 2772 EXPORT_SYMBOL_GPL(enetc_setup_bpf); 2773 2774 struct net_device_stats *enetc_get_stats(struct net_device *ndev) 2775 { 2776 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2777 struct net_device_stats *stats = &ndev->stats; 2778 unsigned long packets = 0, bytes = 0; 2779 unsigned long tx_dropped = 0; 2780 int i; 2781 2782 for (i = 0; i < priv->num_rx_rings; i++) { 2783 packets += priv->rx_ring[i]->stats.packets; 2784 bytes += priv->rx_ring[i]->stats.bytes; 2785 } 2786 2787 stats->rx_packets = packets; 2788 stats->rx_bytes = bytes; 2789 bytes = 0; 2790 packets = 0; 2791 2792 for (i = 0; i < priv->num_tx_rings; i++) { 2793 packets += priv->tx_ring[i]->stats.packets; 2794 bytes += priv->tx_ring[i]->stats.bytes; 2795 tx_dropped += priv->tx_ring[i]->stats.win_drop; 2796 } 2797 2798 stats->tx_packets = packets; 2799 stats->tx_bytes = bytes; 2800 stats->tx_dropped = tx_dropped; 2801 2802 return stats; 2803 } 2804 EXPORT_SYMBOL_GPL(enetc_get_stats); 2805 2806 static int enetc_set_rss(struct net_device *ndev, int en) 2807 { 2808 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2809 struct enetc_hw *hw = &priv->si->hw; 2810 u32 reg; 2811 2812 enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings); 2813 2814 reg = enetc_rd(hw, ENETC_SIMR); 2815 reg &= ~ENETC_SIMR_RSSE; 2816 reg |= (en) ? ENETC_SIMR_RSSE : 0; 2817 enetc_wr(hw, ENETC_SIMR, reg); 2818 2819 return 0; 2820 } 2821 2822 static void enetc_enable_rxvlan(struct net_device *ndev, bool en) 2823 { 2824 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2825 struct enetc_hw *hw = &priv->si->hw; 2826 int i; 2827 2828 for (i = 0; i < priv->num_rx_rings; i++) 2829 enetc_bdr_enable_rxvlan(hw, i, en); 2830 } 2831 2832 static void enetc_enable_txvlan(struct net_device *ndev, bool en) 2833 { 2834 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2835 struct enetc_hw *hw = &priv->si->hw; 2836 int i; 2837 2838 for (i = 0; i < priv->num_tx_rings; i++) 2839 enetc_bdr_enable_txvlan(hw, i, en); 2840 } 2841 2842 void enetc_set_features(struct net_device *ndev, netdev_features_t features) 2843 { 2844 netdev_features_t changed = ndev->features ^ features; 2845 2846 if (changed & NETIF_F_RXHASH) 2847 enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH)); 2848 2849 if (changed & NETIF_F_HW_VLAN_CTAG_RX) 2850 enetc_enable_rxvlan(ndev, 2851 !!(features & NETIF_F_HW_VLAN_CTAG_RX)); 2852 2853 if (changed & NETIF_F_HW_VLAN_CTAG_TX) 2854 enetc_enable_txvlan(ndev, 2855 !!(features & NETIF_F_HW_VLAN_CTAG_TX)); 2856 } 2857 EXPORT_SYMBOL_GPL(enetc_set_features); 2858 2859 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 2860 static int enetc_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr) 2861 { 2862 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2863 int err, new_offloads = priv->active_offloads; 2864 struct hwtstamp_config config; 2865 2866 if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) 2867 return -EFAULT; 2868 2869 switch (config.tx_type) { 2870 case HWTSTAMP_TX_OFF: 2871 new_offloads &= ~ENETC_F_TX_TSTAMP_MASK; 2872 break; 2873 case HWTSTAMP_TX_ON: 2874 new_offloads &= ~ENETC_F_TX_TSTAMP_MASK; 2875 new_offloads |= ENETC_F_TX_TSTAMP; 2876 break; 2877 case HWTSTAMP_TX_ONESTEP_SYNC: 2878 new_offloads &= ~ENETC_F_TX_TSTAMP_MASK; 2879 new_offloads |= ENETC_F_TX_ONESTEP_SYNC_TSTAMP; 2880 break; 2881 default: 2882 return -ERANGE; 2883 } 2884 2885 switch (config.rx_filter) { 2886 case HWTSTAMP_FILTER_NONE: 2887 new_offloads &= ~ENETC_F_RX_TSTAMP; 2888 break; 2889 default: 2890 new_offloads |= ENETC_F_RX_TSTAMP; 2891 config.rx_filter = HWTSTAMP_FILTER_ALL; 2892 } 2893 2894 if ((new_offloads ^ priv->active_offloads) & ENETC_F_RX_TSTAMP) { 2895 bool extended = !!(new_offloads & ENETC_F_RX_TSTAMP); 2896 2897 err = enetc_reconfigure(priv, extended, NULL, NULL); 2898 if (err) 2899 return err; 2900 } 2901 2902 priv->active_offloads = new_offloads; 2903 2904 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? 2905 -EFAULT : 0; 2906 } 2907 2908 static int enetc_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr) 2909 { 2910 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2911 struct hwtstamp_config config; 2912 2913 config.flags = 0; 2914 2915 if (priv->active_offloads & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) 2916 config.tx_type = HWTSTAMP_TX_ONESTEP_SYNC; 2917 else if (priv->active_offloads & ENETC_F_TX_TSTAMP) 2918 config.tx_type = HWTSTAMP_TX_ON; 2919 else 2920 config.tx_type = HWTSTAMP_TX_OFF; 2921 2922 config.rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ? 2923 HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE; 2924 2925 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? 2926 -EFAULT : 0; 2927 } 2928 #endif 2929 2930 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd) 2931 { 2932 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2933 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 2934 if (cmd == SIOCSHWTSTAMP) 2935 return enetc_hwtstamp_set(ndev, rq); 2936 if (cmd == SIOCGHWTSTAMP) 2937 return enetc_hwtstamp_get(ndev, rq); 2938 #endif 2939 2940 if (!priv->phylink) 2941 return -EOPNOTSUPP; 2942 2943 return phylink_mii_ioctl(priv->phylink, rq, cmd); 2944 } 2945 EXPORT_SYMBOL_GPL(enetc_ioctl); 2946 2947 int enetc_alloc_msix(struct enetc_ndev_priv *priv) 2948 { 2949 struct pci_dev *pdev = priv->si->pdev; 2950 int num_stack_tx_queues; 2951 int first_xdp_tx_ring; 2952 int i, n, err, nvec; 2953 int v_tx_rings; 2954 2955 nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num; 2956 /* allocate MSIX for both messaging and Rx/Tx interrupts */ 2957 n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX); 2958 2959 if (n < 0) 2960 return n; 2961 2962 if (n != nvec) 2963 return -EPERM; 2964 2965 /* # of tx rings per int vector */ 2966 v_tx_rings = priv->num_tx_rings / priv->bdr_int_num; 2967 2968 for (i = 0; i < priv->bdr_int_num; i++) { 2969 struct enetc_int_vector *v; 2970 struct enetc_bdr *bdr; 2971 int j; 2972 2973 v = kzalloc(struct_size(v, tx_ring, v_tx_rings), GFP_KERNEL); 2974 if (!v) { 2975 err = -ENOMEM; 2976 goto fail; 2977 } 2978 2979 priv->int_vector[i] = v; 2980 2981 bdr = &v->rx_ring; 2982 bdr->index = i; 2983 bdr->ndev = priv->ndev; 2984 bdr->dev = priv->dev; 2985 bdr->bd_count = priv->rx_bd_count; 2986 bdr->buffer_offset = ENETC_RXB_PAD; 2987 priv->rx_ring[i] = bdr; 2988 2989 err = xdp_rxq_info_reg(&bdr->xdp.rxq, priv->ndev, i, 0); 2990 if (err) { 2991 kfree(v); 2992 goto fail; 2993 } 2994 2995 err = xdp_rxq_info_reg_mem_model(&bdr->xdp.rxq, 2996 MEM_TYPE_PAGE_SHARED, NULL); 2997 if (err) { 2998 xdp_rxq_info_unreg(&bdr->xdp.rxq); 2999 kfree(v); 3000 goto fail; 3001 } 3002 3003 /* init defaults for adaptive IC */ 3004 if (priv->ic_mode & ENETC_IC_RX_ADAPTIVE) { 3005 v->rx_ictt = 0x1; 3006 v->rx_dim_en = true; 3007 } 3008 INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work); 3009 netif_napi_add(priv->ndev, &v->napi, enetc_poll); 3010 v->count_tx_rings = v_tx_rings; 3011 3012 for (j = 0; j < v_tx_rings; j++) { 3013 int idx; 3014 3015 /* default tx ring mapping policy */ 3016 idx = priv->bdr_int_num * j + i; 3017 __set_bit(idx, &v->tx_rings_map); 3018 bdr = &v->tx_ring[j]; 3019 bdr->index = idx; 3020 bdr->ndev = priv->ndev; 3021 bdr->dev = priv->dev; 3022 bdr->bd_count = priv->tx_bd_count; 3023 priv->tx_ring[idx] = bdr; 3024 } 3025 } 3026 3027 num_stack_tx_queues = enetc_num_stack_tx_queues(priv); 3028 3029 err = netif_set_real_num_tx_queues(priv->ndev, num_stack_tx_queues); 3030 if (err) 3031 goto fail; 3032 3033 err = netif_set_real_num_rx_queues(priv->ndev, priv->num_rx_rings); 3034 if (err) 3035 goto fail; 3036 3037 priv->min_num_stack_tx_queues = num_possible_cpus(); 3038 first_xdp_tx_ring = priv->num_tx_rings - num_possible_cpus(); 3039 priv->xdp_tx_ring = &priv->tx_ring[first_xdp_tx_ring]; 3040 3041 return 0; 3042 3043 fail: 3044 while (i--) { 3045 struct enetc_int_vector *v = priv->int_vector[i]; 3046 struct enetc_bdr *rx_ring = &v->rx_ring; 3047 3048 xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq); 3049 xdp_rxq_info_unreg(&rx_ring->xdp.rxq); 3050 netif_napi_del(&v->napi); 3051 cancel_work_sync(&v->rx_dim.work); 3052 kfree(v); 3053 } 3054 3055 pci_free_irq_vectors(pdev); 3056 3057 return err; 3058 } 3059 EXPORT_SYMBOL_GPL(enetc_alloc_msix); 3060 3061 void enetc_free_msix(struct enetc_ndev_priv *priv) 3062 { 3063 int i; 3064 3065 for (i = 0; i < priv->bdr_int_num; i++) { 3066 struct enetc_int_vector *v = priv->int_vector[i]; 3067 struct enetc_bdr *rx_ring = &v->rx_ring; 3068 3069 xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq); 3070 xdp_rxq_info_unreg(&rx_ring->xdp.rxq); 3071 netif_napi_del(&v->napi); 3072 cancel_work_sync(&v->rx_dim.work); 3073 } 3074 3075 for (i = 0; i < priv->num_rx_rings; i++) 3076 priv->rx_ring[i] = NULL; 3077 3078 for (i = 0; i < priv->num_tx_rings; i++) 3079 priv->tx_ring[i] = NULL; 3080 3081 for (i = 0; i < priv->bdr_int_num; i++) { 3082 kfree(priv->int_vector[i]); 3083 priv->int_vector[i] = NULL; 3084 } 3085 3086 /* disable all MSIX for this device */ 3087 pci_free_irq_vectors(priv->si->pdev); 3088 } 3089 EXPORT_SYMBOL_GPL(enetc_free_msix); 3090 3091 static void enetc_kfree_si(struct enetc_si *si) 3092 { 3093 char *p = (char *)si - si->pad; 3094 3095 kfree(p); 3096 } 3097 3098 static void enetc_detect_errata(struct enetc_si *si) 3099 { 3100 if (si->pdev->revision == ENETC_REV1) 3101 si->errata = ENETC_ERR_VLAN_ISOL | ENETC_ERR_UCMCSWP; 3102 } 3103 3104 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv) 3105 { 3106 struct enetc_si *si, *p; 3107 struct enetc_hw *hw; 3108 size_t alloc_size; 3109 int err, len; 3110 3111 pcie_flr(pdev); 3112 err = pci_enable_device_mem(pdev); 3113 if (err) 3114 return dev_err_probe(&pdev->dev, err, "device enable failed\n"); 3115 3116 /* set up for high or low dma */ 3117 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 3118 if (err) { 3119 dev_err(&pdev->dev, "DMA configuration failed: 0x%x\n", err); 3120 goto err_dma; 3121 } 3122 3123 err = pci_request_mem_regions(pdev, name); 3124 if (err) { 3125 dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err); 3126 goto err_pci_mem_reg; 3127 } 3128 3129 pci_set_master(pdev); 3130 3131 alloc_size = sizeof(struct enetc_si); 3132 if (sizeof_priv) { 3133 /* align priv to 32B */ 3134 alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN); 3135 alloc_size += sizeof_priv; 3136 } 3137 /* force 32B alignment for enetc_si */ 3138 alloc_size += ENETC_SI_ALIGN - 1; 3139 3140 p = kzalloc(alloc_size, GFP_KERNEL); 3141 if (!p) { 3142 err = -ENOMEM; 3143 goto err_alloc_si; 3144 } 3145 3146 si = PTR_ALIGN(p, ENETC_SI_ALIGN); 3147 si->pad = (char *)si - (char *)p; 3148 3149 pci_set_drvdata(pdev, si); 3150 si->pdev = pdev; 3151 hw = &si->hw; 3152 3153 len = pci_resource_len(pdev, ENETC_BAR_REGS); 3154 hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len); 3155 if (!hw->reg) { 3156 err = -ENXIO; 3157 dev_err(&pdev->dev, "ioremap() failed\n"); 3158 goto err_ioremap; 3159 } 3160 if (len > ENETC_PORT_BASE) 3161 hw->port = hw->reg + ENETC_PORT_BASE; 3162 if (len > ENETC_GLOBAL_BASE) 3163 hw->global = hw->reg + ENETC_GLOBAL_BASE; 3164 3165 enetc_detect_errata(si); 3166 3167 return 0; 3168 3169 err_ioremap: 3170 enetc_kfree_si(si); 3171 err_alloc_si: 3172 pci_release_mem_regions(pdev); 3173 err_pci_mem_reg: 3174 err_dma: 3175 pci_disable_device(pdev); 3176 3177 return err; 3178 } 3179 EXPORT_SYMBOL_GPL(enetc_pci_probe); 3180 3181 void enetc_pci_remove(struct pci_dev *pdev) 3182 { 3183 struct enetc_si *si = pci_get_drvdata(pdev); 3184 struct enetc_hw *hw = &si->hw; 3185 3186 iounmap(hw->reg); 3187 enetc_kfree_si(si); 3188 pci_release_mem_regions(pdev); 3189 pci_disable_device(pdev); 3190 } 3191 EXPORT_SYMBOL_GPL(enetc_pci_remove); 3192 3193 MODULE_LICENSE("Dual BSD/GPL"); 3194