1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Atlantic Network Driver 3 * 4 * Copyright (C) 2014-2019 aQuantia Corporation 5 * Copyright (C) 2019-2020 Marvell International Ltd. 6 */ 7 8 /* File aq_ring.c: Definition of functions for Rx/Tx rings. */ 9 10 #include "aq_nic.h" 11 #include "aq_hw.h" 12 #include "aq_hw_utils.h" 13 #include "aq_ptp.h" 14 #include "aq_vec.h" 15 #include "aq_main.h" 16 17 #include <net/xdp.h> 18 #include <linux/filter.h> 19 #include <linux/bpf_trace.h> 20 #include <linux/netdevice.h> 21 #include <linux/etherdevice.h> 22 23 static void aq_get_rxpages_xdp(struct aq_ring_buff_s *buff, 24 struct xdp_buff *xdp) 25 { 26 struct skb_shared_info *sinfo; 27 int i; 28 29 if (xdp_buff_has_frags(xdp)) { 30 sinfo = xdp_get_shared_info_from_buff(xdp); 31 32 for (i = 0; i < sinfo->nr_frags; i++) { 33 skb_frag_t *frag = &sinfo->frags[i]; 34 35 page_ref_inc(skb_frag_page(frag)); 36 } 37 } 38 page_ref_inc(buff->rxdata.page); 39 } 40 41 static inline void aq_free_rxpage(struct aq_rxpage *rxpage, struct device *dev) 42 { 43 unsigned int len = PAGE_SIZE << rxpage->order; 44 45 dma_unmap_page(dev, rxpage->daddr, len, DMA_FROM_DEVICE); 46 47 /* Drop the ref for being in the ring. */ 48 __free_pages(rxpage->page, rxpage->order); 49 rxpage->page = NULL; 50 } 51 52 static int aq_alloc_rxpages(struct aq_rxpage *rxpage, struct aq_ring_s *rx_ring) 53 { 54 struct device *dev = aq_nic_get_dev(rx_ring->aq_nic); 55 unsigned int order = rx_ring->page_order; 56 struct page *page; 57 int ret = -ENOMEM; 58 dma_addr_t daddr; 59 60 page = dev_alloc_pages(order); 61 if (unlikely(!page)) 62 goto err_exit; 63 64 daddr = dma_map_page(dev, page, 0, PAGE_SIZE << order, 65 DMA_FROM_DEVICE); 66 67 if (unlikely(dma_mapping_error(dev, daddr))) 68 goto free_page; 69 70 rxpage->page = page; 71 rxpage->daddr = daddr; 72 rxpage->order = order; 73 rxpage->pg_off = rx_ring->page_offset; 74 75 return 0; 76 77 free_page: 78 __free_pages(page, order); 79 80 err_exit: 81 return ret; 82 } 83 84 static int aq_get_rxpages(struct aq_ring_s *self, struct aq_ring_buff_s *rxbuf) 85 { 86 unsigned int order = self->page_order; 87 u16 page_offset = self->page_offset; 88 u16 frame_max = self->frame_max; 89 u16 tail_size = self->tail_size; 90 int ret; 91 92 if (rxbuf->rxdata.page) { 93 /* One means ring is the only user and can reuse */ 94 if (page_ref_count(rxbuf->rxdata.page) > 1) { 95 /* Try reuse buffer */ 96 rxbuf->rxdata.pg_off += frame_max + page_offset + 97 tail_size; 98 if (rxbuf->rxdata.pg_off + frame_max + tail_size <= 99 (PAGE_SIZE << order)) { 100 u64_stats_update_begin(&self->stats.rx.syncp); 101 self->stats.rx.pg_flips++; 102 u64_stats_update_end(&self->stats.rx.syncp); 103 104 } else { 105 /* Buffer exhausted. We have other users and 106 * should release this page and realloc 107 */ 108 aq_free_rxpage(&rxbuf->rxdata, 109 aq_nic_get_dev(self->aq_nic)); 110 u64_stats_update_begin(&self->stats.rx.syncp); 111 self->stats.rx.pg_losts++; 112 u64_stats_update_end(&self->stats.rx.syncp); 113 } 114 } else { 115 rxbuf->rxdata.pg_off = page_offset; 116 u64_stats_update_begin(&self->stats.rx.syncp); 117 self->stats.rx.pg_reuses++; 118 u64_stats_update_end(&self->stats.rx.syncp); 119 } 120 } 121 122 if (!rxbuf->rxdata.page) { 123 ret = aq_alloc_rxpages(&rxbuf->rxdata, self); 124 if (ret) { 125 u64_stats_update_begin(&self->stats.rx.syncp); 126 self->stats.rx.alloc_fails++; 127 u64_stats_update_end(&self->stats.rx.syncp); 128 } 129 return ret; 130 } 131 132 return 0; 133 } 134 135 static struct aq_ring_s *aq_ring_alloc(struct aq_ring_s *self, 136 struct aq_nic_s *aq_nic) 137 { 138 int err = 0; 139 140 self->buff_ring = 141 kcalloc(self->size, sizeof(struct aq_ring_buff_s), GFP_KERNEL); 142 143 if (!self->buff_ring) { 144 err = -ENOMEM; 145 goto err_exit; 146 } 147 148 self->dx_ring = dma_alloc_coherent(aq_nic_get_dev(aq_nic), 149 self->size * self->dx_size, 150 &self->dx_ring_pa, GFP_KERNEL); 151 if (!self->dx_ring) { 152 err = -ENOMEM; 153 goto err_exit; 154 } 155 156 err_exit: 157 if (err < 0) { 158 aq_ring_free(self); 159 self = NULL; 160 } 161 162 return self; 163 } 164 165 struct aq_ring_s *aq_ring_tx_alloc(struct aq_ring_s *self, 166 struct aq_nic_s *aq_nic, 167 unsigned int idx, 168 struct aq_nic_cfg_s *aq_nic_cfg) 169 { 170 int err = 0; 171 172 self->aq_nic = aq_nic; 173 self->idx = idx; 174 self->size = aq_nic_cfg->txds; 175 self->dx_size = aq_nic_cfg->aq_hw_caps->txd_size; 176 177 self = aq_ring_alloc(self, aq_nic); 178 if (!self) { 179 err = -ENOMEM; 180 goto err_exit; 181 } 182 183 err_exit: 184 if (err < 0) { 185 aq_ring_free(self); 186 self = NULL; 187 } 188 189 return self; 190 } 191 192 struct aq_ring_s *aq_ring_rx_alloc(struct aq_ring_s *self, 193 struct aq_nic_s *aq_nic, 194 unsigned int idx, 195 struct aq_nic_cfg_s *aq_nic_cfg) 196 { 197 int err = 0; 198 199 self->aq_nic = aq_nic; 200 self->idx = idx; 201 self->size = aq_nic_cfg->rxds; 202 self->dx_size = aq_nic_cfg->aq_hw_caps->rxd_size; 203 self->xdp_prog = aq_nic->xdp_prog; 204 self->frame_max = AQ_CFG_RX_FRAME_MAX; 205 206 /* Only order-2 is allowed if XDP is enabled */ 207 if (READ_ONCE(self->xdp_prog)) { 208 self->page_offset = AQ_XDP_HEADROOM; 209 self->page_order = AQ_CFG_XDP_PAGEORDER; 210 self->tail_size = AQ_XDP_TAILROOM; 211 } else { 212 self->page_offset = 0; 213 self->page_order = fls(self->frame_max / PAGE_SIZE + 214 (self->frame_max % PAGE_SIZE ? 1 : 0)) - 1; 215 if (aq_nic_cfg->rxpageorder > self->page_order) 216 self->page_order = aq_nic_cfg->rxpageorder; 217 self->tail_size = 0; 218 } 219 220 self = aq_ring_alloc(self, aq_nic); 221 if (!self) { 222 err = -ENOMEM; 223 goto err_exit; 224 } 225 226 err_exit: 227 if (err < 0) { 228 aq_ring_free(self); 229 self = NULL; 230 } 231 232 return self; 233 } 234 235 struct aq_ring_s * 236 aq_ring_hwts_rx_alloc(struct aq_ring_s *self, struct aq_nic_s *aq_nic, 237 unsigned int idx, unsigned int size, unsigned int dx_size) 238 { 239 struct device *dev = aq_nic_get_dev(aq_nic); 240 size_t sz = size * dx_size + AQ_CFG_RXDS_DEF; 241 242 memset(self, 0, sizeof(*self)); 243 244 self->aq_nic = aq_nic; 245 self->idx = idx; 246 self->size = size; 247 self->dx_size = dx_size; 248 249 self->dx_ring = dma_alloc_coherent(dev, sz, &self->dx_ring_pa, 250 GFP_KERNEL); 251 if (!self->dx_ring) { 252 aq_ring_free(self); 253 return NULL; 254 } 255 256 return self; 257 } 258 259 int aq_ring_init(struct aq_ring_s *self, const enum atl_ring_type ring_type) 260 { 261 self->hw_head = 0; 262 self->sw_head = 0; 263 self->sw_tail = 0; 264 self->ring_type = ring_type; 265 266 if (self->ring_type == ATL_RING_RX) 267 u64_stats_init(&self->stats.rx.syncp); 268 else 269 u64_stats_init(&self->stats.tx.syncp); 270 271 return 0; 272 } 273 274 static inline bool aq_ring_dx_in_range(unsigned int h, unsigned int i, 275 unsigned int t) 276 { 277 return (h < t) ? ((h < i) && (i < t)) : ((h < i) || (i < t)); 278 } 279 280 void aq_ring_update_queue_state(struct aq_ring_s *ring) 281 { 282 if (aq_ring_avail_dx(ring) <= AQ_CFG_SKB_FRAGS_MAX) 283 aq_ring_queue_stop(ring); 284 else if (aq_ring_avail_dx(ring) > AQ_CFG_RESTART_DESC_THRES) 285 aq_ring_queue_wake(ring); 286 } 287 288 void aq_ring_queue_wake(struct aq_ring_s *ring) 289 { 290 struct net_device *ndev = aq_nic_get_ndev(ring->aq_nic); 291 292 if (__netif_subqueue_stopped(ndev, 293 AQ_NIC_RING2QMAP(ring->aq_nic, 294 ring->idx))) { 295 netif_wake_subqueue(ndev, 296 AQ_NIC_RING2QMAP(ring->aq_nic, ring->idx)); 297 u64_stats_update_begin(&ring->stats.tx.syncp); 298 ring->stats.tx.queue_restarts++; 299 u64_stats_update_end(&ring->stats.tx.syncp); 300 } 301 } 302 303 void aq_ring_queue_stop(struct aq_ring_s *ring) 304 { 305 struct net_device *ndev = aq_nic_get_ndev(ring->aq_nic); 306 307 if (!__netif_subqueue_stopped(ndev, 308 AQ_NIC_RING2QMAP(ring->aq_nic, 309 ring->idx))) 310 netif_stop_subqueue(ndev, 311 AQ_NIC_RING2QMAP(ring->aq_nic, ring->idx)); 312 } 313 314 bool aq_ring_tx_clean(struct aq_ring_s *self) 315 { 316 struct device *dev = aq_nic_get_dev(self->aq_nic); 317 unsigned int budget; 318 319 for (budget = AQ_CFG_TX_CLEAN_BUDGET; 320 budget && self->sw_head != self->hw_head; budget--) { 321 struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head]; 322 323 if (likely(buff->is_mapped)) { 324 if (unlikely(buff->is_sop)) { 325 if (!buff->is_eop && 326 buff->eop_index != 0xffffU && 327 (!aq_ring_dx_in_range(self->sw_head, 328 buff->eop_index, 329 self->hw_head))) 330 break; 331 332 dma_unmap_single(dev, buff->pa, buff->len, 333 DMA_TO_DEVICE); 334 } else { 335 dma_unmap_page(dev, buff->pa, buff->len, 336 DMA_TO_DEVICE); 337 } 338 } 339 340 if (likely(!buff->is_eop)) 341 goto out; 342 343 if (buff->skb) { 344 u64_stats_update_begin(&self->stats.tx.syncp); 345 ++self->stats.tx.packets; 346 self->stats.tx.bytes += buff->skb->len; 347 u64_stats_update_end(&self->stats.tx.syncp); 348 dev_kfree_skb_any(buff->skb); 349 } else if (buff->xdpf) { 350 u64_stats_update_begin(&self->stats.tx.syncp); 351 ++self->stats.tx.packets; 352 self->stats.tx.bytes += xdp_get_frame_len(buff->xdpf); 353 u64_stats_update_end(&self->stats.tx.syncp); 354 xdp_return_frame_rx_napi(buff->xdpf); 355 } 356 357 out: 358 buff->skb = NULL; 359 buff->xdpf = NULL; 360 buff->pa = 0U; 361 buff->eop_index = 0xffffU; 362 self->sw_head = aq_ring_next_dx(self, self->sw_head); 363 } 364 365 return !!budget; 366 } 367 368 static void aq_rx_checksum(struct aq_ring_s *self, 369 struct aq_ring_buff_s *buff, 370 struct sk_buff *skb) 371 { 372 if (!(self->aq_nic->ndev->features & NETIF_F_RXCSUM)) 373 return; 374 375 if (unlikely(buff->is_cso_err)) { 376 u64_stats_update_begin(&self->stats.rx.syncp); 377 ++self->stats.rx.errors; 378 u64_stats_update_end(&self->stats.rx.syncp); 379 skb->ip_summed = CHECKSUM_NONE; 380 return; 381 } 382 if (buff->is_ip_cso) { 383 __skb_incr_checksum_unnecessary(skb); 384 } else { 385 skb->ip_summed = CHECKSUM_NONE; 386 } 387 388 if (buff->is_udp_cso || buff->is_tcp_cso) 389 __skb_incr_checksum_unnecessary(skb); 390 } 391 392 int aq_xdp_xmit(struct net_device *dev, int num_frames, 393 struct xdp_frame **frames, u32 flags) 394 { 395 struct aq_nic_s *aq_nic = netdev_priv(dev); 396 unsigned int vec, i, drop = 0; 397 int cpu = smp_processor_id(); 398 struct aq_nic_cfg_s *aq_cfg; 399 struct aq_ring_s *ring; 400 401 aq_cfg = aq_nic_get_cfg(aq_nic); 402 vec = cpu % aq_cfg->vecs; 403 ring = aq_nic->aq_ring_tx[AQ_NIC_CFG_TCVEC2RING(aq_cfg, 0, vec)]; 404 405 for (i = 0; i < num_frames; i++) { 406 struct xdp_frame *xdpf = frames[i]; 407 408 if (aq_nic_xmit_xdpf(aq_nic, ring, xdpf) == NETDEV_TX_BUSY) 409 drop++; 410 } 411 412 return num_frames - drop; 413 } 414 415 static struct sk_buff *aq_xdp_build_skb(struct xdp_buff *xdp, 416 struct net_device *dev, 417 struct aq_ring_buff_s *buff) 418 { 419 struct xdp_frame *xdpf; 420 struct sk_buff *skb; 421 422 xdpf = xdp_convert_buff_to_frame(xdp); 423 if (unlikely(!xdpf)) 424 return NULL; 425 426 skb = xdp_build_skb_from_frame(xdpf, dev); 427 if (!skb) 428 return NULL; 429 430 aq_get_rxpages_xdp(buff, xdp); 431 return skb; 432 } 433 434 static struct sk_buff *aq_xdp_run_prog(struct aq_nic_s *aq_nic, 435 struct xdp_buff *xdp, 436 struct aq_ring_s *rx_ring, 437 struct aq_ring_buff_s *buff) 438 { 439 int result = NETDEV_TX_BUSY; 440 struct aq_ring_s *tx_ring; 441 struct xdp_frame *xdpf; 442 struct bpf_prog *prog; 443 u32 act = XDP_ABORTED; 444 struct sk_buff *skb; 445 446 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 447 ++rx_ring->stats.rx.packets; 448 rx_ring->stats.rx.bytes += xdp_get_buff_len(xdp); 449 u64_stats_update_end(&rx_ring->stats.rx.syncp); 450 451 prog = READ_ONCE(rx_ring->xdp_prog); 452 if (!prog) 453 return aq_xdp_build_skb(xdp, aq_nic->ndev, buff); 454 455 prefetchw(xdp->data_hard_start); /* xdp_frame write */ 456 457 /* single buffer XDP program, but packet is multi buffer, aborted */ 458 if (xdp_buff_has_frags(xdp) && !prog->aux->xdp_has_frags) 459 goto out_aborted; 460 461 act = bpf_prog_run_xdp(prog, xdp); 462 switch (act) { 463 case XDP_PASS: 464 skb = aq_xdp_build_skb(xdp, aq_nic->ndev, buff); 465 if (!skb) 466 goto out_aborted; 467 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 468 ++rx_ring->stats.rx.xdp_pass; 469 u64_stats_update_end(&rx_ring->stats.rx.syncp); 470 return skb; 471 case XDP_TX: 472 xdpf = xdp_convert_buff_to_frame(xdp); 473 if (unlikely(!xdpf)) 474 goto out_aborted; 475 tx_ring = aq_nic->aq_ring_tx[rx_ring->idx]; 476 result = aq_nic_xmit_xdpf(aq_nic, tx_ring, xdpf); 477 if (result == NETDEV_TX_BUSY) 478 goto out_aborted; 479 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 480 ++rx_ring->stats.rx.xdp_tx; 481 u64_stats_update_end(&rx_ring->stats.rx.syncp); 482 aq_get_rxpages_xdp(buff, xdp); 483 break; 484 case XDP_REDIRECT: 485 if (xdp_do_redirect(aq_nic->ndev, xdp, prog) < 0) 486 goto out_aborted; 487 xdp_do_flush(); 488 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 489 ++rx_ring->stats.rx.xdp_redirect; 490 u64_stats_update_end(&rx_ring->stats.rx.syncp); 491 aq_get_rxpages_xdp(buff, xdp); 492 break; 493 default: 494 fallthrough; 495 case XDP_ABORTED: 496 out_aborted: 497 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 498 ++rx_ring->stats.rx.xdp_aborted; 499 u64_stats_update_end(&rx_ring->stats.rx.syncp); 500 trace_xdp_exception(aq_nic->ndev, prog, act); 501 bpf_warn_invalid_xdp_action(aq_nic->ndev, prog, act); 502 break; 503 case XDP_DROP: 504 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 505 ++rx_ring->stats.rx.xdp_drop; 506 u64_stats_update_end(&rx_ring->stats.rx.syncp); 507 break; 508 } 509 510 return ERR_PTR(-result); 511 } 512 513 static bool aq_add_rx_fragment(struct device *dev, 514 struct aq_ring_s *ring, 515 struct aq_ring_buff_s *buff, 516 struct xdp_buff *xdp) 517 { 518 struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); 519 struct aq_ring_buff_s *buff_ = buff; 520 521 memset(sinfo, 0, sizeof(*sinfo)); 522 do { 523 skb_frag_t *frag; 524 525 if (unlikely(sinfo->nr_frags >= MAX_SKB_FRAGS)) 526 return true; 527 528 frag = &sinfo->frags[sinfo->nr_frags++]; 529 buff_ = &ring->buff_ring[buff_->next]; 530 dma_sync_single_range_for_cpu(dev, 531 buff_->rxdata.daddr, 532 buff_->rxdata.pg_off, 533 buff_->len, 534 DMA_FROM_DEVICE); 535 sinfo->xdp_frags_size += buff_->len; 536 skb_frag_fill_page_desc(frag, buff_->rxdata.page, 537 buff_->rxdata.pg_off, 538 buff_->len); 539 540 buff_->is_cleaned = 1; 541 542 buff->is_ip_cso &= buff_->is_ip_cso; 543 buff->is_udp_cso &= buff_->is_udp_cso; 544 buff->is_tcp_cso &= buff_->is_tcp_cso; 545 buff->is_cso_err |= buff_->is_cso_err; 546 547 if (page_is_pfmemalloc(buff_->rxdata.page)) 548 xdp_buff_set_frag_pfmemalloc(xdp); 549 550 } while (!buff_->is_eop); 551 552 xdp_buff_set_frags_flag(xdp); 553 554 return false; 555 } 556 557 static int __aq_ring_rx_clean(struct aq_ring_s *self, struct napi_struct *napi, 558 int *work_done, int budget) 559 { 560 struct net_device *ndev = aq_nic_get_ndev(self->aq_nic); 561 int err = 0; 562 563 for (; (self->sw_head != self->hw_head) && budget; 564 self->sw_head = aq_ring_next_dx(self, self->sw_head), 565 --budget, ++(*work_done)) { 566 struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head]; 567 bool is_ptp_ring = aq_ptp_ring(self->aq_nic, self); 568 struct aq_ring_buff_s *buff_ = NULL; 569 struct sk_buff *skb = NULL; 570 unsigned int next_ = 0U; 571 unsigned int i = 0U; 572 u16 hdr_len; 573 574 if (buff->is_cleaned) 575 continue; 576 577 if (!buff->is_eop) { 578 unsigned int frag_cnt = 0U; 579 buff_ = buff; 580 do { 581 bool is_rsc_completed = true; 582 583 if (buff_->next >= self->size) { 584 err = -EIO; 585 goto err_exit; 586 } 587 588 frag_cnt++; 589 next_ = buff_->next, 590 buff_ = &self->buff_ring[next_]; 591 is_rsc_completed = 592 aq_ring_dx_in_range(self->sw_head, 593 next_, 594 self->hw_head); 595 596 if (unlikely(!is_rsc_completed) || 597 frag_cnt > MAX_SKB_FRAGS) { 598 err = 0; 599 goto err_exit; 600 } 601 602 buff->is_error |= buff_->is_error; 603 buff->is_cso_err |= buff_->is_cso_err; 604 605 } while (!buff_->is_eop); 606 607 if (buff->is_error || 608 (buff->is_lro && buff->is_cso_err)) { 609 buff_ = buff; 610 do { 611 if (buff_->next >= self->size) { 612 err = -EIO; 613 goto err_exit; 614 } 615 next_ = buff_->next, 616 buff_ = &self->buff_ring[next_]; 617 618 buff_->is_cleaned = true; 619 } while (!buff_->is_eop); 620 621 u64_stats_update_begin(&self->stats.rx.syncp); 622 ++self->stats.rx.errors; 623 u64_stats_update_end(&self->stats.rx.syncp); 624 continue; 625 } 626 } 627 628 if (buff->is_error) { 629 u64_stats_update_begin(&self->stats.rx.syncp); 630 ++self->stats.rx.errors; 631 u64_stats_update_end(&self->stats.rx.syncp); 632 continue; 633 } 634 635 dma_sync_single_range_for_cpu(aq_nic_get_dev(self->aq_nic), 636 buff->rxdata.daddr, 637 buff->rxdata.pg_off, 638 buff->len, DMA_FROM_DEVICE); 639 640 skb = napi_alloc_skb(napi, AQ_CFG_RX_HDR_SIZE); 641 if (unlikely(!skb)) { 642 u64_stats_update_begin(&self->stats.rx.syncp); 643 self->stats.rx.skb_alloc_fails++; 644 u64_stats_update_end(&self->stats.rx.syncp); 645 err = -ENOMEM; 646 goto err_exit; 647 } 648 if (is_ptp_ring) 649 buff->len -= 650 aq_ptp_extract_ts(self->aq_nic, skb_hwtstamps(skb), 651 aq_buf_vaddr(&buff->rxdata), 652 buff->len); 653 654 hdr_len = buff->len; 655 if (hdr_len > AQ_CFG_RX_HDR_SIZE) 656 hdr_len = eth_get_headlen(skb->dev, 657 aq_buf_vaddr(&buff->rxdata), 658 AQ_CFG_RX_HDR_SIZE); 659 660 memcpy(__skb_put(skb, hdr_len), aq_buf_vaddr(&buff->rxdata), 661 ALIGN(hdr_len, sizeof(long))); 662 663 if (buff->len - hdr_len > 0) { 664 skb_add_rx_frag(skb, i++, buff->rxdata.page, 665 buff->rxdata.pg_off + hdr_len, 666 buff->len - hdr_len, 667 self->frame_max); 668 page_ref_inc(buff->rxdata.page); 669 } 670 671 if (!buff->is_eop) { 672 buff_ = buff; 673 do { 674 next_ = buff_->next; 675 buff_ = &self->buff_ring[next_]; 676 677 dma_sync_single_range_for_cpu(aq_nic_get_dev(self->aq_nic), 678 buff_->rxdata.daddr, 679 buff_->rxdata.pg_off, 680 buff_->len, 681 DMA_FROM_DEVICE); 682 skb_add_rx_frag(skb, i++, 683 buff_->rxdata.page, 684 buff_->rxdata.pg_off, 685 buff_->len, 686 self->frame_max); 687 page_ref_inc(buff_->rxdata.page); 688 buff_->is_cleaned = 1; 689 690 buff->is_ip_cso &= buff_->is_ip_cso; 691 buff->is_udp_cso &= buff_->is_udp_cso; 692 buff->is_tcp_cso &= buff_->is_tcp_cso; 693 buff->is_cso_err |= buff_->is_cso_err; 694 695 } while (!buff_->is_eop); 696 } 697 698 if (buff->is_vlan) 699 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 700 buff->vlan_rx_tag); 701 702 skb->protocol = eth_type_trans(skb, ndev); 703 704 aq_rx_checksum(self, buff, skb); 705 706 skb_set_hash(skb, buff->rss_hash, 707 buff->is_hash_l4 ? PKT_HASH_TYPE_L4 : 708 PKT_HASH_TYPE_NONE); 709 /* Send all PTP traffic to 0 queue */ 710 skb_record_rx_queue(skb, 711 is_ptp_ring ? 0 712 : AQ_NIC_RING2QMAP(self->aq_nic, 713 self->idx)); 714 715 u64_stats_update_begin(&self->stats.rx.syncp); 716 ++self->stats.rx.packets; 717 self->stats.rx.bytes += skb->len; 718 u64_stats_update_end(&self->stats.rx.syncp); 719 720 napi_gro_receive(napi, skb); 721 } 722 723 err_exit: 724 return err; 725 } 726 727 static int __aq_ring_xdp_clean(struct aq_ring_s *rx_ring, 728 struct napi_struct *napi, int *work_done, 729 int budget) 730 { 731 int frame_sz = rx_ring->page_offset + rx_ring->frame_max + 732 rx_ring->tail_size; 733 struct aq_nic_s *aq_nic = rx_ring->aq_nic; 734 bool is_rsc_completed = true; 735 struct device *dev; 736 int err = 0; 737 738 dev = aq_nic_get_dev(aq_nic); 739 for (; (rx_ring->sw_head != rx_ring->hw_head) && budget; 740 rx_ring->sw_head = aq_ring_next_dx(rx_ring, rx_ring->sw_head), 741 --budget, ++(*work_done)) { 742 struct aq_ring_buff_s *buff = &rx_ring->buff_ring[rx_ring->sw_head]; 743 bool is_ptp_ring = aq_ptp_ring(rx_ring->aq_nic, rx_ring); 744 struct aq_ring_buff_s *buff_ = NULL; 745 u16 ptp_hwtstamp_len = 0; 746 struct skb_shared_hwtstamps shhwtstamps; 747 struct sk_buff *skb = NULL; 748 unsigned int next_ = 0U; 749 struct xdp_buff xdp; 750 void *hard_start; 751 752 if (buff->is_cleaned) 753 continue; 754 755 if (!buff->is_eop) { 756 buff_ = buff; 757 do { 758 if (buff_->next >= rx_ring->size) { 759 err = -EIO; 760 goto err_exit; 761 } 762 next_ = buff_->next; 763 buff_ = &rx_ring->buff_ring[next_]; 764 is_rsc_completed = 765 aq_ring_dx_in_range(rx_ring->sw_head, 766 next_, 767 rx_ring->hw_head); 768 769 if (unlikely(!is_rsc_completed)) 770 break; 771 772 buff->is_error |= buff_->is_error; 773 buff->is_cso_err |= buff_->is_cso_err; 774 } while (!buff_->is_eop); 775 776 if (!is_rsc_completed) { 777 err = 0; 778 goto err_exit; 779 } 780 if (buff->is_error || 781 (buff->is_lro && buff->is_cso_err)) { 782 buff_ = buff; 783 do { 784 if (buff_->next >= rx_ring->size) { 785 err = -EIO; 786 goto err_exit; 787 } 788 next_ = buff_->next; 789 buff_ = &rx_ring->buff_ring[next_]; 790 791 buff_->is_cleaned = true; 792 } while (!buff_->is_eop); 793 794 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 795 ++rx_ring->stats.rx.errors; 796 u64_stats_update_end(&rx_ring->stats.rx.syncp); 797 continue; 798 } 799 } 800 801 if (buff->is_error) { 802 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 803 ++rx_ring->stats.rx.errors; 804 u64_stats_update_end(&rx_ring->stats.rx.syncp); 805 continue; 806 } 807 808 dma_sync_single_range_for_cpu(dev, 809 buff->rxdata.daddr, 810 buff->rxdata.pg_off, 811 buff->len, DMA_FROM_DEVICE); 812 hard_start = page_address(buff->rxdata.page) + 813 buff->rxdata.pg_off - rx_ring->page_offset; 814 815 if (is_ptp_ring) { 816 ptp_hwtstamp_len = aq_ptp_extract_ts(rx_ring->aq_nic, &shhwtstamps, 817 aq_buf_vaddr(&buff->rxdata), 818 buff->len); 819 buff->len -= ptp_hwtstamp_len; 820 } 821 822 xdp_init_buff(&xdp, frame_sz, &rx_ring->xdp_rxq); 823 xdp_prepare_buff(&xdp, hard_start, rx_ring->page_offset, 824 buff->len, false); 825 if (!buff->is_eop) { 826 if (aq_add_rx_fragment(dev, rx_ring, buff, &xdp)) { 827 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 828 ++rx_ring->stats.rx.packets; 829 rx_ring->stats.rx.bytes += xdp_get_buff_len(&xdp); 830 ++rx_ring->stats.rx.xdp_aborted; 831 u64_stats_update_end(&rx_ring->stats.rx.syncp); 832 continue; 833 } 834 } 835 836 skb = aq_xdp_run_prog(aq_nic, &xdp, rx_ring, buff); 837 if (IS_ERR(skb) || !skb) 838 continue; 839 840 if (ptp_hwtstamp_len > 0) 841 *skb_hwtstamps(skb) = shhwtstamps; 842 843 if (buff->is_vlan) 844 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 845 buff->vlan_rx_tag); 846 847 aq_rx_checksum(rx_ring, buff, skb); 848 849 skb_set_hash(skb, buff->rss_hash, 850 buff->is_hash_l4 ? PKT_HASH_TYPE_L4 : 851 PKT_HASH_TYPE_NONE); 852 /* Send all PTP traffic to 0 queue */ 853 skb_record_rx_queue(skb, 854 is_ptp_ring ? 0 855 : AQ_NIC_RING2QMAP(rx_ring->aq_nic, 856 rx_ring->idx)); 857 858 napi_gro_receive(napi, skb); 859 } 860 861 err_exit: 862 return err; 863 } 864 865 int aq_ring_rx_clean(struct aq_ring_s *self, 866 struct napi_struct *napi, 867 int *work_done, 868 int budget) 869 { 870 if (static_branch_unlikely(&aq_xdp_locking_key)) 871 return __aq_ring_xdp_clean(self, napi, work_done, budget); 872 else 873 return __aq_ring_rx_clean(self, napi, work_done, budget); 874 } 875 876 void aq_ring_hwts_rx_clean(struct aq_ring_s *self, struct aq_nic_s *aq_nic) 877 { 878 #if IS_REACHABLE(CONFIG_PTP_1588_CLOCK) 879 while (self->sw_head != self->hw_head) { 880 u64 ns; 881 882 aq_nic->aq_hw_ops->extract_hwts(aq_nic->aq_hw, 883 self->dx_ring + 884 (self->sw_head * self->dx_size), 885 self->dx_size, &ns); 886 aq_ptp_tx_hwtstamp(aq_nic, ns); 887 888 self->sw_head = aq_ring_next_dx(self, self->sw_head); 889 } 890 #endif 891 } 892 893 int aq_ring_rx_fill(struct aq_ring_s *self) 894 { 895 struct aq_ring_buff_s *buff = NULL; 896 int err = 0; 897 int i = 0; 898 899 if (aq_ring_avail_dx(self) < min_t(unsigned int, AQ_CFG_RX_REFILL_THRES, 900 self->size / 2)) 901 return err; 902 903 for (i = aq_ring_avail_dx(self); i--; 904 self->sw_tail = aq_ring_next_dx(self, self->sw_tail)) { 905 buff = &self->buff_ring[self->sw_tail]; 906 907 buff->flags = 0U; 908 buff->len = self->frame_max; 909 910 err = aq_get_rxpages(self, buff); 911 if (err) 912 goto err_exit; 913 914 buff->pa = aq_buf_daddr(&buff->rxdata); 915 buff = NULL; 916 } 917 918 err_exit: 919 return err; 920 } 921 922 void aq_ring_rx_deinit(struct aq_ring_s *self) 923 { 924 if (!self) 925 return; 926 927 for (; self->sw_head != self->sw_tail; 928 self->sw_head = aq_ring_next_dx(self, self->sw_head)) { 929 struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head]; 930 931 aq_free_rxpage(&buff->rxdata, aq_nic_get_dev(self->aq_nic)); 932 } 933 } 934 935 void aq_ring_free(struct aq_ring_s *self) 936 { 937 if (!self) 938 return; 939 940 kfree(self->buff_ring); 941 942 if (self->dx_ring) 943 dma_free_coherent(aq_nic_get_dev(self->aq_nic), 944 self->size * self->dx_size, self->dx_ring, 945 self->dx_ring_pa); 946 } 947 948 unsigned int aq_ring_fill_stats_data(struct aq_ring_s *self, u64 *data) 949 { 950 unsigned int count; 951 unsigned int start; 952 953 if (self->ring_type == ATL_RING_RX) { 954 /* This data should mimic aq_ethtool_queue_rx_stat_names structure */ 955 do { 956 count = 0; 957 start = u64_stats_fetch_begin(&self->stats.rx.syncp); 958 data[count] = self->stats.rx.packets; 959 data[++count] = self->stats.rx.jumbo_packets; 960 data[++count] = self->stats.rx.lro_packets; 961 data[++count] = self->stats.rx.errors; 962 data[++count] = self->stats.rx.alloc_fails; 963 data[++count] = self->stats.rx.skb_alloc_fails; 964 data[++count] = self->stats.rx.polls; 965 data[++count] = self->stats.rx.pg_flips; 966 data[++count] = self->stats.rx.pg_reuses; 967 data[++count] = self->stats.rx.pg_losts; 968 data[++count] = self->stats.rx.xdp_aborted; 969 data[++count] = self->stats.rx.xdp_drop; 970 data[++count] = self->stats.rx.xdp_pass; 971 data[++count] = self->stats.rx.xdp_tx; 972 data[++count] = self->stats.rx.xdp_invalid; 973 data[++count] = self->stats.rx.xdp_redirect; 974 } while (u64_stats_fetch_retry(&self->stats.rx.syncp, start)); 975 } else { 976 /* This data should mimic aq_ethtool_queue_tx_stat_names structure */ 977 do { 978 count = 0; 979 start = u64_stats_fetch_begin(&self->stats.tx.syncp); 980 data[count] = self->stats.tx.packets; 981 data[++count] = self->stats.tx.queue_restarts; 982 } while (u64_stats_fetch_retry(&self->stats.tx.syncp, start)); 983 } 984 985 return ++count; 986 } 987