1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Atlantic Network Driver 3 * 4 * Copyright (C) 2014-2019 aQuantia Corporation 5 * Copyright (C) 2019-2020 Marvell International Ltd. 6 */ 7 8 /* File aq_ring.c: Definition of functions for Rx/Tx rings. */ 9 10 #include "aq_nic.h" 11 #include "aq_hw.h" 12 #include "aq_hw_utils.h" 13 #include "aq_ptp.h" 14 #include "aq_vec.h" 15 #include "aq_main.h" 16 17 #include <net/xdp.h> 18 #include <linux/filter.h> 19 #include <linux/bpf_trace.h> 20 #include <linux/netdevice.h> 21 #include <linux/etherdevice.h> 22 23 static void aq_get_rxpages_xdp(struct aq_ring_buff_s *buff, 24 struct xdp_buff *xdp) 25 { 26 struct skb_shared_info *sinfo; 27 int i; 28 29 if (xdp_buff_has_frags(xdp)) { 30 sinfo = xdp_get_shared_info_from_buff(xdp); 31 32 for (i = 0; i < sinfo->nr_frags; i++) { 33 skb_frag_t *frag = &sinfo->frags[i]; 34 35 page_ref_inc(skb_frag_page(frag)); 36 } 37 } 38 page_ref_inc(buff->rxdata.page); 39 } 40 41 static inline void aq_free_rxpage(struct aq_rxpage *rxpage, struct device *dev) 42 { 43 unsigned int len = PAGE_SIZE << rxpage->order; 44 45 dma_unmap_page(dev, rxpage->daddr, len, DMA_FROM_DEVICE); 46 47 /* Drop the ref for being in the ring. */ 48 __free_pages(rxpage->page, rxpage->order); 49 rxpage->page = NULL; 50 } 51 52 static int aq_alloc_rxpages(struct aq_rxpage *rxpage, struct aq_ring_s *rx_ring) 53 { 54 struct device *dev = aq_nic_get_dev(rx_ring->aq_nic); 55 unsigned int order = rx_ring->page_order; 56 struct page *page; 57 int ret = -ENOMEM; 58 dma_addr_t daddr; 59 60 page = dev_alloc_pages(order); 61 if (unlikely(!page)) 62 goto err_exit; 63 64 daddr = dma_map_page(dev, page, 0, PAGE_SIZE << order, 65 DMA_FROM_DEVICE); 66 67 if (unlikely(dma_mapping_error(dev, daddr))) 68 goto free_page; 69 70 rxpage->page = page; 71 rxpage->daddr = daddr; 72 rxpage->order = order; 73 rxpage->pg_off = rx_ring->page_offset; 74 75 return 0; 76 77 free_page: 78 __free_pages(page, order); 79 80 err_exit: 81 return ret; 82 } 83 84 static int aq_get_rxpages(struct aq_ring_s *self, struct aq_ring_buff_s *rxbuf) 85 { 86 unsigned int order = self->page_order; 87 u16 page_offset = self->page_offset; 88 u16 frame_max = self->frame_max; 89 u16 tail_size = self->tail_size; 90 int ret; 91 92 if (rxbuf->rxdata.page) { 93 /* One means ring is the only user and can reuse */ 94 if (page_ref_count(rxbuf->rxdata.page) > 1) { 95 /* Try reuse buffer */ 96 rxbuf->rxdata.pg_off += frame_max + page_offset + 97 tail_size; 98 if (rxbuf->rxdata.pg_off + frame_max + tail_size <= 99 (PAGE_SIZE << order)) { 100 u64_stats_update_begin(&self->stats.rx.syncp); 101 self->stats.rx.pg_flips++; 102 u64_stats_update_end(&self->stats.rx.syncp); 103 104 } else { 105 /* Buffer exhausted. We have other users and 106 * should release this page and realloc 107 */ 108 aq_free_rxpage(&rxbuf->rxdata, 109 aq_nic_get_dev(self->aq_nic)); 110 u64_stats_update_begin(&self->stats.rx.syncp); 111 self->stats.rx.pg_losts++; 112 u64_stats_update_end(&self->stats.rx.syncp); 113 } 114 } else { 115 rxbuf->rxdata.pg_off = page_offset; 116 u64_stats_update_begin(&self->stats.rx.syncp); 117 self->stats.rx.pg_reuses++; 118 u64_stats_update_end(&self->stats.rx.syncp); 119 } 120 } 121 122 if (!rxbuf->rxdata.page) { 123 ret = aq_alloc_rxpages(&rxbuf->rxdata, self); 124 if (ret) { 125 u64_stats_update_begin(&self->stats.rx.syncp); 126 self->stats.rx.alloc_fails++; 127 u64_stats_update_end(&self->stats.rx.syncp); 128 } 129 return ret; 130 } 131 132 return 0; 133 } 134 135 static struct aq_ring_s *aq_ring_alloc(struct aq_ring_s *self, 136 struct aq_nic_s *aq_nic) 137 { 138 int err = 0; 139 140 self->buff_ring = 141 kcalloc(self->size, sizeof(struct aq_ring_buff_s), GFP_KERNEL); 142 143 if (!self->buff_ring) { 144 err = -ENOMEM; 145 goto err_exit; 146 } 147 148 self->dx_ring = dma_alloc_coherent(aq_nic_get_dev(aq_nic), 149 self->size * self->dx_size, 150 &self->dx_ring_pa, GFP_KERNEL); 151 if (!self->dx_ring) { 152 err = -ENOMEM; 153 goto err_exit; 154 } 155 156 err_exit: 157 if (err < 0) { 158 aq_ring_free(self); 159 self = NULL; 160 } 161 162 return self; 163 } 164 165 struct aq_ring_s *aq_ring_tx_alloc(struct aq_ring_s *self, 166 struct aq_nic_s *aq_nic, 167 unsigned int idx, 168 struct aq_nic_cfg_s *aq_nic_cfg) 169 { 170 int err = 0; 171 172 self->aq_nic = aq_nic; 173 self->idx = idx; 174 self->size = aq_nic_cfg->txds; 175 self->dx_size = aq_nic_cfg->aq_hw_caps->txd_size; 176 177 self = aq_ring_alloc(self, aq_nic); 178 if (!self) { 179 err = -ENOMEM; 180 goto err_exit; 181 } 182 183 err_exit: 184 if (err < 0) { 185 aq_ring_free(self); 186 self = NULL; 187 } 188 189 return self; 190 } 191 192 struct aq_ring_s *aq_ring_rx_alloc(struct aq_ring_s *self, 193 struct aq_nic_s *aq_nic, 194 unsigned int idx, 195 struct aq_nic_cfg_s *aq_nic_cfg) 196 { 197 int err = 0; 198 199 self->aq_nic = aq_nic; 200 self->idx = idx; 201 self->size = aq_nic_cfg->rxds; 202 self->dx_size = aq_nic_cfg->aq_hw_caps->rxd_size; 203 self->xdp_prog = aq_nic->xdp_prog; 204 self->frame_max = AQ_CFG_RX_FRAME_MAX; 205 206 /* Only order-2 is allowed if XDP is enabled */ 207 if (READ_ONCE(self->xdp_prog)) { 208 self->page_offset = AQ_XDP_HEADROOM; 209 self->page_order = AQ_CFG_XDP_PAGEORDER; 210 self->tail_size = AQ_XDP_TAILROOM; 211 } else { 212 self->page_offset = 0; 213 self->page_order = fls(self->frame_max / PAGE_SIZE + 214 (self->frame_max % PAGE_SIZE ? 1 : 0)) - 1; 215 if (aq_nic_cfg->rxpageorder > self->page_order) 216 self->page_order = aq_nic_cfg->rxpageorder; 217 self->tail_size = 0; 218 } 219 220 self = aq_ring_alloc(self, aq_nic); 221 if (!self) { 222 err = -ENOMEM; 223 goto err_exit; 224 } 225 226 err_exit: 227 if (err < 0) { 228 aq_ring_free(self); 229 self = NULL; 230 } 231 232 return self; 233 } 234 235 struct aq_ring_s * 236 aq_ring_hwts_rx_alloc(struct aq_ring_s *self, struct aq_nic_s *aq_nic, 237 unsigned int idx, unsigned int size, unsigned int dx_size) 238 { 239 struct device *dev = aq_nic_get_dev(aq_nic); 240 size_t sz = size * dx_size + AQ_CFG_RXDS_DEF; 241 242 memset(self, 0, sizeof(*self)); 243 244 self->aq_nic = aq_nic; 245 self->idx = idx; 246 self->size = size; 247 self->dx_size = dx_size; 248 249 self->dx_ring = dma_alloc_coherent(dev, sz, &self->dx_ring_pa, 250 GFP_KERNEL); 251 if (!self->dx_ring) { 252 aq_ring_free(self); 253 return NULL; 254 } 255 256 return self; 257 } 258 259 int aq_ring_init(struct aq_ring_s *self, const enum atl_ring_type ring_type) 260 { 261 self->hw_head = 0; 262 self->sw_head = 0; 263 self->sw_tail = 0; 264 self->ring_type = ring_type; 265 266 if (self->ring_type == ATL_RING_RX) 267 u64_stats_init(&self->stats.rx.syncp); 268 else 269 u64_stats_init(&self->stats.tx.syncp); 270 271 return 0; 272 } 273 274 static inline bool aq_ring_dx_in_range(unsigned int h, unsigned int i, 275 unsigned int t) 276 { 277 return (h < t) ? ((h < i) && (i < t)) : ((h < i) || (i < t)); 278 } 279 280 void aq_ring_update_queue_state(struct aq_ring_s *ring) 281 { 282 if (aq_ring_avail_dx(ring) <= AQ_CFG_SKB_FRAGS_MAX) 283 aq_ring_queue_stop(ring); 284 else if (aq_ring_avail_dx(ring) > AQ_CFG_RESTART_DESC_THRES) 285 aq_ring_queue_wake(ring); 286 } 287 288 void aq_ring_queue_wake(struct aq_ring_s *ring) 289 { 290 struct net_device *ndev = aq_nic_get_ndev(ring->aq_nic); 291 292 if (__netif_subqueue_stopped(ndev, 293 AQ_NIC_RING2QMAP(ring->aq_nic, 294 ring->idx))) { 295 netif_wake_subqueue(ndev, 296 AQ_NIC_RING2QMAP(ring->aq_nic, ring->idx)); 297 u64_stats_update_begin(&ring->stats.tx.syncp); 298 ring->stats.tx.queue_restarts++; 299 u64_stats_update_end(&ring->stats.tx.syncp); 300 } 301 } 302 303 void aq_ring_queue_stop(struct aq_ring_s *ring) 304 { 305 struct net_device *ndev = aq_nic_get_ndev(ring->aq_nic); 306 307 if (!__netif_subqueue_stopped(ndev, 308 AQ_NIC_RING2QMAP(ring->aq_nic, 309 ring->idx))) 310 netif_stop_subqueue(ndev, 311 AQ_NIC_RING2QMAP(ring->aq_nic, ring->idx)); 312 } 313 314 bool aq_ring_tx_clean(struct aq_ring_s *self) 315 { 316 struct device *dev = aq_nic_get_dev(self->aq_nic); 317 unsigned int budget; 318 319 for (budget = AQ_CFG_TX_CLEAN_BUDGET; 320 budget && self->sw_head != self->hw_head; budget--) { 321 struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head]; 322 323 if (likely(buff->is_mapped)) { 324 if (unlikely(buff->is_sop)) { 325 if (!buff->is_eop && 326 buff->eop_index != 0xffffU && 327 (!aq_ring_dx_in_range(self->sw_head, 328 buff->eop_index, 329 self->hw_head))) 330 break; 331 332 dma_unmap_single(dev, buff->pa, buff->len, 333 DMA_TO_DEVICE); 334 } else { 335 dma_unmap_page(dev, buff->pa, buff->len, 336 DMA_TO_DEVICE); 337 } 338 } 339 340 if (likely(!buff->is_eop)) 341 goto out; 342 343 if (buff->skb) { 344 u64_stats_update_begin(&self->stats.tx.syncp); 345 ++self->stats.tx.packets; 346 self->stats.tx.bytes += buff->skb->len; 347 u64_stats_update_end(&self->stats.tx.syncp); 348 dev_kfree_skb_any(buff->skb); 349 } else if (buff->xdpf) { 350 u64_stats_update_begin(&self->stats.tx.syncp); 351 ++self->stats.tx.packets; 352 self->stats.tx.bytes += xdp_get_frame_len(buff->xdpf); 353 u64_stats_update_end(&self->stats.tx.syncp); 354 xdp_return_frame_rx_napi(buff->xdpf); 355 } 356 357 out: 358 buff->skb = NULL; 359 buff->xdpf = NULL; 360 buff->pa = 0U; 361 buff->eop_index = 0xffffU; 362 self->sw_head = aq_ring_next_dx(self, self->sw_head); 363 } 364 365 return !!budget; 366 } 367 368 static void aq_rx_checksum(struct aq_ring_s *self, 369 struct aq_ring_buff_s *buff, 370 struct sk_buff *skb) 371 { 372 if (!(self->aq_nic->ndev->features & NETIF_F_RXCSUM)) 373 return; 374 375 if (unlikely(buff->is_cso_err)) { 376 u64_stats_update_begin(&self->stats.rx.syncp); 377 ++self->stats.rx.errors; 378 u64_stats_update_end(&self->stats.rx.syncp); 379 skb->ip_summed = CHECKSUM_NONE; 380 return; 381 } 382 if (buff->is_ip_cso) { 383 __skb_incr_checksum_unnecessary(skb); 384 } else { 385 skb->ip_summed = CHECKSUM_NONE; 386 } 387 388 if (buff->is_udp_cso || buff->is_tcp_cso) 389 __skb_incr_checksum_unnecessary(skb); 390 } 391 392 int aq_xdp_xmit(struct net_device *dev, int num_frames, 393 struct xdp_frame **frames, u32 flags) 394 { 395 struct aq_nic_s *aq_nic = netdev_priv(dev); 396 unsigned int vec, i, drop = 0; 397 int cpu = smp_processor_id(); 398 struct aq_nic_cfg_s *aq_cfg; 399 struct aq_ring_s *ring; 400 401 aq_cfg = aq_nic_get_cfg(aq_nic); 402 vec = cpu % aq_cfg->vecs; 403 ring = aq_nic->aq_ring_tx[AQ_NIC_CFG_TCVEC2RING(aq_cfg, 0, vec)]; 404 405 for (i = 0; i < num_frames; i++) { 406 struct xdp_frame *xdpf = frames[i]; 407 408 if (aq_nic_xmit_xdpf(aq_nic, ring, xdpf) == NETDEV_TX_BUSY) 409 drop++; 410 } 411 412 return num_frames - drop; 413 } 414 415 static struct sk_buff *aq_xdp_run_prog(struct aq_nic_s *aq_nic, 416 struct xdp_buff *xdp, 417 struct aq_ring_s *rx_ring, 418 struct aq_ring_buff_s *buff) 419 { 420 int result = NETDEV_TX_BUSY; 421 struct aq_ring_s *tx_ring; 422 struct xdp_frame *xdpf; 423 struct bpf_prog *prog; 424 u32 act = XDP_ABORTED; 425 struct sk_buff *skb; 426 427 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 428 ++rx_ring->stats.rx.packets; 429 rx_ring->stats.rx.bytes += xdp_get_buff_len(xdp); 430 u64_stats_update_end(&rx_ring->stats.rx.syncp); 431 432 prog = READ_ONCE(rx_ring->xdp_prog); 433 if (!prog) 434 goto pass; 435 436 prefetchw(xdp->data_hard_start); /* xdp_frame write */ 437 438 /* single buffer XDP program, but packet is multi buffer, aborted */ 439 if (xdp_buff_has_frags(xdp) && !prog->aux->xdp_has_frags) 440 goto out_aborted; 441 442 act = bpf_prog_run_xdp(prog, xdp); 443 switch (act) { 444 case XDP_PASS: 445 pass: 446 xdpf = xdp_convert_buff_to_frame(xdp); 447 if (unlikely(!xdpf)) 448 goto out_aborted; 449 skb = xdp_build_skb_from_frame(xdpf, aq_nic->ndev); 450 if (!skb) 451 goto out_aborted; 452 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 453 ++rx_ring->stats.rx.xdp_pass; 454 u64_stats_update_end(&rx_ring->stats.rx.syncp); 455 aq_get_rxpages_xdp(buff, xdp); 456 return skb; 457 case XDP_TX: 458 xdpf = xdp_convert_buff_to_frame(xdp); 459 if (unlikely(!xdpf)) 460 goto out_aborted; 461 tx_ring = aq_nic->aq_ring_tx[rx_ring->idx]; 462 result = aq_nic_xmit_xdpf(aq_nic, tx_ring, xdpf); 463 if (result == NETDEV_TX_BUSY) 464 goto out_aborted; 465 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 466 ++rx_ring->stats.rx.xdp_tx; 467 u64_stats_update_end(&rx_ring->stats.rx.syncp); 468 aq_get_rxpages_xdp(buff, xdp); 469 break; 470 case XDP_REDIRECT: 471 if (xdp_do_redirect(aq_nic->ndev, xdp, prog) < 0) 472 goto out_aborted; 473 xdp_do_flush(); 474 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 475 ++rx_ring->stats.rx.xdp_redirect; 476 u64_stats_update_end(&rx_ring->stats.rx.syncp); 477 aq_get_rxpages_xdp(buff, xdp); 478 break; 479 default: 480 fallthrough; 481 case XDP_ABORTED: 482 out_aborted: 483 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 484 ++rx_ring->stats.rx.xdp_aborted; 485 u64_stats_update_end(&rx_ring->stats.rx.syncp); 486 trace_xdp_exception(aq_nic->ndev, prog, act); 487 bpf_warn_invalid_xdp_action(aq_nic->ndev, prog, act); 488 break; 489 case XDP_DROP: 490 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 491 ++rx_ring->stats.rx.xdp_drop; 492 u64_stats_update_end(&rx_ring->stats.rx.syncp); 493 break; 494 } 495 496 return ERR_PTR(-result); 497 } 498 499 static bool aq_add_rx_fragment(struct device *dev, 500 struct aq_ring_s *ring, 501 struct aq_ring_buff_s *buff, 502 struct xdp_buff *xdp) 503 { 504 struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); 505 struct aq_ring_buff_s *buff_ = buff; 506 507 memset(sinfo, 0, sizeof(*sinfo)); 508 do { 509 skb_frag_t *frag; 510 511 if (unlikely(sinfo->nr_frags >= MAX_SKB_FRAGS)) 512 return true; 513 514 frag = &sinfo->frags[sinfo->nr_frags++]; 515 buff_ = &ring->buff_ring[buff_->next]; 516 dma_sync_single_range_for_cpu(dev, 517 buff_->rxdata.daddr, 518 buff_->rxdata.pg_off, 519 buff_->len, 520 DMA_FROM_DEVICE); 521 skb_frag_off_set(frag, buff_->rxdata.pg_off); 522 skb_frag_size_set(frag, buff_->len); 523 sinfo->xdp_frags_size += buff_->len; 524 __skb_frag_set_page(frag, buff_->rxdata.page); 525 526 buff_->is_cleaned = 1; 527 528 buff->is_ip_cso &= buff_->is_ip_cso; 529 buff->is_udp_cso &= buff_->is_udp_cso; 530 buff->is_tcp_cso &= buff_->is_tcp_cso; 531 buff->is_cso_err |= buff_->is_cso_err; 532 533 if (page_is_pfmemalloc(buff_->rxdata.page)) 534 xdp_buff_set_frag_pfmemalloc(xdp); 535 536 } while (!buff_->is_eop); 537 538 xdp_buff_set_frags_flag(xdp); 539 540 return false; 541 } 542 543 static int __aq_ring_rx_clean(struct aq_ring_s *self, struct napi_struct *napi, 544 int *work_done, int budget) 545 { 546 struct net_device *ndev = aq_nic_get_ndev(self->aq_nic); 547 int err = 0; 548 549 for (; (self->sw_head != self->hw_head) && budget; 550 self->sw_head = aq_ring_next_dx(self, self->sw_head), 551 --budget, ++(*work_done)) { 552 struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head]; 553 bool is_ptp_ring = aq_ptp_ring(self->aq_nic, self); 554 struct aq_ring_buff_s *buff_ = NULL; 555 struct sk_buff *skb = NULL; 556 unsigned int next_ = 0U; 557 unsigned int i = 0U; 558 u16 hdr_len; 559 560 if (buff->is_cleaned) 561 continue; 562 563 if (!buff->is_eop) { 564 unsigned int frag_cnt = 0U; 565 buff_ = buff; 566 do { 567 bool is_rsc_completed = true; 568 569 if (buff_->next >= self->size) { 570 err = -EIO; 571 goto err_exit; 572 } 573 574 frag_cnt++; 575 next_ = buff_->next, 576 buff_ = &self->buff_ring[next_]; 577 is_rsc_completed = 578 aq_ring_dx_in_range(self->sw_head, 579 next_, 580 self->hw_head); 581 582 if (unlikely(!is_rsc_completed) || 583 frag_cnt > MAX_SKB_FRAGS) { 584 err = 0; 585 goto err_exit; 586 } 587 588 buff->is_error |= buff_->is_error; 589 buff->is_cso_err |= buff_->is_cso_err; 590 591 } while (!buff_->is_eop); 592 593 if (buff->is_error || 594 (buff->is_lro && buff->is_cso_err)) { 595 buff_ = buff; 596 do { 597 if (buff_->next >= self->size) { 598 err = -EIO; 599 goto err_exit; 600 } 601 next_ = buff_->next, 602 buff_ = &self->buff_ring[next_]; 603 604 buff_->is_cleaned = true; 605 } while (!buff_->is_eop); 606 607 u64_stats_update_begin(&self->stats.rx.syncp); 608 ++self->stats.rx.errors; 609 u64_stats_update_end(&self->stats.rx.syncp); 610 continue; 611 } 612 } 613 614 if (buff->is_error) { 615 u64_stats_update_begin(&self->stats.rx.syncp); 616 ++self->stats.rx.errors; 617 u64_stats_update_end(&self->stats.rx.syncp); 618 continue; 619 } 620 621 dma_sync_single_range_for_cpu(aq_nic_get_dev(self->aq_nic), 622 buff->rxdata.daddr, 623 buff->rxdata.pg_off, 624 buff->len, DMA_FROM_DEVICE); 625 626 skb = napi_alloc_skb(napi, AQ_CFG_RX_HDR_SIZE); 627 if (unlikely(!skb)) { 628 u64_stats_update_begin(&self->stats.rx.syncp); 629 self->stats.rx.skb_alloc_fails++; 630 u64_stats_update_end(&self->stats.rx.syncp); 631 err = -ENOMEM; 632 goto err_exit; 633 } 634 if (is_ptp_ring) 635 buff->len -= 636 aq_ptp_extract_ts(self->aq_nic, skb, 637 aq_buf_vaddr(&buff->rxdata), 638 buff->len); 639 640 hdr_len = buff->len; 641 if (hdr_len > AQ_CFG_RX_HDR_SIZE) 642 hdr_len = eth_get_headlen(skb->dev, 643 aq_buf_vaddr(&buff->rxdata), 644 AQ_CFG_RX_HDR_SIZE); 645 646 memcpy(__skb_put(skb, hdr_len), aq_buf_vaddr(&buff->rxdata), 647 ALIGN(hdr_len, sizeof(long))); 648 649 if (buff->len - hdr_len > 0) { 650 skb_add_rx_frag(skb, i++, buff->rxdata.page, 651 buff->rxdata.pg_off + hdr_len, 652 buff->len - hdr_len, 653 self->frame_max); 654 page_ref_inc(buff->rxdata.page); 655 } 656 657 if (!buff->is_eop) { 658 buff_ = buff; 659 do { 660 next_ = buff_->next; 661 buff_ = &self->buff_ring[next_]; 662 663 dma_sync_single_range_for_cpu(aq_nic_get_dev(self->aq_nic), 664 buff_->rxdata.daddr, 665 buff_->rxdata.pg_off, 666 buff_->len, 667 DMA_FROM_DEVICE); 668 skb_add_rx_frag(skb, i++, 669 buff_->rxdata.page, 670 buff_->rxdata.pg_off, 671 buff_->len, 672 self->frame_max); 673 page_ref_inc(buff_->rxdata.page); 674 buff_->is_cleaned = 1; 675 676 buff->is_ip_cso &= buff_->is_ip_cso; 677 buff->is_udp_cso &= buff_->is_udp_cso; 678 buff->is_tcp_cso &= buff_->is_tcp_cso; 679 buff->is_cso_err |= buff_->is_cso_err; 680 681 } while (!buff_->is_eop); 682 } 683 684 if (buff->is_vlan) 685 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 686 buff->vlan_rx_tag); 687 688 skb->protocol = eth_type_trans(skb, ndev); 689 690 aq_rx_checksum(self, buff, skb); 691 692 skb_set_hash(skb, buff->rss_hash, 693 buff->is_hash_l4 ? PKT_HASH_TYPE_L4 : 694 PKT_HASH_TYPE_NONE); 695 /* Send all PTP traffic to 0 queue */ 696 skb_record_rx_queue(skb, 697 is_ptp_ring ? 0 698 : AQ_NIC_RING2QMAP(self->aq_nic, 699 self->idx)); 700 701 u64_stats_update_begin(&self->stats.rx.syncp); 702 ++self->stats.rx.packets; 703 self->stats.rx.bytes += skb->len; 704 u64_stats_update_end(&self->stats.rx.syncp); 705 706 napi_gro_receive(napi, skb); 707 } 708 709 err_exit: 710 return err; 711 } 712 713 static int __aq_ring_xdp_clean(struct aq_ring_s *rx_ring, 714 struct napi_struct *napi, int *work_done, 715 int budget) 716 { 717 int frame_sz = rx_ring->page_offset + rx_ring->frame_max + 718 rx_ring->tail_size; 719 struct aq_nic_s *aq_nic = rx_ring->aq_nic; 720 bool is_rsc_completed = true; 721 struct device *dev; 722 int err = 0; 723 724 dev = aq_nic_get_dev(aq_nic); 725 for (; (rx_ring->sw_head != rx_ring->hw_head) && budget; 726 rx_ring->sw_head = aq_ring_next_dx(rx_ring, rx_ring->sw_head), 727 --budget, ++(*work_done)) { 728 struct aq_ring_buff_s *buff = &rx_ring->buff_ring[rx_ring->sw_head]; 729 bool is_ptp_ring = aq_ptp_ring(rx_ring->aq_nic, rx_ring); 730 struct aq_ring_buff_s *buff_ = NULL; 731 struct sk_buff *skb = NULL; 732 unsigned int next_ = 0U; 733 struct xdp_buff xdp; 734 void *hard_start; 735 736 if (buff->is_cleaned) 737 continue; 738 739 if (!buff->is_eop) { 740 buff_ = buff; 741 do { 742 if (buff_->next >= rx_ring->size) { 743 err = -EIO; 744 goto err_exit; 745 } 746 next_ = buff_->next; 747 buff_ = &rx_ring->buff_ring[next_]; 748 is_rsc_completed = 749 aq_ring_dx_in_range(rx_ring->sw_head, 750 next_, 751 rx_ring->hw_head); 752 753 if (unlikely(!is_rsc_completed)) 754 break; 755 756 buff->is_error |= buff_->is_error; 757 buff->is_cso_err |= buff_->is_cso_err; 758 } while (!buff_->is_eop); 759 760 if (!is_rsc_completed) { 761 err = 0; 762 goto err_exit; 763 } 764 if (buff->is_error || 765 (buff->is_lro && buff->is_cso_err)) { 766 buff_ = buff; 767 do { 768 if (buff_->next >= rx_ring->size) { 769 err = -EIO; 770 goto err_exit; 771 } 772 next_ = buff_->next; 773 buff_ = &rx_ring->buff_ring[next_]; 774 775 buff_->is_cleaned = true; 776 } while (!buff_->is_eop); 777 778 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 779 ++rx_ring->stats.rx.errors; 780 u64_stats_update_end(&rx_ring->stats.rx.syncp); 781 continue; 782 } 783 } 784 785 if (buff->is_error) { 786 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 787 ++rx_ring->stats.rx.errors; 788 u64_stats_update_end(&rx_ring->stats.rx.syncp); 789 continue; 790 } 791 792 dma_sync_single_range_for_cpu(dev, 793 buff->rxdata.daddr, 794 buff->rxdata.pg_off, 795 buff->len, DMA_FROM_DEVICE); 796 hard_start = page_address(buff->rxdata.page) + 797 buff->rxdata.pg_off - rx_ring->page_offset; 798 799 if (is_ptp_ring) 800 buff->len -= 801 aq_ptp_extract_ts(rx_ring->aq_nic, skb, 802 aq_buf_vaddr(&buff->rxdata), 803 buff->len); 804 805 xdp_init_buff(&xdp, frame_sz, &rx_ring->xdp_rxq); 806 xdp_prepare_buff(&xdp, hard_start, rx_ring->page_offset, 807 buff->len, false); 808 if (!buff->is_eop) { 809 if (aq_add_rx_fragment(dev, rx_ring, buff, &xdp)) { 810 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 811 ++rx_ring->stats.rx.packets; 812 rx_ring->stats.rx.bytes += xdp_get_buff_len(&xdp); 813 ++rx_ring->stats.rx.xdp_aborted; 814 u64_stats_update_end(&rx_ring->stats.rx.syncp); 815 continue; 816 } 817 } 818 819 skb = aq_xdp_run_prog(aq_nic, &xdp, rx_ring, buff); 820 if (IS_ERR(skb) || !skb) 821 continue; 822 823 if (buff->is_vlan) 824 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 825 buff->vlan_rx_tag); 826 827 aq_rx_checksum(rx_ring, buff, skb); 828 829 skb_set_hash(skb, buff->rss_hash, 830 buff->is_hash_l4 ? PKT_HASH_TYPE_L4 : 831 PKT_HASH_TYPE_NONE); 832 /* Send all PTP traffic to 0 queue */ 833 skb_record_rx_queue(skb, 834 is_ptp_ring ? 0 835 : AQ_NIC_RING2QMAP(rx_ring->aq_nic, 836 rx_ring->idx)); 837 838 napi_gro_receive(napi, skb); 839 } 840 841 err_exit: 842 return err; 843 } 844 845 int aq_ring_rx_clean(struct aq_ring_s *self, 846 struct napi_struct *napi, 847 int *work_done, 848 int budget) 849 { 850 if (static_branch_unlikely(&aq_xdp_locking_key)) 851 return __aq_ring_xdp_clean(self, napi, work_done, budget); 852 else 853 return __aq_ring_rx_clean(self, napi, work_done, budget); 854 } 855 856 void aq_ring_hwts_rx_clean(struct aq_ring_s *self, struct aq_nic_s *aq_nic) 857 { 858 #if IS_REACHABLE(CONFIG_PTP_1588_CLOCK) 859 while (self->sw_head != self->hw_head) { 860 u64 ns; 861 862 aq_nic->aq_hw_ops->extract_hwts(aq_nic->aq_hw, 863 self->dx_ring + 864 (self->sw_head * self->dx_size), 865 self->dx_size, &ns); 866 aq_ptp_tx_hwtstamp(aq_nic, ns); 867 868 self->sw_head = aq_ring_next_dx(self, self->sw_head); 869 } 870 #endif 871 } 872 873 int aq_ring_rx_fill(struct aq_ring_s *self) 874 { 875 struct aq_ring_buff_s *buff = NULL; 876 int err = 0; 877 int i = 0; 878 879 if (aq_ring_avail_dx(self) < min_t(unsigned int, AQ_CFG_RX_REFILL_THRES, 880 self->size / 2)) 881 return err; 882 883 for (i = aq_ring_avail_dx(self); i--; 884 self->sw_tail = aq_ring_next_dx(self, self->sw_tail)) { 885 buff = &self->buff_ring[self->sw_tail]; 886 887 buff->flags = 0U; 888 buff->len = self->frame_max; 889 890 err = aq_get_rxpages(self, buff); 891 if (err) 892 goto err_exit; 893 894 buff->pa = aq_buf_daddr(&buff->rxdata); 895 buff = NULL; 896 } 897 898 err_exit: 899 return err; 900 } 901 902 void aq_ring_rx_deinit(struct aq_ring_s *self) 903 { 904 if (!self) 905 return; 906 907 for (; self->sw_head != self->sw_tail; 908 self->sw_head = aq_ring_next_dx(self, self->sw_head)) { 909 struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head]; 910 911 aq_free_rxpage(&buff->rxdata, aq_nic_get_dev(self->aq_nic)); 912 } 913 } 914 915 void aq_ring_free(struct aq_ring_s *self) 916 { 917 if (!self) 918 return; 919 920 kfree(self->buff_ring); 921 922 if (self->dx_ring) 923 dma_free_coherent(aq_nic_get_dev(self->aq_nic), 924 self->size * self->dx_size, self->dx_ring, 925 self->dx_ring_pa); 926 } 927 928 unsigned int aq_ring_fill_stats_data(struct aq_ring_s *self, u64 *data) 929 { 930 unsigned int count; 931 unsigned int start; 932 933 if (self->ring_type == ATL_RING_RX) { 934 /* This data should mimic aq_ethtool_queue_rx_stat_names structure */ 935 do { 936 count = 0; 937 start = u64_stats_fetch_begin_irq(&self->stats.rx.syncp); 938 data[count] = self->stats.rx.packets; 939 data[++count] = self->stats.rx.jumbo_packets; 940 data[++count] = self->stats.rx.lro_packets; 941 data[++count] = self->stats.rx.errors; 942 data[++count] = self->stats.rx.alloc_fails; 943 data[++count] = self->stats.rx.skb_alloc_fails; 944 data[++count] = self->stats.rx.polls; 945 data[++count] = self->stats.rx.pg_flips; 946 data[++count] = self->stats.rx.pg_reuses; 947 data[++count] = self->stats.rx.pg_losts; 948 data[++count] = self->stats.rx.xdp_aborted; 949 data[++count] = self->stats.rx.xdp_drop; 950 data[++count] = self->stats.rx.xdp_pass; 951 data[++count] = self->stats.rx.xdp_tx; 952 data[++count] = self->stats.rx.xdp_invalid; 953 data[++count] = self->stats.rx.xdp_redirect; 954 } while (u64_stats_fetch_retry_irq(&self->stats.rx.syncp, start)); 955 } else { 956 /* This data should mimic aq_ethtool_queue_tx_stat_names structure */ 957 do { 958 count = 0; 959 start = u64_stats_fetch_begin_irq(&self->stats.tx.syncp); 960 data[count] = self->stats.tx.packets; 961 data[++count] = self->stats.tx.queue_restarts; 962 } while (u64_stats_fetch_retry_irq(&self->stats.tx.syncp, start)); 963 } 964 965 return ++count; 966 } 967