11751cc36SAlex Maftei (amaftei) // SPDX-License-Identifier: GPL-2.0-only
21751cc36SAlex Maftei (amaftei) /****************************************************************************
31751cc36SAlex Maftei (amaftei) * Driver for Solarflare network controllers and boards
41751cc36SAlex Maftei (amaftei) * Copyright 2018 Solarflare Communications Inc.
51751cc36SAlex Maftei (amaftei) *
61751cc36SAlex Maftei (amaftei) * This program is free software; you can redistribute it and/or modify it
71751cc36SAlex Maftei (amaftei) * under the terms of the GNU General Public License version 2 as published
81751cc36SAlex Maftei (amaftei) * by the Free Software Foundation, incorporated herein by reference.
91751cc36SAlex Maftei (amaftei) */
101751cc36SAlex Maftei (amaftei)
111751cc36SAlex Maftei (amaftei) #include "net_driver.h"
121751cc36SAlex Maftei (amaftei) #include <linux/module.h>
133d95b884SAlex Maftei (amaftei) #include <linux/iommu.h>
14490a79faSEric Dumazet #include <net/rps.h>
151751cc36SAlex Maftei (amaftei) #include "efx.h"
161751cc36SAlex Maftei (amaftei) #include "nic.h"
171751cc36SAlex Maftei (amaftei) #include "rx_common.h"
181751cc36SAlex Maftei (amaftei)
191751cc36SAlex Maftei (amaftei) /* This is the percentage fill level below which new RX descriptors
201751cc36SAlex Maftei (amaftei) * will be added to the RX descriptor ring.
211751cc36SAlex Maftei (amaftei) */
221751cc36SAlex Maftei (amaftei) static unsigned int rx_refill_threshold;
231751cc36SAlex Maftei (amaftei) module_param(rx_refill_threshold, uint, 0444);
241751cc36SAlex Maftei (amaftei) MODULE_PARM_DESC(rx_refill_threshold,
251751cc36SAlex Maftei (amaftei) "RX descriptor ring refill threshold (%)");
261751cc36SAlex Maftei (amaftei)
271751cc36SAlex Maftei (amaftei) /* RX maximum head room required.
281751cc36SAlex Maftei (amaftei) *
291751cc36SAlex Maftei (amaftei) * This must be at least 1 to prevent overflow, plus one packet-worth
301751cc36SAlex Maftei (amaftei) * to allow pipelined receives.
311751cc36SAlex Maftei (amaftei) */
321751cc36SAlex Maftei (amaftei) #define EFX_RXD_HEAD_ROOM (1 + EFX_RX_MAX_FRAGS)
331751cc36SAlex Maftei (amaftei)
343d95b884SAlex Maftei (amaftei) /* Check the RX page recycle ring for a page that can be reused. */
efx_reuse_page(struct efx_rx_queue * rx_queue)353d95b884SAlex Maftei (amaftei) static struct page *efx_reuse_page(struct efx_rx_queue *rx_queue)
363d95b884SAlex Maftei (amaftei) {
373d95b884SAlex Maftei (amaftei) struct efx_nic *efx = rx_queue->efx;
383d95b884SAlex Maftei (amaftei) struct efx_rx_page_state *state;
393d95b884SAlex Maftei (amaftei) unsigned int index;
403d95b884SAlex Maftei (amaftei) struct page *page;
413d95b884SAlex Maftei (amaftei)
421d5a4742SMartin Habets if (unlikely(!rx_queue->page_ring))
431d5a4742SMartin Habets return NULL;
443d95b884SAlex Maftei (amaftei) index = rx_queue->page_remove & rx_queue->page_ptr_mask;
453d95b884SAlex Maftei (amaftei) page = rx_queue->page_ring[index];
463d95b884SAlex Maftei (amaftei) if (page == NULL)
473d95b884SAlex Maftei (amaftei) return NULL;
483d95b884SAlex Maftei (amaftei)
493d95b884SAlex Maftei (amaftei) rx_queue->page_ring[index] = NULL;
503d95b884SAlex Maftei (amaftei) /* page_remove cannot exceed page_add. */
513d95b884SAlex Maftei (amaftei) if (rx_queue->page_remove != rx_queue->page_add)
523d95b884SAlex Maftei (amaftei) ++rx_queue->page_remove;
533d95b884SAlex Maftei (amaftei)
543d95b884SAlex Maftei (amaftei) /* If page_count is 1 then we hold the only reference to this page. */
553d95b884SAlex Maftei (amaftei) if (page_count(page) == 1) {
563d95b884SAlex Maftei (amaftei) ++rx_queue->page_recycle_count;
573d95b884SAlex Maftei (amaftei) return page;
583d95b884SAlex Maftei (amaftei) } else {
593d95b884SAlex Maftei (amaftei) state = page_address(page);
603d95b884SAlex Maftei (amaftei) dma_unmap_page(&efx->pci_dev->dev, state->dma_addr,
613d95b884SAlex Maftei (amaftei) PAGE_SIZE << efx->rx_buffer_order,
623d95b884SAlex Maftei (amaftei) DMA_FROM_DEVICE);
633d95b884SAlex Maftei (amaftei) put_page(page);
643d95b884SAlex Maftei (amaftei) ++rx_queue->page_recycle_failed;
653d95b884SAlex Maftei (amaftei) }
663d95b884SAlex Maftei (amaftei)
673d95b884SAlex Maftei (amaftei) return NULL;
683d95b884SAlex Maftei (amaftei) }
693d95b884SAlex Maftei (amaftei)
703d95b884SAlex Maftei (amaftei) /* Attempt to recycle the page if there is an RX recycle ring; the page can
713d95b884SAlex Maftei (amaftei) * only be added if this is the final RX buffer, to prevent pages being used in
723d95b884SAlex Maftei (amaftei) * the descriptor ring and appearing in the recycle ring simultaneously.
733d95b884SAlex Maftei (amaftei) */
efx_recycle_rx_page(struct efx_channel * channel,struct efx_rx_buffer * rx_buf)743d95b884SAlex Maftei (amaftei) static void efx_recycle_rx_page(struct efx_channel *channel,
753d95b884SAlex Maftei (amaftei) struct efx_rx_buffer *rx_buf)
763d95b884SAlex Maftei (amaftei) {
773d95b884SAlex Maftei (amaftei) struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel);
783d95b884SAlex Maftei (amaftei) struct efx_nic *efx = rx_queue->efx;
793d95b884SAlex Maftei (amaftei) struct page *page = rx_buf->page;
803d95b884SAlex Maftei (amaftei) unsigned int index;
813d95b884SAlex Maftei (amaftei)
823d95b884SAlex Maftei (amaftei) /* Only recycle the page after processing the final buffer. */
833d95b884SAlex Maftei (amaftei) if (!(rx_buf->flags & EFX_RX_BUF_LAST_IN_PAGE))
843d95b884SAlex Maftei (amaftei) return;
853d95b884SAlex Maftei (amaftei)
863d95b884SAlex Maftei (amaftei) index = rx_queue->page_add & rx_queue->page_ptr_mask;
873d95b884SAlex Maftei (amaftei) if (rx_queue->page_ring[index] == NULL) {
883d95b884SAlex Maftei (amaftei) unsigned int read_index = rx_queue->page_remove &
893d95b884SAlex Maftei (amaftei) rx_queue->page_ptr_mask;
903d95b884SAlex Maftei (amaftei)
913d95b884SAlex Maftei (amaftei) /* The next slot in the recycle ring is available, but
923d95b884SAlex Maftei (amaftei) * increment page_remove if the read pointer currently
933d95b884SAlex Maftei (amaftei) * points here.
943d95b884SAlex Maftei (amaftei) */
953d95b884SAlex Maftei (amaftei) if (read_index == index)
963d95b884SAlex Maftei (amaftei) ++rx_queue->page_remove;
973d95b884SAlex Maftei (amaftei) rx_queue->page_ring[index] = page;
983d95b884SAlex Maftei (amaftei) ++rx_queue->page_add;
993d95b884SAlex Maftei (amaftei) return;
1003d95b884SAlex Maftei (amaftei) }
1013d95b884SAlex Maftei (amaftei) ++rx_queue->page_recycle_full;
1023d95b884SAlex Maftei (amaftei) efx_unmap_rx_buffer(efx, rx_buf);
1033d95b884SAlex Maftei (amaftei) put_page(rx_buf->page);
1043d95b884SAlex Maftei (amaftei) }
1053d95b884SAlex Maftei (amaftei)
1063d95b884SAlex Maftei (amaftei) /* Recycle the pages that are used by buffers that have just been received. */
efx_recycle_rx_pages(struct efx_channel * channel,struct efx_rx_buffer * rx_buf,unsigned int n_frags)1073d95b884SAlex Maftei (amaftei) void efx_recycle_rx_pages(struct efx_channel *channel,
1083d95b884SAlex Maftei (amaftei) struct efx_rx_buffer *rx_buf,
1093d95b884SAlex Maftei (amaftei) unsigned int n_frags)
1103d95b884SAlex Maftei (amaftei) {
1113d95b884SAlex Maftei (amaftei) struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel);
1123d95b884SAlex Maftei (amaftei)
1131d5a4742SMartin Habets if (unlikely(!rx_queue->page_ring))
1141d5a4742SMartin Habets return;
1151d5a4742SMartin Habets
1163d95b884SAlex Maftei (amaftei) do {
1173d95b884SAlex Maftei (amaftei) efx_recycle_rx_page(channel, rx_buf);
1183d95b884SAlex Maftei (amaftei) rx_buf = efx_rx_buf_next(rx_queue, rx_buf);
1193d95b884SAlex Maftei (amaftei) } while (--n_frags);
1203d95b884SAlex Maftei (amaftei) }
1213d95b884SAlex Maftei (amaftei)
efx_discard_rx_packet(struct efx_channel * channel,struct efx_rx_buffer * rx_buf,unsigned int n_frags)1223d95b884SAlex Maftei (amaftei) void efx_discard_rx_packet(struct efx_channel *channel,
1233d95b884SAlex Maftei (amaftei) struct efx_rx_buffer *rx_buf,
1243d95b884SAlex Maftei (amaftei) unsigned int n_frags)
1253d95b884SAlex Maftei (amaftei) {
1263d95b884SAlex Maftei (amaftei) struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel);
1273d95b884SAlex Maftei (amaftei)
1283d95b884SAlex Maftei (amaftei) efx_recycle_rx_pages(channel, rx_buf, n_frags);
1293d95b884SAlex Maftei (amaftei)
1303d95b884SAlex Maftei (amaftei) efx_free_rx_buffers(rx_queue, rx_buf, n_frags);
1313d95b884SAlex Maftei (amaftei) }
1323d95b884SAlex Maftei (amaftei)
efx_init_rx_recycle_ring(struct efx_rx_queue * rx_queue)1333d95b884SAlex Maftei (amaftei) static void efx_init_rx_recycle_ring(struct efx_rx_queue *rx_queue)
1343d95b884SAlex Maftei (amaftei) {
1353d95b884SAlex Maftei (amaftei) unsigned int bufs_in_recycle_ring, page_ring_size;
1363d95b884SAlex Maftei (amaftei) struct efx_nic *efx = rx_queue->efx;
1373d95b884SAlex Maftei (amaftei)
138000fe940SMartin Habets bufs_in_recycle_ring = efx_rx_recycle_ring_size(efx);
1393d95b884SAlex Maftei (amaftei) page_ring_size = roundup_pow_of_two(bufs_in_recycle_ring /
1403d95b884SAlex Maftei (amaftei) efx->rx_bufs_per_page);
1413d95b884SAlex Maftei (amaftei) rx_queue->page_ring = kcalloc(page_ring_size,
1423d95b884SAlex Maftei (amaftei) sizeof(*rx_queue->page_ring), GFP_KERNEL);
143bdf1b5c3SJiasheng Jiang if (!rx_queue->page_ring)
144bdf1b5c3SJiasheng Jiang rx_queue->page_ptr_mask = 0;
145bdf1b5c3SJiasheng Jiang else
1463d95b884SAlex Maftei (amaftei) rx_queue->page_ptr_mask = page_ring_size - 1;
1473d95b884SAlex Maftei (amaftei) }
1483d95b884SAlex Maftei (amaftei)
efx_fini_rx_recycle_ring(struct efx_rx_queue * rx_queue)1493d95b884SAlex Maftei (amaftei) static void efx_fini_rx_recycle_ring(struct efx_rx_queue *rx_queue)
1503d95b884SAlex Maftei (amaftei) {
1513d95b884SAlex Maftei (amaftei) struct efx_nic *efx = rx_queue->efx;
1523d95b884SAlex Maftei (amaftei) int i;
1533d95b884SAlex Maftei (amaftei)
154458f5d92SMartin Habets if (unlikely(!rx_queue->page_ring))
155458f5d92SMartin Habets return;
156458f5d92SMartin Habets
1573d95b884SAlex Maftei (amaftei) /* Unmap and release the pages in the recycle ring. Remove the ring. */
1583d95b884SAlex Maftei (amaftei) for (i = 0; i <= rx_queue->page_ptr_mask; i++) {
1593d95b884SAlex Maftei (amaftei) struct page *page = rx_queue->page_ring[i];
1603d95b884SAlex Maftei (amaftei) struct efx_rx_page_state *state;
1613d95b884SAlex Maftei (amaftei)
1623d95b884SAlex Maftei (amaftei) if (page == NULL)
1633d95b884SAlex Maftei (amaftei) continue;
1643d95b884SAlex Maftei (amaftei)
1653d95b884SAlex Maftei (amaftei) state = page_address(page);
1663d95b884SAlex Maftei (amaftei) dma_unmap_page(&efx->pci_dev->dev, state->dma_addr,
1673d95b884SAlex Maftei (amaftei) PAGE_SIZE << efx->rx_buffer_order,
1683d95b884SAlex Maftei (amaftei) DMA_FROM_DEVICE);
1693d95b884SAlex Maftei (amaftei) put_page(page);
1703d95b884SAlex Maftei (amaftei) }
1713d95b884SAlex Maftei (amaftei) kfree(rx_queue->page_ring);
1723d95b884SAlex Maftei (amaftei) rx_queue->page_ring = NULL;
1733d95b884SAlex Maftei (amaftei) }
1743d95b884SAlex Maftei (amaftei)
efx_fini_rx_buffer(struct efx_rx_queue * rx_queue,struct efx_rx_buffer * rx_buf)1751751cc36SAlex Maftei (amaftei) static void efx_fini_rx_buffer(struct efx_rx_queue *rx_queue,
1761751cc36SAlex Maftei (amaftei) struct efx_rx_buffer *rx_buf)
1771751cc36SAlex Maftei (amaftei) {
1781751cc36SAlex Maftei (amaftei) /* Release the page reference we hold for the buffer. */
1791751cc36SAlex Maftei (amaftei) if (rx_buf->page)
1801751cc36SAlex Maftei (amaftei) put_page(rx_buf->page);
1811751cc36SAlex Maftei (amaftei)
1821751cc36SAlex Maftei (amaftei) /* If this is the last buffer in a page, unmap and free it. */
1831751cc36SAlex Maftei (amaftei) if (rx_buf->flags & EFX_RX_BUF_LAST_IN_PAGE) {
1841751cc36SAlex Maftei (amaftei) efx_unmap_rx_buffer(rx_queue->efx, rx_buf);
1851751cc36SAlex Maftei (amaftei) efx_free_rx_buffers(rx_queue, rx_buf, 1);
1861751cc36SAlex Maftei (amaftei) }
1871751cc36SAlex Maftei (amaftei) rx_buf->page = NULL;
1881751cc36SAlex Maftei (amaftei) }
1891751cc36SAlex Maftei (amaftei)
efx_probe_rx_queue(struct efx_rx_queue * rx_queue)1901751cc36SAlex Maftei (amaftei) int efx_probe_rx_queue(struct efx_rx_queue *rx_queue)
1911751cc36SAlex Maftei (amaftei) {
1921751cc36SAlex Maftei (amaftei) struct efx_nic *efx = rx_queue->efx;
1931751cc36SAlex Maftei (amaftei) unsigned int entries;
1941751cc36SAlex Maftei (amaftei) int rc;
1951751cc36SAlex Maftei (amaftei)
1961751cc36SAlex Maftei (amaftei) /* Create the smallest power-of-two aligned ring */
1971751cc36SAlex Maftei (amaftei) entries = max(roundup_pow_of_two(efx->rxq_entries), EFX_MIN_DMAQ_SIZE);
1981751cc36SAlex Maftei (amaftei) EFX_WARN_ON_PARANOID(entries > EFX_MAX_DMAQ_SIZE);
1991751cc36SAlex Maftei (amaftei) rx_queue->ptr_mask = entries - 1;
2001751cc36SAlex Maftei (amaftei)
2011751cc36SAlex Maftei (amaftei) netif_dbg(efx, probe, efx->net_dev,
2021751cc36SAlex Maftei (amaftei) "creating RX queue %d size %#x mask %#x\n",
2031751cc36SAlex Maftei (amaftei) efx_rx_queue_index(rx_queue), efx->rxq_entries,
2041751cc36SAlex Maftei (amaftei) rx_queue->ptr_mask);
2051751cc36SAlex Maftei (amaftei)
2061751cc36SAlex Maftei (amaftei) /* Allocate RX buffers */
2071751cc36SAlex Maftei (amaftei) rx_queue->buffer = kcalloc(entries, sizeof(*rx_queue->buffer),
2081751cc36SAlex Maftei (amaftei) GFP_KERNEL);
2091751cc36SAlex Maftei (amaftei) if (!rx_queue->buffer)
2101751cc36SAlex Maftei (amaftei) return -ENOMEM;
2111751cc36SAlex Maftei (amaftei)
2121751cc36SAlex Maftei (amaftei) rc = efx_nic_probe_rx(rx_queue);
2131751cc36SAlex Maftei (amaftei) if (rc) {
2141751cc36SAlex Maftei (amaftei) kfree(rx_queue->buffer);
2151751cc36SAlex Maftei (amaftei) rx_queue->buffer = NULL;
2161751cc36SAlex Maftei (amaftei) }
2171751cc36SAlex Maftei (amaftei)
2181751cc36SAlex Maftei (amaftei) return rc;
2191751cc36SAlex Maftei (amaftei) }
2201751cc36SAlex Maftei (amaftei)
efx_init_rx_queue(struct efx_rx_queue * rx_queue)2211751cc36SAlex Maftei (amaftei) void efx_init_rx_queue(struct efx_rx_queue *rx_queue)
2221751cc36SAlex Maftei (amaftei) {
2231751cc36SAlex Maftei (amaftei) unsigned int max_fill, trigger, max_trigger;
2241751cc36SAlex Maftei (amaftei) struct efx_nic *efx = rx_queue->efx;
2251751cc36SAlex Maftei (amaftei) int rc = 0;
2261751cc36SAlex Maftei (amaftei)
2271751cc36SAlex Maftei (amaftei) netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev,
2281751cc36SAlex Maftei (amaftei) "initialising RX queue %d\n", efx_rx_queue_index(rx_queue));
2291751cc36SAlex Maftei (amaftei)
2301751cc36SAlex Maftei (amaftei) /* Initialise ptr fields */
2311751cc36SAlex Maftei (amaftei) rx_queue->added_count = 0;
2321751cc36SAlex Maftei (amaftei) rx_queue->notified_count = 0;
233e3951539SEdward Cree rx_queue->granted_count = 0;
2341751cc36SAlex Maftei (amaftei) rx_queue->removed_count = 0;
2351751cc36SAlex Maftei (amaftei) rx_queue->min_fill = -1U;
2361751cc36SAlex Maftei (amaftei) efx_init_rx_recycle_ring(rx_queue);
2371751cc36SAlex Maftei (amaftei)
2381751cc36SAlex Maftei (amaftei) rx_queue->page_remove = 0;
2391751cc36SAlex Maftei (amaftei) rx_queue->page_add = rx_queue->page_ptr_mask + 1;
2401751cc36SAlex Maftei (amaftei) rx_queue->page_recycle_count = 0;
2411751cc36SAlex Maftei (amaftei) rx_queue->page_recycle_failed = 0;
2421751cc36SAlex Maftei (amaftei) rx_queue->page_recycle_full = 0;
2431751cc36SAlex Maftei (amaftei)
2441751cc36SAlex Maftei (amaftei) /* Initialise limit fields */
2451751cc36SAlex Maftei (amaftei) max_fill = efx->rxq_entries - EFX_RXD_HEAD_ROOM;
2461751cc36SAlex Maftei (amaftei) max_trigger =
2471751cc36SAlex Maftei (amaftei) max_fill - efx->rx_pages_per_batch * efx->rx_bufs_per_page;
2481751cc36SAlex Maftei (amaftei) if (rx_refill_threshold != 0) {
2491751cc36SAlex Maftei (amaftei) trigger = max_fill * min(rx_refill_threshold, 100U) / 100U;
2501751cc36SAlex Maftei (amaftei) if (trigger > max_trigger)
2511751cc36SAlex Maftei (amaftei) trigger = max_trigger;
2521751cc36SAlex Maftei (amaftei) } else {
2531751cc36SAlex Maftei (amaftei) trigger = max_trigger;
2541751cc36SAlex Maftei (amaftei) }
2551751cc36SAlex Maftei (amaftei)
2561751cc36SAlex Maftei (amaftei) rx_queue->max_fill = max_fill;
2571751cc36SAlex Maftei (amaftei) rx_queue->fast_fill_trigger = trigger;
2581751cc36SAlex Maftei (amaftei) rx_queue->refill_enabled = true;
2591751cc36SAlex Maftei (amaftei)
2601751cc36SAlex Maftei (amaftei) /* Initialise XDP queue information */
2611751cc36SAlex Maftei (amaftei) rc = xdp_rxq_info_reg(&rx_queue->xdp_rxq_info, efx->net_dev,
262b02e5a0eSBjörn Töpel rx_queue->core_index, 0);
2631751cc36SAlex Maftei (amaftei)
2641751cc36SAlex Maftei (amaftei) if (rc) {
2651751cc36SAlex Maftei (amaftei) netif_err(efx, rx_err, efx->net_dev,
2661751cc36SAlex Maftei (amaftei) "Failure to initialise XDP queue information rc=%d\n",
2671751cc36SAlex Maftei (amaftei) rc);
2681751cc36SAlex Maftei (amaftei) efx->xdp_rxq_info_failed = true;
2691751cc36SAlex Maftei (amaftei) } else {
2701751cc36SAlex Maftei (amaftei) rx_queue->xdp_rxq_info_valid = true;
2711751cc36SAlex Maftei (amaftei) }
2721751cc36SAlex Maftei (amaftei)
2731751cc36SAlex Maftei (amaftei) /* Set up RX descriptor ring */
2741751cc36SAlex Maftei (amaftei) efx_nic_init_rx(rx_queue);
2751751cc36SAlex Maftei (amaftei) }
2761751cc36SAlex Maftei (amaftei)
efx_fini_rx_queue(struct efx_rx_queue * rx_queue)2771751cc36SAlex Maftei (amaftei) void efx_fini_rx_queue(struct efx_rx_queue *rx_queue)
2781751cc36SAlex Maftei (amaftei) {
2791751cc36SAlex Maftei (amaftei) struct efx_rx_buffer *rx_buf;
2801751cc36SAlex Maftei (amaftei) int i;
2811751cc36SAlex Maftei (amaftei)
2821751cc36SAlex Maftei (amaftei) netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev,
2831751cc36SAlex Maftei (amaftei) "shutting down RX queue %d\n", efx_rx_queue_index(rx_queue));
2841751cc36SAlex Maftei (amaftei)
2851751cc36SAlex Maftei (amaftei) del_timer_sync(&rx_queue->slow_fill);
286e3951539SEdward Cree if (rx_queue->grant_credits)
287e3951539SEdward Cree flush_work(&rx_queue->grant_work);
2881751cc36SAlex Maftei (amaftei)
2891751cc36SAlex Maftei (amaftei) /* Release RX buffers from the current read ptr to the write ptr */
2901751cc36SAlex Maftei (amaftei) if (rx_queue->buffer) {
2911751cc36SAlex Maftei (amaftei) for (i = rx_queue->removed_count; i < rx_queue->added_count;
2921751cc36SAlex Maftei (amaftei) i++) {
2931751cc36SAlex Maftei (amaftei) unsigned int index = i & rx_queue->ptr_mask;
2941751cc36SAlex Maftei (amaftei)
2951751cc36SAlex Maftei (amaftei) rx_buf = efx_rx_buffer(rx_queue, index);
2961751cc36SAlex Maftei (amaftei) efx_fini_rx_buffer(rx_queue, rx_buf);
2971751cc36SAlex Maftei (amaftei) }
2981751cc36SAlex Maftei (amaftei) }
2991751cc36SAlex Maftei (amaftei)
3003d95b884SAlex Maftei (amaftei) efx_fini_rx_recycle_ring(rx_queue);
3011751cc36SAlex Maftei (amaftei)
3021751cc36SAlex Maftei (amaftei) if (rx_queue->xdp_rxq_info_valid)
3031751cc36SAlex Maftei (amaftei) xdp_rxq_info_unreg(&rx_queue->xdp_rxq_info);
3041751cc36SAlex Maftei (amaftei)
3051751cc36SAlex Maftei (amaftei) rx_queue->xdp_rxq_info_valid = false;
3061751cc36SAlex Maftei (amaftei) }
3071751cc36SAlex Maftei (amaftei)
efx_remove_rx_queue(struct efx_rx_queue * rx_queue)3081751cc36SAlex Maftei (amaftei) void efx_remove_rx_queue(struct efx_rx_queue *rx_queue)
3091751cc36SAlex Maftei (amaftei) {
3101751cc36SAlex Maftei (amaftei) netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev,
3111751cc36SAlex Maftei (amaftei) "destroying RX queue %d\n", efx_rx_queue_index(rx_queue));
3121751cc36SAlex Maftei (amaftei)
3131751cc36SAlex Maftei (amaftei) efx_nic_remove_rx(rx_queue);
3141751cc36SAlex Maftei (amaftei)
3151751cc36SAlex Maftei (amaftei) kfree(rx_queue->buffer);
3161751cc36SAlex Maftei (amaftei) rx_queue->buffer = NULL;
3171751cc36SAlex Maftei (amaftei) }
3181751cc36SAlex Maftei (amaftei)
3191751cc36SAlex Maftei (amaftei) /* Unmap a DMA-mapped page. This function is only called for the final RX
3201751cc36SAlex Maftei (amaftei) * buffer in a page.
3211751cc36SAlex Maftei (amaftei) */
efx_unmap_rx_buffer(struct efx_nic * efx,struct efx_rx_buffer * rx_buf)3221751cc36SAlex Maftei (amaftei) void efx_unmap_rx_buffer(struct efx_nic *efx,
3231751cc36SAlex Maftei (amaftei) struct efx_rx_buffer *rx_buf)
3241751cc36SAlex Maftei (amaftei) {
3251751cc36SAlex Maftei (amaftei) struct page *page = rx_buf->page;
3261751cc36SAlex Maftei (amaftei)
3271751cc36SAlex Maftei (amaftei) if (page) {
3281751cc36SAlex Maftei (amaftei) struct efx_rx_page_state *state = page_address(page);
3291751cc36SAlex Maftei (amaftei)
3301751cc36SAlex Maftei (amaftei) dma_unmap_page(&efx->pci_dev->dev,
3311751cc36SAlex Maftei (amaftei) state->dma_addr,
3321751cc36SAlex Maftei (amaftei) PAGE_SIZE << efx->rx_buffer_order,
3331751cc36SAlex Maftei (amaftei) DMA_FROM_DEVICE);
3341751cc36SAlex Maftei (amaftei) }
3351751cc36SAlex Maftei (amaftei) }
3361751cc36SAlex Maftei (amaftei)
efx_free_rx_buffers(struct efx_rx_queue * rx_queue,struct efx_rx_buffer * rx_buf,unsigned int num_bufs)3371751cc36SAlex Maftei (amaftei) void efx_free_rx_buffers(struct efx_rx_queue *rx_queue,
3381751cc36SAlex Maftei (amaftei) struct efx_rx_buffer *rx_buf,
3391751cc36SAlex Maftei (amaftei) unsigned int num_bufs)
3401751cc36SAlex Maftei (amaftei) {
3411751cc36SAlex Maftei (amaftei) do {
3421751cc36SAlex Maftei (amaftei) if (rx_buf->page) {
3431751cc36SAlex Maftei (amaftei) put_page(rx_buf->page);
3441751cc36SAlex Maftei (amaftei) rx_buf->page = NULL;
3451751cc36SAlex Maftei (amaftei) }
3461751cc36SAlex Maftei (amaftei) rx_buf = efx_rx_buf_next(rx_queue, rx_buf);
3471751cc36SAlex Maftei (amaftei) } while (--num_bufs);
3481751cc36SAlex Maftei (amaftei) }
3491751cc36SAlex Maftei (amaftei)
efx_rx_slow_fill(struct timer_list * t)3501751cc36SAlex Maftei (amaftei) void efx_rx_slow_fill(struct timer_list *t)
3511751cc36SAlex Maftei (amaftei) {
3521751cc36SAlex Maftei (amaftei) struct efx_rx_queue *rx_queue = from_timer(rx_queue, t, slow_fill);
3531751cc36SAlex Maftei (amaftei)
3541751cc36SAlex Maftei (amaftei) /* Post an event to cause NAPI to run and refill the queue */
3551751cc36SAlex Maftei (amaftei) efx_nic_generate_fill_event(rx_queue);
3561751cc36SAlex Maftei (amaftei) ++rx_queue->slow_fill_count;
3571751cc36SAlex Maftei (amaftei) }
3581751cc36SAlex Maftei (amaftei)
efx_schedule_slow_fill(struct efx_rx_queue * rx_queue)3591751cc36SAlex Maftei (amaftei) void efx_schedule_slow_fill(struct efx_rx_queue *rx_queue)
3601751cc36SAlex Maftei (amaftei) {
3611751cc36SAlex Maftei (amaftei) mod_timer(&rx_queue->slow_fill, jiffies + msecs_to_jiffies(10));
3621751cc36SAlex Maftei (amaftei) }
3631751cc36SAlex Maftei (amaftei)
3641751cc36SAlex Maftei (amaftei) /* efx_init_rx_buffers - create EFX_RX_BATCH page-based RX buffers
3651751cc36SAlex Maftei (amaftei) *
3661751cc36SAlex Maftei (amaftei) * @rx_queue: Efx RX queue
3671751cc36SAlex Maftei (amaftei) *
3681751cc36SAlex Maftei (amaftei) * This allocates a batch of pages, maps them for DMA, and populates
3691751cc36SAlex Maftei (amaftei) * struct efx_rx_buffers for each one. Return a negative error code or
3701751cc36SAlex Maftei (amaftei) * 0 on success. If a single page can be used for multiple buffers,
3711751cc36SAlex Maftei (amaftei) * then the page will either be inserted fully, or not at all.
3721751cc36SAlex Maftei (amaftei) */
efx_init_rx_buffers(struct efx_rx_queue * rx_queue,bool atomic)3731751cc36SAlex Maftei (amaftei) static int efx_init_rx_buffers(struct efx_rx_queue *rx_queue, bool atomic)
3741751cc36SAlex Maftei (amaftei) {
3751751cc36SAlex Maftei (amaftei) unsigned int page_offset, index, count;
3761751cc36SAlex Maftei (amaftei) struct efx_nic *efx = rx_queue->efx;
3771751cc36SAlex Maftei (amaftei) struct efx_rx_page_state *state;
3781751cc36SAlex Maftei (amaftei) struct efx_rx_buffer *rx_buf;
3791751cc36SAlex Maftei (amaftei) dma_addr_t dma_addr;
3801751cc36SAlex Maftei (amaftei) struct page *page;
3811751cc36SAlex Maftei (amaftei)
3821751cc36SAlex Maftei (amaftei) count = 0;
3831751cc36SAlex Maftei (amaftei) do {
3841751cc36SAlex Maftei (amaftei) page = efx_reuse_page(rx_queue);
3851751cc36SAlex Maftei (amaftei) if (page == NULL) {
3861751cc36SAlex Maftei (amaftei) page = alloc_pages(__GFP_COMP |
3871751cc36SAlex Maftei (amaftei) (atomic ? GFP_ATOMIC : GFP_KERNEL),
3881751cc36SAlex Maftei (amaftei) efx->rx_buffer_order);
3891751cc36SAlex Maftei (amaftei) if (unlikely(page == NULL))
3901751cc36SAlex Maftei (amaftei) return -ENOMEM;
3911751cc36SAlex Maftei (amaftei) dma_addr =
3921751cc36SAlex Maftei (amaftei) dma_map_page(&efx->pci_dev->dev, page, 0,
3931751cc36SAlex Maftei (amaftei) PAGE_SIZE << efx->rx_buffer_order,
3941751cc36SAlex Maftei (amaftei) DMA_FROM_DEVICE);
3951751cc36SAlex Maftei (amaftei) if (unlikely(dma_mapping_error(&efx->pci_dev->dev,
3961751cc36SAlex Maftei (amaftei) dma_addr))) {
3971751cc36SAlex Maftei (amaftei) __free_pages(page, efx->rx_buffer_order);
3981751cc36SAlex Maftei (amaftei) return -EIO;
3991751cc36SAlex Maftei (amaftei) }
4001751cc36SAlex Maftei (amaftei) state = page_address(page);
4011751cc36SAlex Maftei (amaftei) state->dma_addr = dma_addr;
4021751cc36SAlex Maftei (amaftei) } else {
4031751cc36SAlex Maftei (amaftei) state = page_address(page);
4041751cc36SAlex Maftei (amaftei) dma_addr = state->dma_addr;
4051751cc36SAlex Maftei (amaftei) }
4061751cc36SAlex Maftei (amaftei)
4071751cc36SAlex Maftei (amaftei) dma_addr += sizeof(struct efx_rx_page_state);
4081751cc36SAlex Maftei (amaftei) page_offset = sizeof(struct efx_rx_page_state);
4091751cc36SAlex Maftei (amaftei)
4101751cc36SAlex Maftei (amaftei) do {
4111751cc36SAlex Maftei (amaftei) index = rx_queue->added_count & rx_queue->ptr_mask;
4121751cc36SAlex Maftei (amaftei) rx_buf = efx_rx_buffer(rx_queue, index);
4131751cc36SAlex Maftei (amaftei) rx_buf->dma_addr = dma_addr + efx->rx_ip_align +
41486e85bf6SJesper Dangaard Brouer EFX_XDP_HEADROOM;
4151751cc36SAlex Maftei (amaftei) rx_buf->page = page;
4161751cc36SAlex Maftei (amaftei) rx_buf->page_offset = page_offset + efx->rx_ip_align +
41786e85bf6SJesper Dangaard Brouer EFX_XDP_HEADROOM;
4181751cc36SAlex Maftei (amaftei) rx_buf->len = efx->rx_dma_len;
4191751cc36SAlex Maftei (amaftei) rx_buf->flags = 0;
4201751cc36SAlex Maftei (amaftei) ++rx_queue->added_count;
4211751cc36SAlex Maftei (amaftei) get_page(page);
4221751cc36SAlex Maftei (amaftei) dma_addr += efx->rx_page_buf_step;
4231751cc36SAlex Maftei (amaftei) page_offset += efx->rx_page_buf_step;
4241751cc36SAlex Maftei (amaftei) } while (page_offset + efx->rx_page_buf_step <= PAGE_SIZE);
4251751cc36SAlex Maftei (amaftei)
4261751cc36SAlex Maftei (amaftei) rx_buf->flags = EFX_RX_BUF_LAST_IN_PAGE;
4271751cc36SAlex Maftei (amaftei) } while (++count < efx->rx_pages_per_batch);
4281751cc36SAlex Maftei (amaftei)
4291751cc36SAlex Maftei (amaftei) return 0;
4301751cc36SAlex Maftei (amaftei) }
4311751cc36SAlex Maftei (amaftei)
efx_rx_config_page_split(struct efx_nic * efx)4321751cc36SAlex Maftei (amaftei) void efx_rx_config_page_split(struct efx_nic *efx)
4331751cc36SAlex Maftei (amaftei) {
4341751cc36SAlex Maftei (amaftei) efx->rx_page_buf_step = ALIGN(efx->rx_dma_len + efx->rx_ip_align +
43586e85bf6SJesper Dangaard Brouer EFX_XDP_HEADROOM + EFX_XDP_TAILROOM,
4361751cc36SAlex Maftei (amaftei) EFX_RX_BUF_ALIGNMENT);
4371751cc36SAlex Maftei (amaftei) efx->rx_bufs_per_page = efx->rx_buffer_order ? 1 :
4381751cc36SAlex Maftei (amaftei) ((PAGE_SIZE - sizeof(struct efx_rx_page_state)) /
4391751cc36SAlex Maftei (amaftei) efx->rx_page_buf_step);
4401751cc36SAlex Maftei (amaftei) efx->rx_buffer_truesize = (PAGE_SIZE << efx->rx_buffer_order) /
4411751cc36SAlex Maftei (amaftei) efx->rx_bufs_per_page;
4421751cc36SAlex Maftei (amaftei) efx->rx_pages_per_batch = DIV_ROUND_UP(EFX_RX_PREFERRED_BATCH,
4431751cc36SAlex Maftei (amaftei) efx->rx_bufs_per_page);
4441751cc36SAlex Maftei (amaftei) }
4451751cc36SAlex Maftei (amaftei)
4461751cc36SAlex Maftei (amaftei) /* efx_fast_push_rx_descriptors - push new RX descriptors quickly
4471751cc36SAlex Maftei (amaftei) * @rx_queue: RX descriptor queue
4481751cc36SAlex Maftei (amaftei) *
4491751cc36SAlex Maftei (amaftei) * This will aim to fill the RX descriptor queue up to
4501751cc36SAlex Maftei (amaftei) * @rx_queue->@max_fill. If there is insufficient atomic
4511751cc36SAlex Maftei (amaftei) * memory to do so, a slow fill will be scheduled.
4521751cc36SAlex Maftei (amaftei) *
4531751cc36SAlex Maftei (amaftei) * The caller must provide serialisation (none is used here). In practise,
4541751cc36SAlex Maftei (amaftei) * this means this function must run from the NAPI handler, or be called
4551751cc36SAlex Maftei (amaftei) * when NAPI is disabled.
4561751cc36SAlex Maftei (amaftei) */
efx_fast_push_rx_descriptors(struct efx_rx_queue * rx_queue,bool atomic)4571751cc36SAlex Maftei (amaftei) void efx_fast_push_rx_descriptors(struct efx_rx_queue *rx_queue, bool atomic)
4581751cc36SAlex Maftei (amaftei) {
4591751cc36SAlex Maftei (amaftei) struct efx_nic *efx = rx_queue->efx;
4601751cc36SAlex Maftei (amaftei) unsigned int fill_level, batch_size;
4611751cc36SAlex Maftei (amaftei) int space, rc = 0;
4621751cc36SAlex Maftei (amaftei)
4631751cc36SAlex Maftei (amaftei) if (!rx_queue->refill_enabled)
4641751cc36SAlex Maftei (amaftei) return;
4651751cc36SAlex Maftei (amaftei)
4661751cc36SAlex Maftei (amaftei) /* Calculate current fill level, and exit if we don't need to fill */
4671751cc36SAlex Maftei (amaftei) fill_level = (rx_queue->added_count - rx_queue->removed_count);
4681751cc36SAlex Maftei (amaftei) EFX_WARN_ON_ONCE_PARANOID(fill_level > rx_queue->efx->rxq_entries);
4691751cc36SAlex Maftei (amaftei) if (fill_level >= rx_queue->fast_fill_trigger)
4701751cc36SAlex Maftei (amaftei) goto out;
4711751cc36SAlex Maftei (amaftei)
4721751cc36SAlex Maftei (amaftei) /* Record minimum fill level */
4731751cc36SAlex Maftei (amaftei) if (unlikely(fill_level < rx_queue->min_fill)) {
4741751cc36SAlex Maftei (amaftei) if (fill_level)
4751751cc36SAlex Maftei (amaftei) rx_queue->min_fill = fill_level;
4761751cc36SAlex Maftei (amaftei) }
4771751cc36SAlex Maftei (amaftei)
4781751cc36SAlex Maftei (amaftei) batch_size = efx->rx_pages_per_batch * efx->rx_bufs_per_page;
4791751cc36SAlex Maftei (amaftei) space = rx_queue->max_fill - fill_level;
4801751cc36SAlex Maftei (amaftei) EFX_WARN_ON_ONCE_PARANOID(space < batch_size);
4811751cc36SAlex Maftei (amaftei)
4821751cc36SAlex Maftei (amaftei) netif_vdbg(rx_queue->efx, rx_status, rx_queue->efx->net_dev,
4831751cc36SAlex Maftei (amaftei) "RX queue %d fast-filling descriptor ring from"
4841751cc36SAlex Maftei (amaftei) " level %d to level %d\n",
4851751cc36SAlex Maftei (amaftei) efx_rx_queue_index(rx_queue), fill_level,
4861751cc36SAlex Maftei (amaftei) rx_queue->max_fill);
4871751cc36SAlex Maftei (amaftei)
4881751cc36SAlex Maftei (amaftei) do {
4891751cc36SAlex Maftei (amaftei) rc = efx_init_rx_buffers(rx_queue, atomic);
4901751cc36SAlex Maftei (amaftei) if (unlikely(rc)) {
4911751cc36SAlex Maftei (amaftei) /* Ensure that we don't leave the rx queue empty */
4921751cc36SAlex Maftei (amaftei) efx_schedule_slow_fill(rx_queue);
4931751cc36SAlex Maftei (amaftei) goto out;
4941751cc36SAlex Maftei (amaftei) }
4951751cc36SAlex Maftei (amaftei) } while ((space -= batch_size) >= batch_size);
4961751cc36SAlex Maftei (amaftei)
4971751cc36SAlex Maftei (amaftei) netif_vdbg(rx_queue->efx, rx_status, rx_queue->efx->net_dev,
4981751cc36SAlex Maftei (amaftei) "RX queue %d fast-filled descriptor ring "
4991751cc36SAlex Maftei (amaftei) "to level %d\n", efx_rx_queue_index(rx_queue),
5001751cc36SAlex Maftei (amaftei) rx_queue->added_count - rx_queue->removed_count);
5011751cc36SAlex Maftei (amaftei)
5021751cc36SAlex Maftei (amaftei) out:
5031751cc36SAlex Maftei (amaftei) if (rx_queue->notified_count != rx_queue->added_count)
5041751cc36SAlex Maftei (amaftei) efx_nic_notify_rx_desc(rx_queue);
5051751cc36SAlex Maftei (amaftei) }
5063d95b884SAlex Maftei (amaftei)
5073d95b884SAlex Maftei (amaftei) /* Pass a received packet up through GRO. GRO can handle pages
5083d95b884SAlex Maftei (amaftei) * regardless of checksum state and skbs with a good checksum.
5093d95b884SAlex Maftei (amaftei) */
5103d95b884SAlex Maftei (amaftei) void
efx_rx_packet_gro(struct efx_channel * channel,struct efx_rx_buffer * rx_buf,unsigned int n_frags,u8 * eh,__wsum csum)5113d95b884SAlex Maftei (amaftei) efx_rx_packet_gro(struct efx_channel *channel, struct efx_rx_buffer *rx_buf,
5124d9c0a2dSEdward Cree unsigned int n_frags, u8 *eh, __wsum csum)
5133d95b884SAlex Maftei (amaftei) {
5143d95b884SAlex Maftei (amaftei) struct napi_struct *napi = &channel->napi_str;
5153d95b884SAlex Maftei (amaftei) struct efx_nic *efx = channel->efx;
5163d95b884SAlex Maftei (amaftei) struct sk_buff *skb;
5173d95b884SAlex Maftei (amaftei)
5183d95b884SAlex Maftei (amaftei) skb = napi_get_frags(napi);
5193d95b884SAlex Maftei (amaftei) if (unlikely(!skb)) {
5203d95b884SAlex Maftei (amaftei) struct efx_rx_queue *rx_queue;
5213d95b884SAlex Maftei (amaftei)
5223d95b884SAlex Maftei (amaftei) rx_queue = efx_channel_get_rx_queue(channel);
5233d95b884SAlex Maftei (amaftei) efx_free_rx_buffers(rx_queue, rx_buf, n_frags);
5243d95b884SAlex Maftei (amaftei) return;
5253d95b884SAlex Maftei (amaftei) }
5263d95b884SAlex Maftei (amaftei)
52706888543SEdward Cree if (efx->net_dev->features & NETIF_F_RXHASH &&
52806888543SEdward Cree efx_rx_buf_hash_valid(efx, eh))
5293d95b884SAlex Maftei (amaftei) skb_set_hash(skb, efx_rx_buf_hash(efx, eh),
5303d95b884SAlex Maftei (amaftei) PKT_HASH_TYPE_L3);
5314d9c0a2dSEdward Cree if (csum) {
5324d9c0a2dSEdward Cree skb->csum = csum;
5334d9c0a2dSEdward Cree skb->ip_summed = CHECKSUM_COMPLETE;
5344d9c0a2dSEdward Cree } else {
5353d95b884SAlex Maftei (amaftei) skb->ip_summed = ((rx_buf->flags & EFX_RX_PKT_CSUMMED) ?
5363d95b884SAlex Maftei (amaftei) CHECKSUM_UNNECESSARY : CHECKSUM_NONE);
5374d9c0a2dSEdward Cree }
5383d95b884SAlex Maftei (amaftei) skb->csum_level = !!(rx_buf->flags & EFX_RX_PKT_CSUM_LEVEL);
5393d95b884SAlex Maftei (amaftei)
5403d95b884SAlex Maftei (amaftei) for (;;) {
5413d95b884SAlex Maftei (amaftei) skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
5423d95b884SAlex Maftei (amaftei) rx_buf->page, rx_buf->page_offset,
5433d95b884SAlex Maftei (amaftei) rx_buf->len);
5443d95b884SAlex Maftei (amaftei) rx_buf->page = NULL;
5453d95b884SAlex Maftei (amaftei) skb->len += rx_buf->len;
5463d95b884SAlex Maftei (amaftei) if (skb_shinfo(skb)->nr_frags == n_frags)
5473d95b884SAlex Maftei (amaftei) break;
5483d95b884SAlex Maftei (amaftei)
5493d95b884SAlex Maftei (amaftei) rx_buf = efx_rx_buf_next(&channel->rx_queue, rx_buf);
5503d95b884SAlex Maftei (amaftei) }
5513d95b884SAlex Maftei (amaftei)
5523d95b884SAlex Maftei (amaftei) skb->data_len = skb->len;
5533d95b884SAlex Maftei (amaftei) skb->truesize += n_frags * efx->rx_buffer_truesize;
5543d95b884SAlex Maftei (amaftei)
5553d95b884SAlex Maftei (amaftei) skb_record_rx_queue(skb, channel->rx_queue.core_index);
5563d95b884SAlex Maftei (amaftei)
5573d95b884SAlex Maftei (amaftei) napi_gro_frags(napi);
5583d95b884SAlex Maftei (amaftei) }
559960f1627SAlex Maftei (amaftei)
efx_find_rss_context_entry(struct efx_nic * efx,u32 id)560*a9ee8d4aSEdward Cree struct efx_rss_context_priv *efx_find_rss_context_entry(struct efx_nic *efx,
561*a9ee8d4aSEdward Cree u32 id)
562960f1627SAlex Maftei (amaftei) {
563*a9ee8d4aSEdward Cree struct ethtool_rxfh_context *ctx;
564960f1627SAlex Maftei (amaftei)
565*a9ee8d4aSEdward Cree WARN_ON(!mutex_is_locked(&efx->net_dev->ethtool->rss_lock));
566960f1627SAlex Maftei (amaftei)
567*a9ee8d4aSEdward Cree ctx = xa_load(&efx->net_dev->ethtool->rss_ctx, id);
568*a9ee8d4aSEdward Cree if (!ctx)
569960f1627SAlex Maftei (amaftei) return NULL;
570*a9ee8d4aSEdward Cree return ethtool_rxfh_context_priv(ctx);
571960f1627SAlex Maftei (amaftei) }
572960f1627SAlex Maftei (amaftei)
efx_set_default_rx_indir_table(struct efx_nic * efx,u32 * indir)573*a9ee8d4aSEdward Cree void efx_set_default_rx_indir_table(struct efx_nic *efx, u32 *indir)
574960f1627SAlex Maftei (amaftei) {
575960f1627SAlex Maftei (amaftei) size_t i;
576960f1627SAlex Maftei (amaftei)
577*a9ee8d4aSEdward Cree for (i = 0; i < ARRAY_SIZE(efx->rss_context.rx_indir_table); i++)
578*a9ee8d4aSEdward Cree indir[i] = ethtool_rxfh_indir_default(i, efx->rss_spread);
579960f1627SAlex Maftei (amaftei) }
580f7226e0fSAlex Maftei (amaftei)
581f7226e0fSAlex Maftei (amaftei) /**
582f7226e0fSAlex Maftei (amaftei) * efx_filter_is_mc_recipient - test whether spec is a multicast recipient
583f7226e0fSAlex Maftei (amaftei) * @spec: Specification to test
584f7226e0fSAlex Maftei (amaftei) *
585f7226e0fSAlex Maftei (amaftei) * Return: %true if the specification is a non-drop RX filter that
586f7226e0fSAlex Maftei (amaftei) * matches a local MAC address I/G bit value of 1 or matches a local
587f7226e0fSAlex Maftei (amaftei) * IPv4 or IPv6 address value in the respective multicast address
588f7226e0fSAlex Maftei (amaftei) * range. Otherwise %false.
589f7226e0fSAlex Maftei (amaftei) */
efx_filter_is_mc_recipient(const struct efx_filter_spec * spec)590f7226e0fSAlex Maftei (amaftei) bool efx_filter_is_mc_recipient(const struct efx_filter_spec *spec)
591f7226e0fSAlex Maftei (amaftei) {
592f7226e0fSAlex Maftei (amaftei) if (!(spec->flags & EFX_FILTER_FLAG_RX) ||
593f7226e0fSAlex Maftei (amaftei) spec->dmaq_id == EFX_FILTER_RX_DMAQ_ID_DROP)
594f7226e0fSAlex Maftei (amaftei) return false;
595f7226e0fSAlex Maftei (amaftei)
596f7226e0fSAlex Maftei (amaftei) if (spec->match_flags &
597f7226e0fSAlex Maftei (amaftei) (EFX_FILTER_MATCH_LOC_MAC | EFX_FILTER_MATCH_LOC_MAC_IG) &&
598f7226e0fSAlex Maftei (amaftei) is_multicast_ether_addr(spec->loc_mac))
599f7226e0fSAlex Maftei (amaftei) return true;
600f7226e0fSAlex Maftei (amaftei)
601f7226e0fSAlex Maftei (amaftei) if ((spec->match_flags &
602f7226e0fSAlex Maftei (amaftei) (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) ==
603f7226e0fSAlex Maftei (amaftei) (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) {
604f7226e0fSAlex Maftei (amaftei) if (spec->ether_type == htons(ETH_P_IP) &&
605f7226e0fSAlex Maftei (amaftei) ipv4_is_multicast(spec->loc_host[0]))
606f7226e0fSAlex Maftei (amaftei) return true;
607f7226e0fSAlex Maftei (amaftei) if (spec->ether_type == htons(ETH_P_IPV6) &&
608f7226e0fSAlex Maftei (amaftei) ((const u8 *)spec->loc_host)[0] == 0xff)
609f7226e0fSAlex Maftei (amaftei) return true;
610f7226e0fSAlex Maftei (amaftei) }
611f7226e0fSAlex Maftei (amaftei)
612f7226e0fSAlex Maftei (amaftei) return false;
613f7226e0fSAlex Maftei (amaftei) }
614f7226e0fSAlex Maftei (amaftei)
efx_filter_spec_equal(const struct efx_filter_spec * left,const struct efx_filter_spec * right)615f7226e0fSAlex Maftei (amaftei) bool efx_filter_spec_equal(const struct efx_filter_spec *left,
616f7226e0fSAlex Maftei (amaftei) const struct efx_filter_spec *right)
617f7226e0fSAlex Maftei (amaftei) {
618f7226e0fSAlex Maftei (amaftei) if ((left->match_flags ^ right->match_flags) |
619f7226e0fSAlex Maftei (amaftei) ((left->flags ^ right->flags) &
620f7226e0fSAlex Maftei (amaftei) (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX)))
621f7226e0fSAlex Maftei (amaftei) return false;
622f7226e0fSAlex Maftei (amaftei)
623c2bf23e4SPieter Jansen van Vuuren return memcmp(&left->vport_id, &right->vport_id,
624f7226e0fSAlex Maftei (amaftei) sizeof(struct efx_filter_spec) -
625c2bf23e4SPieter Jansen van Vuuren offsetof(struct efx_filter_spec, vport_id)) == 0;
626f7226e0fSAlex Maftei (amaftei) }
627f7226e0fSAlex Maftei (amaftei)
efx_filter_spec_hash(const struct efx_filter_spec * spec)628f7226e0fSAlex Maftei (amaftei) u32 efx_filter_spec_hash(const struct efx_filter_spec *spec)
629f7226e0fSAlex Maftei (amaftei) {
630c2bf23e4SPieter Jansen van Vuuren BUILD_BUG_ON(offsetof(struct efx_filter_spec, vport_id) & 3);
631c2bf23e4SPieter Jansen van Vuuren return jhash2((const u32 *)&spec->vport_id,
632f7226e0fSAlex Maftei (amaftei) (sizeof(struct efx_filter_spec) -
633c2bf23e4SPieter Jansen van Vuuren offsetof(struct efx_filter_spec, vport_id)) / 4,
634f7226e0fSAlex Maftei (amaftei) 0);
635f7226e0fSAlex Maftei (amaftei) }
636f7226e0fSAlex Maftei (amaftei)
637f7226e0fSAlex Maftei (amaftei) #ifdef CONFIG_RFS_ACCEL
efx_rps_check_rule(struct efx_arfs_rule * rule,unsigned int filter_idx,bool * force)638f7226e0fSAlex Maftei (amaftei) bool efx_rps_check_rule(struct efx_arfs_rule *rule, unsigned int filter_idx,
639f7226e0fSAlex Maftei (amaftei) bool *force)
640f7226e0fSAlex Maftei (amaftei) {
641f7226e0fSAlex Maftei (amaftei) if (rule->filter_id == EFX_ARFS_FILTER_ID_PENDING) {
642f7226e0fSAlex Maftei (amaftei) /* ARFS is currently updating this entry, leave it */
643f7226e0fSAlex Maftei (amaftei) return false;
644f7226e0fSAlex Maftei (amaftei) }
645f7226e0fSAlex Maftei (amaftei) if (rule->filter_id == EFX_ARFS_FILTER_ID_ERROR) {
646f7226e0fSAlex Maftei (amaftei) /* ARFS tried and failed to update this, so it's probably out
647f7226e0fSAlex Maftei (amaftei) * of date. Remove the filter and the ARFS rule entry.
648f7226e0fSAlex Maftei (amaftei) */
649f7226e0fSAlex Maftei (amaftei) rule->filter_id = EFX_ARFS_FILTER_ID_REMOVING;
650f7226e0fSAlex Maftei (amaftei) *force = true;
651f7226e0fSAlex Maftei (amaftei) return true;
652f7226e0fSAlex Maftei (amaftei) } else if (WARN_ON(rule->filter_id != filter_idx)) { /* can't happen */
653f7226e0fSAlex Maftei (amaftei) /* ARFS has moved on, so old filter is not needed. Since we did
654f7226e0fSAlex Maftei (amaftei) * not mark the rule with EFX_ARFS_FILTER_ID_REMOVING, it will
655f7226e0fSAlex Maftei (amaftei) * not be removed by efx_rps_hash_del() subsequently.
656f7226e0fSAlex Maftei (amaftei) */
657f7226e0fSAlex Maftei (amaftei) *force = true;
658f7226e0fSAlex Maftei (amaftei) return true;
659f7226e0fSAlex Maftei (amaftei) }
660f7226e0fSAlex Maftei (amaftei) /* Remove it iff ARFS wants to. */
661f7226e0fSAlex Maftei (amaftei) return true;
662f7226e0fSAlex Maftei (amaftei) }
663f7226e0fSAlex Maftei (amaftei)
664f7226e0fSAlex Maftei (amaftei) static
efx_rps_hash_bucket(struct efx_nic * efx,const struct efx_filter_spec * spec)665f7226e0fSAlex Maftei (amaftei) struct hlist_head *efx_rps_hash_bucket(struct efx_nic *efx,
666f7226e0fSAlex Maftei (amaftei) const struct efx_filter_spec *spec)
667f7226e0fSAlex Maftei (amaftei) {
668f7226e0fSAlex Maftei (amaftei) u32 hash = efx_filter_spec_hash(spec);
669f7226e0fSAlex Maftei (amaftei)
670f7226e0fSAlex Maftei (amaftei) lockdep_assert_held(&efx->rps_hash_lock);
671f7226e0fSAlex Maftei (amaftei) if (!efx->rps_hash_table)
672f7226e0fSAlex Maftei (amaftei) return NULL;
673f7226e0fSAlex Maftei (amaftei) return &efx->rps_hash_table[hash % EFX_ARFS_HASH_TABLE_SIZE];
674f7226e0fSAlex Maftei (amaftei) }
675f7226e0fSAlex Maftei (amaftei)
efx_rps_hash_find(struct efx_nic * efx,const struct efx_filter_spec * spec)676f7226e0fSAlex Maftei (amaftei) struct efx_arfs_rule *efx_rps_hash_find(struct efx_nic *efx,
677f7226e0fSAlex Maftei (amaftei) const struct efx_filter_spec *spec)
678f7226e0fSAlex Maftei (amaftei) {
679f7226e0fSAlex Maftei (amaftei) struct efx_arfs_rule *rule;
680f7226e0fSAlex Maftei (amaftei) struct hlist_head *head;
681f7226e0fSAlex Maftei (amaftei) struct hlist_node *node;
682f7226e0fSAlex Maftei (amaftei)
683f7226e0fSAlex Maftei (amaftei) head = efx_rps_hash_bucket(efx, spec);
684f7226e0fSAlex Maftei (amaftei) if (!head)
685f7226e0fSAlex Maftei (amaftei) return NULL;
686f7226e0fSAlex Maftei (amaftei) hlist_for_each(node, head) {
687f7226e0fSAlex Maftei (amaftei) rule = container_of(node, struct efx_arfs_rule, node);
688f7226e0fSAlex Maftei (amaftei) if (efx_filter_spec_equal(spec, &rule->spec))
689f7226e0fSAlex Maftei (amaftei) return rule;
690f7226e0fSAlex Maftei (amaftei) }
691f7226e0fSAlex Maftei (amaftei) return NULL;
692f7226e0fSAlex Maftei (amaftei) }
693f7226e0fSAlex Maftei (amaftei)
efx_rps_hash_add(struct efx_nic * efx,const struct efx_filter_spec * spec,bool * new)694f7226e0fSAlex Maftei (amaftei) struct efx_arfs_rule *efx_rps_hash_add(struct efx_nic *efx,
695f7226e0fSAlex Maftei (amaftei) const struct efx_filter_spec *spec,
696f7226e0fSAlex Maftei (amaftei) bool *new)
697f7226e0fSAlex Maftei (amaftei) {
698f7226e0fSAlex Maftei (amaftei) struct efx_arfs_rule *rule;
699f7226e0fSAlex Maftei (amaftei) struct hlist_head *head;
700f7226e0fSAlex Maftei (amaftei) struct hlist_node *node;
701f7226e0fSAlex Maftei (amaftei)
702f7226e0fSAlex Maftei (amaftei) head = efx_rps_hash_bucket(efx, spec);
703f7226e0fSAlex Maftei (amaftei) if (!head)
704f7226e0fSAlex Maftei (amaftei) return NULL;
705f7226e0fSAlex Maftei (amaftei) hlist_for_each(node, head) {
706f7226e0fSAlex Maftei (amaftei) rule = container_of(node, struct efx_arfs_rule, node);
707f7226e0fSAlex Maftei (amaftei) if (efx_filter_spec_equal(spec, &rule->spec)) {
708f7226e0fSAlex Maftei (amaftei) *new = false;
709f7226e0fSAlex Maftei (amaftei) return rule;
710f7226e0fSAlex Maftei (amaftei) }
711f7226e0fSAlex Maftei (amaftei) }
712f7226e0fSAlex Maftei (amaftei) rule = kmalloc(sizeof(*rule), GFP_ATOMIC);
713f7226e0fSAlex Maftei (amaftei) *new = true;
714f7226e0fSAlex Maftei (amaftei) if (rule) {
715f7226e0fSAlex Maftei (amaftei) memcpy(&rule->spec, spec, sizeof(rule->spec));
716f7226e0fSAlex Maftei (amaftei) hlist_add_head(&rule->node, head);
717f7226e0fSAlex Maftei (amaftei) }
718f7226e0fSAlex Maftei (amaftei) return rule;
719f7226e0fSAlex Maftei (amaftei) }
720f7226e0fSAlex Maftei (amaftei)
efx_rps_hash_del(struct efx_nic * efx,const struct efx_filter_spec * spec)721f7226e0fSAlex Maftei (amaftei) void efx_rps_hash_del(struct efx_nic *efx, const struct efx_filter_spec *spec)
722f7226e0fSAlex Maftei (amaftei) {
723f7226e0fSAlex Maftei (amaftei) struct efx_arfs_rule *rule;
724f7226e0fSAlex Maftei (amaftei) struct hlist_head *head;
725f7226e0fSAlex Maftei (amaftei) struct hlist_node *node;
726f7226e0fSAlex Maftei (amaftei)
727f7226e0fSAlex Maftei (amaftei) head = efx_rps_hash_bucket(efx, spec);
728f7226e0fSAlex Maftei (amaftei) if (WARN_ON(!head))
729f7226e0fSAlex Maftei (amaftei) return;
730f7226e0fSAlex Maftei (amaftei) hlist_for_each(node, head) {
731f7226e0fSAlex Maftei (amaftei) rule = container_of(node, struct efx_arfs_rule, node);
732f7226e0fSAlex Maftei (amaftei) if (efx_filter_spec_equal(spec, &rule->spec)) {
733f7226e0fSAlex Maftei (amaftei) /* Someone already reused the entry. We know that if
734f7226e0fSAlex Maftei (amaftei) * this check doesn't fire (i.e. filter_id == REMOVING)
735f7226e0fSAlex Maftei (amaftei) * then the REMOVING mark was put there by our caller,
736f7226e0fSAlex Maftei (amaftei) * because caller is holding a lock on filter table and
737f7226e0fSAlex Maftei (amaftei) * only holders of that lock set REMOVING.
738f7226e0fSAlex Maftei (amaftei) */
739f7226e0fSAlex Maftei (amaftei) if (rule->filter_id != EFX_ARFS_FILTER_ID_REMOVING)
740f7226e0fSAlex Maftei (amaftei) return;
741f7226e0fSAlex Maftei (amaftei) hlist_del(node);
742f7226e0fSAlex Maftei (amaftei) kfree(rule);
743f7226e0fSAlex Maftei (amaftei) return;
744f7226e0fSAlex Maftei (amaftei) }
745f7226e0fSAlex Maftei (amaftei) }
746f7226e0fSAlex Maftei (amaftei) /* We didn't find it. */
747f7226e0fSAlex Maftei (amaftei) WARN_ON(1);
748f7226e0fSAlex Maftei (amaftei) }
749f7226e0fSAlex Maftei (amaftei) #endif
750f7226e0fSAlex Maftei (amaftei)
efx_probe_filters(struct efx_nic * efx)751f7226e0fSAlex Maftei (amaftei) int efx_probe_filters(struct efx_nic *efx)
752f7226e0fSAlex Maftei (amaftei) {
753f7226e0fSAlex Maftei (amaftei) int rc;
754f7226e0fSAlex Maftei (amaftei)
755f7226e0fSAlex Maftei (amaftei) mutex_lock(&efx->mac_lock);
756f7226e0fSAlex Maftei (amaftei) rc = efx->type->filter_table_probe(efx);
757f7226e0fSAlex Maftei (amaftei) if (rc)
758f7226e0fSAlex Maftei (amaftei) goto out_unlock;
759f7226e0fSAlex Maftei (amaftei)
760f7226e0fSAlex Maftei (amaftei) #ifdef CONFIG_RFS_ACCEL
761f7226e0fSAlex Maftei (amaftei) if (efx->type->offload_features & NETIF_F_NTUPLE) {
762f7226e0fSAlex Maftei (amaftei) struct efx_channel *channel;
763f7226e0fSAlex Maftei (amaftei) int i, success = 1;
764f7226e0fSAlex Maftei (amaftei)
765f7226e0fSAlex Maftei (amaftei) efx_for_each_channel(channel, efx) {
766f7226e0fSAlex Maftei (amaftei) channel->rps_flow_id =
767f7226e0fSAlex Maftei (amaftei) kcalloc(efx->type->max_rx_ip_filters,
768f7226e0fSAlex Maftei (amaftei) sizeof(*channel->rps_flow_id),
769f7226e0fSAlex Maftei (amaftei) GFP_KERNEL);
770f7226e0fSAlex Maftei (amaftei) if (!channel->rps_flow_id)
771f7226e0fSAlex Maftei (amaftei) success = 0;
772f7226e0fSAlex Maftei (amaftei) else
773f7226e0fSAlex Maftei (amaftei) for (i = 0;
774f7226e0fSAlex Maftei (amaftei) i < efx->type->max_rx_ip_filters;
775f7226e0fSAlex Maftei (amaftei) ++i)
776f7226e0fSAlex Maftei (amaftei) channel->rps_flow_id[i] =
777f7226e0fSAlex Maftei (amaftei) RPS_FLOW_ID_INVALID;
778f7226e0fSAlex Maftei (amaftei) channel->rfs_expire_index = 0;
779f7226e0fSAlex Maftei (amaftei) channel->rfs_filter_count = 0;
780f7226e0fSAlex Maftei (amaftei) }
781f7226e0fSAlex Maftei (amaftei)
782f7226e0fSAlex Maftei (amaftei) if (!success) {
783d5a306aeSZhipeng Lu efx_for_each_channel(channel, efx) {
784f7226e0fSAlex Maftei (amaftei) kfree(channel->rps_flow_id);
785d5a306aeSZhipeng Lu channel->rps_flow_id = NULL;
786d5a306aeSZhipeng Lu }
787f7226e0fSAlex Maftei (amaftei) efx->type->filter_table_remove(efx);
788f7226e0fSAlex Maftei (amaftei) rc = -ENOMEM;
789f7226e0fSAlex Maftei (amaftei) goto out_unlock;
790f7226e0fSAlex Maftei (amaftei) }
791f7226e0fSAlex Maftei (amaftei) }
792f7226e0fSAlex Maftei (amaftei) #endif
793f7226e0fSAlex Maftei (amaftei) out_unlock:
794f7226e0fSAlex Maftei (amaftei) mutex_unlock(&efx->mac_lock);
795f7226e0fSAlex Maftei (amaftei) return rc;
796f7226e0fSAlex Maftei (amaftei) }
797f7226e0fSAlex Maftei (amaftei)
efx_remove_filters(struct efx_nic * efx)798f7226e0fSAlex Maftei (amaftei) void efx_remove_filters(struct efx_nic *efx)
799f7226e0fSAlex Maftei (amaftei) {
800f7226e0fSAlex Maftei (amaftei) #ifdef CONFIG_RFS_ACCEL
801f7226e0fSAlex Maftei (amaftei) struct efx_channel *channel;
802f7226e0fSAlex Maftei (amaftei)
803f7226e0fSAlex Maftei (amaftei) efx_for_each_channel(channel, efx) {
804f7226e0fSAlex Maftei (amaftei) cancel_delayed_work_sync(&channel->filter_work);
805f7226e0fSAlex Maftei (amaftei) kfree(channel->rps_flow_id);
806788f920aSEdward Cree channel->rps_flow_id = NULL;
807f7226e0fSAlex Maftei (amaftei) }
808f7226e0fSAlex Maftei (amaftei) #endif
809f7226e0fSAlex Maftei (amaftei) efx->type->filter_table_remove(efx);
810f7226e0fSAlex Maftei (amaftei) }
81128abe825SEdward Cree
81228abe825SEdward Cree #ifdef CONFIG_RFS_ACCEL
81328abe825SEdward Cree
efx_filter_rfs_work(struct work_struct * data)81428abe825SEdward Cree static void efx_filter_rfs_work(struct work_struct *data)
81528abe825SEdward Cree {
81628abe825SEdward Cree struct efx_async_filter_insertion *req = container_of(data, struct efx_async_filter_insertion,
81728abe825SEdward Cree work);
8188cb03f4eSJonathan Cooper struct efx_nic *efx = efx_netdev_priv(req->net_dev);
81928abe825SEdward Cree struct efx_channel *channel = efx_get_channel(efx, req->rxq_index);
82028abe825SEdward Cree int slot_idx = req - efx->rps_slot;
82128abe825SEdward Cree struct efx_arfs_rule *rule;
82228abe825SEdward Cree u16 arfs_id = 0;
82328abe825SEdward Cree int rc;
82428abe825SEdward Cree
82528abe825SEdward Cree rc = efx->type->filter_insert(efx, &req->spec, true);
82628abe825SEdward Cree if (rc >= 0)
82728abe825SEdward Cree /* Discard 'priority' part of EF10+ filter ID (mcdi_filters) */
82828abe825SEdward Cree rc %= efx->type->max_rx_ip_filters;
82928abe825SEdward Cree if (efx->rps_hash_table) {
83028abe825SEdward Cree spin_lock_bh(&efx->rps_hash_lock);
83128abe825SEdward Cree rule = efx_rps_hash_find(efx, &req->spec);
83228abe825SEdward Cree /* The rule might have already gone, if someone else's request
83328abe825SEdward Cree * for the same spec was already worked and then expired before
83428abe825SEdward Cree * we got around to our work. In that case we have nothing
83528abe825SEdward Cree * tying us to an arfs_id, meaning that as soon as the filter
83628abe825SEdward Cree * is considered for expiry it will be removed.
83728abe825SEdward Cree */
83828abe825SEdward Cree if (rule) {
83928abe825SEdward Cree if (rc < 0)
84028abe825SEdward Cree rule->filter_id = EFX_ARFS_FILTER_ID_ERROR;
84128abe825SEdward Cree else
84228abe825SEdward Cree rule->filter_id = rc;
84328abe825SEdward Cree arfs_id = rule->arfs_id;
84428abe825SEdward Cree }
84528abe825SEdward Cree spin_unlock_bh(&efx->rps_hash_lock);
84628abe825SEdward Cree }
84728abe825SEdward Cree if (rc >= 0) {
84828abe825SEdward Cree /* Remember this so we can check whether to expire the filter
84928abe825SEdward Cree * later.
85028abe825SEdward Cree */
85128abe825SEdward Cree mutex_lock(&efx->rps_mutex);
85228abe825SEdward Cree if (channel->rps_flow_id[rc] == RPS_FLOW_ID_INVALID)
85328abe825SEdward Cree channel->rfs_filter_count++;
85428abe825SEdward Cree channel->rps_flow_id[rc] = req->flow_id;
85528abe825SEdward Cree mutex_unlock(&efx->rps_mutex);
85628abe825SEdward Cree
85728abe825SEdward Cree if (req->spec.ether_type == htons(ETH_P_IP))
85828abe825SEdward Cree netif_info(efx, rx_status, efx->net_dev,
85928abe825SEdward Cree "steering %s %pI4:%u:%pI4:%u to queue %u [flow %u filter %d id %u]\n",
86028abe825SEdward Cree (req->spec.ip_proto == IPPROTO_TCP) ? "TCP" : "UDP",
86128abe825SEdward Cree req->spec.rem_host, ntohs(req->spec.rem_port),
86228abe825SEdward Cree req->spec.loc_host, ntohs(req->spec.loc_port),
86328abe825SEdward Cree req->rxq_index, req->flow_id, rc, arfs_id);
86428abe825SEdward Cree else
86528abe825SEdward Cree netif_info(efx, rx_status, efx->net_dev,
86628abe825SEdward Cree "steering %s [%pI6]:%u:[%pI6]:%u to queue %u [flow %u filter %d id %u]\n",
86728abe825SEdward Cree (req->spec.ip_proto == IPPROTO_TCP) ? "TCP" : "UDP",
86828abe825SEdward Cree req->spec.rem_host, ntohs(req->spec.rem_port),
86928abe825SEdward Cree req->spec.loc_host, ntohs(req->spec.loc_port),
87028abe825SEdward Cree req->rxq_index, req->flow_id, rc, arfs_id);
87128abe825SEdward Cree channel->n_rfs_succeeded++;
87228abe825SEdward Cree } else {
87328abe825SEdward Cree if (req->spec.ether_type == htons(ETH_P_IP))
87428abe825SEdward Cree netif_dbg(efx, rx_status, efx->net_dev,
87528abe825SEdward Cree "failed to steer %s %pI4:%u:%pI4:%u to queue %u [flow %u rc %d id %u]\n",
87628abe825SEdward Cree (req->spec.ip_proto == IPPROTO_TCP) ? "TCP" : "UDP",
87728abe825SEdward Cree req->spec.rem_host, ntohs(req->spec.rem_port),
87828abe825SEdward Cree req->spec.loc_host, ntohs(req->spec.loc_port),
87928abe825SEdward Cree req->rxq_index, req->flow_id, rc, arfs_id);
88028abe825SEdward Cree else
88128abe825SEdward Cree netif_dbg(efx, rx_status, efx->net_dev,
88228abe825SEdward Cree "failed to steer %s [%pI6]:%u:[%pI6]:%u to queue %u [flow %u rc %d id %u]\n",
88328abe825SEdward Cree (req->spec.ip_proto == IPPROTO_TCP) ? "TCP" : "UDP",
88428abe825SEdward Cree req->spec.rem_host, ntohs(req->spec.rem_port),
88528abe825SEdward Cree req->spec.loc_host, ntohs(req->spec.loc_port),
88628abe825SEdward Cree req->rxq_index, req->flow_id, rc, arfs_id);
88728abe825SEdward Cree channel->n_rfs_failed++;
88828abe825SEdward Cree /* We're overloading the NIC's filter tables, so let's do a
88928abe825SEdward Cree * chunk of extra expiry work.
89028abe825SEdward Cree */
89128abe825SEdward Cree __efx_filter_rfs_expire(channel, min(channel->rfs_filter_count,
89228abe825SEdward Cree 100u));
89328abe825SEdward Cree }
89428abe825SEdward Cree
89528abe825SEdward Cree /* Release references */
89628abe825SEdward Cree clear_bit(slot_idx, &efx->rps_slot_map);
89728abe825SEdward Cree dev_put(req->net_dev);
89828abe825SEdward Cree }
89928abe825SEdward Cree
efx_filter_rfs(struct net_device * net_dev,const struct sk_buff * skb,u16 rxq_index,u32 flow_id)90028abe825SEdward Cree int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb,
90128abe825SEdward Cree u16 rxq_index, u32 flow_id)
90228abe825SEdward Cree {
9038cb03f4eSJonathan Cooper struct efx_nic *efx = efx_netdev_priv(net_dev);
90428abe825SEdward Cree struct efx_async_filter_insertion *req;
90528abe825SEdward Cree struct efx_arfs_rule *rule;
90628abe825SEdward Cree struct flow_keys fk;
90728abe825SEdward Cree int slot_idx;
90828abe825SEdward Cree bool new;
90928abe825SEdward Cree int rc;
91028abe825SEdward Cree
91128abe825SEdward Cree /* find a free slot */
91228abe825SEdward Cree for (slot_idx = 0; slot_idx < EFX_RPS_MAX_IN_FLIGHT; slot_idx++)
91328abe825SEdward Cree if (!test_and_set_bit(slot_idx, &efx->rps_slot_map))
91428abe825SEdward Cree break;
91528abe825SEdward Cree if (slot_idx >= EFX_RPS_MAX_IN_FLIGHT)
91628abe825SEdward Cree return -EBUSY;
91728abe825SEdward Cree
91828abe825SEdward Cree if (flow_id == RPS_FLOW_ID_INVALID) {
91928abe825SEdward Cree rc = -EINVAL;
92028abe825SEdward Cree goto out_clear;
92128abe825SEdward Cree }
92228abe825SEdward Cree
92328abe825SEdward Cree if (!skb_flow_dissect_flow_keys(skb, &fk, 0)) {
92428abe825SEdward Cree rc = -EPROTONOSUPPORT;
92528abe825SEdward Cree goto out_clear;
92628abe825SEdward Cree }
92728abe825SEdward Cree
92828abe825SEdward Cree if (fk.basic.n_proto != htons(ETH_P_IP) && fk.basic.n_proto != htons(ETH_P_IPV6)) {
92928abe825SEdward Cree rc = -EPROTONOSUPPORT;
93028abe825SEdward Cree goto out_clear;
93128abe825SEdward Cree }
93228abe825SEdward Cree if (fk.control.flags & FLOW_DIS_IS_FRAGMENT) {
93328abe825SEdward Cree rc = -EPROTONOSUPPORT;
93428abe825SEdward Cree goto out_clear;
93528abe825SEdward Cree }
93628abe825SEdward Cree
93728abe825SEdward Cree req = efx->rps_slot + slot_idx;
93828abe825SEdward Cree efx_filter_init_rx(&req->spec, EFX_FILTER_PRI_HINT,
93928abe825SEdward Cree efx->rx_scatter ? EFX_FILTER_FLAG_RX_SCATTER : 0,
94028abe825SEdward Cree rxq_index);
94128abe825SEdward Cree req->spec.match_flags =
94228abe825SEdward Cree EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
94328abe825SEdward Cree EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
94428abe825SEdward Cree EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT;
94528abe825SEdward Cree req->spec.ether_type = fk.basic.n_proto;
94628abe825SEdward Cree req->spec.ip_proto = fk.basic.ip_proto;
94728abe825SEdward Cree
94828abe825SEdward Cree if (fk.basic.n_proto == htons(ETH_P_IP)) {
94928abe825SEdward Cree req->spec.rem_host[0] = fk.addrs.v4addrs.src;
95028abe825SEdward Cree req->spec.loc_host[0] = fk.addrs.v4addrs.dst;
95128abe825SEdward Cree } else {
95228abe825SEdward Cree memcpy(req->spec.rem_host, &fk.addrs.v6addrs.src,
95328abe825SEdward Cree sizeof(struct in6_addr));
95428abe825SEdward Cree memcpy(req->spec.loc_host, &fk.addrs.v6addrs.dst,
95528abe825SEdward Cree sizeof(struct in6_addr));
95628abe825SEdward Cree }
95728abe825SEdward Cree
95828abe825SEdward Cree req->spec.rem_port = fk.ports.src;
95928abe825SEdward Cree req->spec.loc_port = fk.ports.dst;
96028abe825SEdward Cree
96128abe825SEdward Cree if (efx->rps_hash_table) {
96228abe825SEdward Cree /* Add it to ARFS hash table */
96328abe825SEdward Cree spin_lock(&efx->rps_hash_lock);
96428abe825SEdward Cree rule = efx_rps_hash_add(efx, &req->spec, &new);
96528abe825SEdward Cree if (!rule) {
96628abe825SEdward Cree rc = -ENOMEM;
96728abe825SEdward Cree goto out_unlock;
96828abe825SEdward Cree }
96928abe825SEdward Cree if (new)
97028abe825SEdward Cree rule->arfs_id = efx->rps_next_id++ % RPS_NO_FILTER;
97128abe825SEdward Cree rc = rule->arfs_id;
97228abe825SEdward Cree /* Skip if existing or pending filter already does the right thing */
97328abe825SEdward Cree if (!new && rule->rxq_index == rxq_index &&
97428abe825SEdward Cree rule->filter_id >= EFX_ARFS_FILTER_ID_PENDING)
97528abe825SEdward Cree goto out_unlock;
97628abe825SEdward Cree rule->rxq_index = rxq_index;
97728abe825SEdward Cree rule->filter_id = EFX_ARFS_FILTER_ID_PENDING;
97828abe825SEdward Cree spin_unlock(&efx->rps_hash_lock);
97928abe825SEdward Cree } else {
98028abe825SEdward Cree /* Without an ARFS hash table, we just use arfs_id 0 for all
98128abe825SEdward Cree * filters. This means if multiple flows hash to the same
98228abe825SEdward Cree * flow_id, all but the most recently touched will be eligible
98328abe825SEdward Cree * for expiry.
98428abe825SEdward Cree */
98528abe825SEdward Cree rc = 0;
98628abe825SEdward Cree }
98728abe825SEdward Cree
98828abe825SEdward Cree /* Queue the request */
98928abe825SEdward Cree dev_hold(req->net_dev = net_dev);
99028abe825SEdward Cree INIT_WORK(&req->work, efx_filter_rfs_work);
99128abe825SEdward Cree req->rxq_index = rxq_index;
99228abe825SEdward Cree req->flow_id = flow_id;
99328abe825SEdward Cree schedule_work(&req->work);
99428abe825SEdward Cree return rc;
99528abe825SEdward Cree out_unlock:
99628abe825SEdward Cree spin_unlock(&efx->rps_hash_lock);
99728abe825SEdward Cree out_clear:
99828abe825SEdward Cree clear_bit(slot_idx, &efx->rps_slot_map);
99928abe825SEdward Cree return rc;
100028abe825SEdward Cree }
100128abe825SEdward Cree
__efx_filter_rfs_expire(struct efx_channel * channel,unsigned int quota)100228abe825SEdward Cree bool __efx_filter_rfs_expire(struct efx_channel *channel, unsigned int quota)
100328abe825SEdward Cree {
100428abe825SEdward Cree bool (*expire_one)(struct efx_nic *efx, u32 flow_id, unsigned int index);
100528abe825SEdward Cree struct efx_nic *efx = channel->efx;
100628abe825SEdward Cree unsigned int index, size, start;
100728abe825SEdward Cree u32 flow_id;
100828abe825SEdward Cree
100928abe825SEdward Cree if (!mutex_trylock(&efx->rps_mutex))
101028abe825SEdward Cree return false;
101128abe825SEdward Cree expire_one = efx->type->filter_rfs_expire_one;
101228abe825SEdward Cree index = channel->rfs_expire_index;
101328abe825SEdward Cree start = index;
101428abe825SEdward Cree size = efx->type->max_rx_ip_filters;
101528abe825SEdward Cree while (quota) {
101628abe825SEdward Cree flow_id = channel->rps_flow_id[index];
101728abe825SEdward Cree
101828abe825SEdward Cree if (flow_id != RPS_FLOW_ID_INVALID) {
101928abe825SEdward Cree quota--;
102028abe825SEdward Cree if (expire_one(efx, flow_id, index)) {
102128abe825SEdward Cree netif_info(efx, rx_status, efx->net_dev,
102228abe825SEdward Cree "expired filter %d [channel %u flow %u]\n",
102328abe825SEdward Cree index, channel->channel, flow_id);
102428abe825SEdward Cree channel->rps_flow_id[index] = RPS_FLOW_ID_INVALID;
102528abe825SEdward Cree channel->rfs_filter_count--;
102628abe825SEdward Cree }
102728abe825SEdward Cree }
102828abe825SEdward Cree if (++index == size)
102928abe825SEdward Cree index = 0;
103028abe825SEdward Cree /* If we were called with a quota that exceeds the total number
103128abe825SEdward Cree * of filters in the table (which shouldn't happen, but could
103228abe825SEdward Cree * if two callers race), ensure that we don't loop forever -
103328abe825SEdward Cree * stop when we've examined every row of the table.
103428abe825SEdward Cree */
103528abe825SEdward Cree if (index == start)
103628abe825SEdward Cree break;
103728abe825SEdward Cree }
103828abe825SEdward Cree
103928abe825SEdward Cree channel->rfs_expire_index = index;
104028abe825SEdward Cree mutex_unlock(&efx->rps_mutex);
104128abe825SEdward Cree return true;
104228abe825SEdward Cree }
104328abe825SEdward Cree
104428abe825SEdward Cree #endif /* CONFIG_RFS_ACCEL */
1045