1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2013 - 2021 Intel Corporation. */
3
4 #include <generated/utsrelease.h>
5 #include <linux/crash_dump.h>
6 #include <linux/if_bridge.h>
7 #include <linux/if_macvlan.h>
8 #include <linux/module.h>
9 #include <net/pkt_cls.h>
10 #include <net/xdp_sock_drv.h>
11
12 /* Local includes */
13 #include "i40e.h"
14 #include "i40e_devids.h"
15 #include "i40e_diag.h"
16 #include "i40e_lan_hmc.h"
17 #include "i40e_virtchnl_pf.h"
18 #include "i40e_xsk.h"
19
20 /* All i40e tracepoints are defined by the include below, which
21 * must be included exactly once across the whole kernel with
22 * CREATE_TRACE_POINTS defined
23 */
24 #define CREATE_TRACE_POINTS
25 #include "i40e_trace.h"
26
27 const char i40e_driver_name[] = "i40e";
28 static const char i40e_driver_string[] =
29 "Intel(R) Ethernet Connection XL710 Network Driver";
30
31 static const char i40e_copyright[] = "Copyright (c) 2013 - 2019 Intel Corporation.";
32
33 /* a bit of forward declarations */
34 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi);
35 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired);
36 static int i40e_add_vsi(struct i40e_vsi *vsi);
37 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi);
38 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit, bool lock_acquired);
39 static int i40e_setup_misc_vector(struct i40e_pf *pf);
40 static void i40e_determine_queue_usage(struct i40e_pf *pf);
41 static int i40e_setup_pf_filter_control(struct i40e_pf *pf);
42 static void i40e_prep_for_reset(struct i40e_pf *pf);
43 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit,
44 bool lock_acquired);
45 static int i40e_reset(struct i40e_pf *pf);
46 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired);
47 static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf);
48 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf);
49 static bool i40e_check_recovery_mode(struct i40e_pf *pf);
50 static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw);
51 static void i40e_fdir_sb_setup(struct i40e_pf *pf);
52 static int i40e_veb_get_bw_info(struct i40e_veb *veb);
53 static int i40e_get_capabilities(struct i40e_pf *pf,
54 enum i40e_admin_queue_opc list_type);
55 static bool i40e_is_total_port_shutdown_enabled(struct i40e_pf *pf);
56
57 /* i40e_pci_tbl - PCI Device ID Table
58 *
59 * Last entry must be all 0s
60 *
61 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
62 * Class, Class Mask, private data (not used) }
63 */
64 static const struct pci_device_id i40e_pci_tbl[] = {
65 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0},
66 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0},
67 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0},
68 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0},
69 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0},
70 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0},
71 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0},
72 {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_BC), 0},
73 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0},
74 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0},
75 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_BC), 0},
76 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_SFP), 0},
77 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_B), 0},
78 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0},
79 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0},
80 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0},
81 {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0},
82 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0},
83 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0},
84 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722_A), 0},
85 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0},
86 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0},
87 {PCI_VDEVICE(INTEL, I40E_DEV_ID_X710_N3000), 0},
88 {PCI_VDEVICE(INTEL, I40E_DEV_ID_XXV710_N3000), 0},
89 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0},
90 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0},
91 /* required last entry */
92 {0, }
93 };
94 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl);
95
96 #define I40E_MAX_VF_COUNT 128
97 static int debug = -1;
98 module_param(debug, uint, 0);
99 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)");
100
101 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
102 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver");
103 MODULE_IMPORT_NS(LIBIE);
104 MODULE_LICENSE("GPL v2");
105
106 static struct workqueue_struct *i40e_wq;
107
netdev_hw_addr_refcnt(struct i40e_mac_filter * f,struct net_device * netdev,int delta)108 static void netdev_hw_addr_refcnt(struct i40e_mac_filter *f,
109 struct net_device *netdev, int delta)
110 {
111 struct netdev_hw_addr_list *ha_list;
112 struct netdev_hw_addr *ha;
113
114 if (!f || !netdev)
115 return;
116
117 if (is_unicast_ether_addr(f->macaddr) || is_link_local_ether_addr(f->macaddr))
118 ha_list = &netdev->uc;
119 else
120 ha_list = &netdev->mc;
121
122 netdev_hw_addr_list_for_each(ha, ha_list) {
123 if (ether_addr_equal(ha->addr, f->macaddr)) {
124 ha->refcount += delta;
125 if (ha->refcount <= 0)
126 ha->refcount = 1;
127 break;
128 }
129 }
130 }
131
132 /**
133 * i40e_hw_to_dev - get device pointer from the hardware structure
134 * @hw: pointer to the device HW structure
135 **/
i40e_hw_to_dev(struct i40e_hw * hw)136 struct device *i40e_hw_to_dev(struct i40e_hw *hw)
137 {
138 struct i40e_pf *pf = i40e_hw_to_pf(hw);
139
140 return &pf->pdev->dev;
141 }
142
143 /**
144 * i40e_allocate_dma_mem - OS specific memory alloc for shared code
145 * @hw: pointer to the HW structure
146 * @mem: ptr to mem struct to fill out
147 * @size: size of memory requested
148 * @alignment: what to align the allocation to
149 **/
i40e_allocate_dma_mem(struct i40e_hw * hw,struct i40e_dma_mem * mem,u64 size,u32 alignment)150 int i40e_allocate_dma_mem(struct i40e_hw *hw, struct i40e_dma_mem *mem,
151 u64 size, u32 alignment)
152 {
153 struct i40e_pf *pf = i40e_hw_to_pf(hw);
154
155 mem->size = ALIGN(size, alignment);
156 mem->va = dma_alloc_coherent(&pf->pdev->dev, mem->size, &mem->pa,
157 GFP_KERNEL);
158 if (!mem->va)
159 return -ENOMEM;
160
161 return 0;
162 }
163
164 /**
165 * i40e_free_dma_mem - OS specific memory free for shared code
166 * @hw: pointer to the HW structure
167 * @mem: ptr to mem struct to free
168 **/
i40e_free_dma_mem(struct i40e_hw * hw,struct i40e_dma_mem * mem)169 int i40e_free_dma_mem(struct i40e_hw *hw, struct i40e_dma_mem *mem)
170 {
171 struct i40e_pf *pf = i40e_hw_to_pf(hw);
172
173 dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa);
174 mem->va = NULL;
175 mem->pa = 0;
176 mem->size = 0;
177
178 return 0;
179 }
180
181 /**
182 * i40e_allocate_virt_mem - OS specific memory alloc for shared code
183 * @hw: pointer to the HW structure
184 * @mem: ptr to mem struct to fill out
185 * @size: size of memory requested
186 **/
i40e_allocate_virt_mem(struct i40e_hw * hw,struct i40e_virt_mem * mem,u32 size)187 int i40e_allocate_virt_mem(struct i40e_hw *hw, struct i40e_virt_mem *mem,
188 u32 size)
189 {
190 mem->size = size;
191 mem->va = kzalloc(size, GFP_KERNEL);
192
193 if (!mem->va)
194 return -ENOMEM;
195
196 return 0;
197 }
198
199 /**
200 * i40e_free_virt_mem - OS specific memory free for shared code
201 * @hw: pointer to the HW structure
202 * @mem: ptr to mem struct to free
203 **/
i40e_free_virt_mem(struct i40e_hw * hw,struct i40e_virt_mem * mem)204 int i40e_free_virt_mem(struct i40e_hw *hw, struct i40e_virt_mem *mem)
205 {
206 /* it's ok to kfree a NULL pointer */
207 kfree(mem->va);
208 mem->va = NULL;
209 mem->size = 0;
210
211 return 0;
212 }
213
214 /**
215 * i40e_get_lump - find a lump of free generic resource
216 * @pf: board private structure
217 * @pile: the pile of resource to search
218 * @needed: the number of items needed
219 * @id: an owner id to stick on the items assigned
220 *
221 * Returns the base item index of the lump, or negative for error
222 **/
i40e_get_lump(struct i40e_pf * pf,struct i40e_lump_tracking * pile,u16 needed,u16 id)223 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile,
224 u16 needed, u16 id)
225 {
226 int ret = -ENOMEM;
227 int i, j;
228
229 if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) {
230 dev_info(&pf->pdev->dev,
231 "param err: pile=%s needed=%d id=0x%04x\n",
232 pile ? "<valid>" : "<null>", needed, id);
233 return -EINVAL;
234 }
235
236 /* Allocate last queue in the pile for FDIR VSI queue
237 * so it doesn't fragment the qp_pile
238 */
239 if (pile == pf->qp_pile && pf->vsi[id]->type == I40E_VSI_FDIR) {
240 if (pile->list[pile->num_entries - 1] & I40E_PILE_VALID_BIT) {
241 dev_err(&pf->pdev->dev,
242 "Cannot allocate queue %d for I40E_VSI_FDIR\n",
243 pile->num_entries - 1);
244 return -ENOMEM;
245 }
246 pile->list[pile->num_entries - 1] = id | I40E_PILE_VALID_BIT;
247 return pile->num_entries - 1;
248 }
249
250 i = 0;
251 while (i < pile->num_entries) {
252 /* skip already allocated entries */
253 if (pile->list[i] & I40E_PILE_VALID_BIT) {
254 i++;
255 continue;
256 }
257
258 /* do we have enough in this lump? */
259 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) {
260 if (pile->list[i+j] & I40E_PILE_VALID_BIT)
261 break;
262 }
263
264 if (j == needed) {
265 /* there was enough, so assign it to the requestor */
266 for (j = 0; j < needed; j++)
267 pile->list[i+j] = id | I40E_PILE_VALID_BIT;
268 ret = i;
269 break;
270 }
271
272 /* not enough, so skip over it and continue looking */
273 i += j;
274 }
275
276 return ret;
277 }
278
279 /**
280 * i40e_put_lump - return a lump of generic resource
281 * @pile: the pile of resource to search
282 * @index: the base item index
283 * @id: the owner id of the items assigned
284 *
285 * Returns the count of items in the lump
286 **/
i40e_put_lump(struct i40e_lump_tracking * pile,u16 index,u16 id)287 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id)
288 {
289 int valid_id = (id | I40E_PILE_VALID_BIT);
290 int count = 0;
291 u16 i;
292
293 if (!pile || index >= pile->num_entries)
294 return -EINVAL;
295
296 for (i = index;
297 i < pile->num_entries && pile->list[i] == valid_id;
298 i++) {
299 pile->list[i] = 0;
300 count++;
301 }
302
303
304 return count;
305 }
306
307 /**
308 * i40e_find_vsi_from_id - searches for the vsi with the given id
309 * @pf: the pf structure to search for the vsi
310 * @id: id of the vsi it is searching for
311 **/
i40e_find_vsi_from_id(struct i40e_pf * pf,u16 id)312 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id)
313 {
314 struct i40e_vsi *vsi;
315 int i;
316
317 i40e_pf_for_each_vsi(pf, i, vsi)
318 if (vsi->id == id)
319 return vsi;
320
321 return NULL;
322 }
323
324 /**
325 * i40e_service_event_schedule - Schedule the service task to wake up
326 * @pf: board private structure
327 *
328 * If not already scheduled, this puts the task into the work queue
329 **/
i40e_service_event_schedule(struct i40e_pf * pf)330 void i40e_service_event_schedule(struct i40e_pf *pf)
331 {
332 if ((!test_bit(__I40E_DOWN, pf->state) &&
333 !test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) ||
334 test_bit(__I40E_RECOVERY_MODE, pf->state))
335 queue_work(i40e_wq, &pf->service_task);
336 }
337
338 /**
339 * i40e_tx_timeout - Respond to a Tx Hang
340 * @netdev: network interface device structure
341 * @txqueue: queue number timing out
342 *
343 * If any port has noticed a Tx timeout, it is likely that the whole
344 * device is munged, not just the one netdev port, so go for the full
345 * reset.
346 **/
i40e_tx_timeout(struct net_device * netdev,unsigned int txqueue)347 static void i40e_tx_timeout(struct net_device *netdev, unsigned int txqueue)
348 {
349 struct i40e_netdev_priv *np = netdev_priv(netdev);
350 struct i40e_vsi *vsi = np->vsi;
351 struct i40e_pf *pf = vsi->back;
352 struct i40e_ring *tx_ring = NULL;
353 unsigned int i;
354 u32 head, val;
355
356 pf->tx_timeout_count++;
357
358 /* with txqueue index, find the tx_ring struct */
359 for (i = 0; i < vsi->num_queue_pairs; i++) {
360 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) {
361 if (txqueue ==
362 vsi->tx_rings[i]->queue_index) {
363 tx_ring = vsi->tx_rings[i];
364 break;
365 }
366 }
367 }
368
369 if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20)))
370 pf->tx_timeout_recovery_level = 1; /* reset after some time */
371 else if (time_before(jiffies,
372 (pf->tx_timeout_last_recovery + netdev->watchdog_timeo)))
373 return; /* don't do any new action before the next timeout */
374
375 /* don't kick off another recovery if one is already pending */
376 if (test_and_set_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state))
377 return;
378
379 if (tx_ring) {
380 head = i40e_get_head(tx_ring);
381 /* Read interrupt register */
382 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
383 val = rd32(&pf->hw,
384 I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx +
385 tx_ring->vsi->base_vector - 1));
386 else
387 val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0);
388
389 netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n",
390 vsi->seid, txqueue, tx_ring->next_to_clean,
391 head, tx_ring->next_to_use,
392 readl(tx_ring->tail), val);
393 }
394
395 pf->tx_timeout_last_recovery = jiffies;
396 netdev_info(netdev, "tx_timeout recovery level %d, txqueue %d\n",
397 pf->tx_timeout_recovery_level, txqueue);
398
399 switch (pf->tx_timeout_recovery_level) {
400 case 1:
401 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
402 break;
403 case 2:
404 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
405 break;
406 case 3:
407 set_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);
408 break;
409 default:
410 netdev_err(netdev, "tx_timeout recovery unsuccessful, device is in non-recoverable state.\n");
411 set_bit(__I40E_DOWN_REQUESTED, pf->state);
412 set_bit(__I40E_VSI_DOWN_REQUESTED, vsi->state);
413 break;
414 }
415
416 i40e_service_event_schedule(pf);
417 pf->tx_timeout_recovery_level++;
418 }
419
420 /**
421 * i40e_get_vsi_stats_struct - Get System Network Statistics
422 * @vsi: the VSI we care about
423 *
424 * Returns the address of the device statistics structure.
425 * The statistics are actually updated from the service task.
426 **/
i40e_get_vsi_stats_struct(struct i40e_vsi * vsi)427 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi)
428 {
429 return &vsi->net_stats;
430 }
431
432 /**
433 * i40e_get_netdev_stats_struct_tx - populate stats from a Tx ring
434 * @ring: Tx ring to get statistics from
435 * @stats: statistics entry to be updated
436 **/
i40e_get_netdev_stats_struct_tx(struct i40e_ring * ring,struct rtnl_link_stats64 * stats)437 static void i40e_get_netdev_stats_struct_tx(struct i40e_ring *ring,
438 struct rtnl_link_stats64 *stats)
439 {
440 u64 bytes, packets;
441 unsigned int start;
442
443 do {
444 start = u64_stats_fetch_begin(&ring->syncp);
445 packets = ring->stats.packets;
446 bytes = ring->stats.bytes;
447 } while (u64_stats_fetch_retry(&ring->syncp, start));
448
449 stats->tx_packets += packets;
450 stats->tx_bytes += bytes;
451 }
452
453 /**
454 * i40e_get_netdev_stats_struct - Get statistics for netdev interface
455 * @netdev: network interface device structure
456 * @stats: data structure to store statistics
457 *
458 * Returns the address of the device statistics structure.
459 * The statistics are actually updated from the service task.
460 **/
i40e_get_netdev_stats_struct(struct net_device * netdev,struct rtnl_link_stats64 * stats)461 static void i40e_get_netdev_stats_struct(struct net_device *netdev,
462 struct rtnl_link_stats64 *stats)
463 {
464 struct i40e_netdev_priv *np = netdev_priv(netdev);
465 struct i40e_vsi *vsi = np->vsi;
466 struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi);
467 struct i40e_ring *ring;
468 int i;
469
470 if (test_bit(__I40E_VSI_DOWN, vsi->state))
471 return;
472
473 if (!vsi->tx_rings)
474 return;
475
476 rcu_read_lock();
477 for (i = 0; i < vsi->num_queue_pairs; i++) {
478 u64 bytes, packets;
479 unsigned int start;
480
481 ring = READ_ONCE(vsi->tx_rings[i]);
482 if (!ring)
483 continue;
484 i40e_get_netdev_stats_struct_tx(ring, stats);
485
486 if (i40e_enabled_xdp_vsi(vsi)) {
487 ring = READ_ONCE(vsi->xdp_rings[i]);
488 if (!ring)
489 continue;
490 i40e_get_netdev_stats_struct_tx(ring, stats);
491 }
492
493 ring = READ_ONCE(vsi->rx_rings[i]);
494 if (!ring)
495 continue;
496 do {
497 start = u64_stats_fetch_begin(&ring->syncp);
498 packets = ring->stats.packets;
499 bytes = ring->stats.bytes;
500 } while (u64_stats_fetch_retry(&ring->syncp, start));
501
502 stats->rx_packets += packets;
503 stats->rx_bytes += bytes;
504
505 }
506 rcu_read_unlock();
507
508 /* following stats updated by i40e_watchdog_subtask() */
509 stats->multicast = vsi_stats->multicast;
510 stats->tx_errors = vsi_stats->tx_errors;
511 stats->tx_dropped = vsi_stats->tx_dropped;
512 stats->rx_errors = vsi_stats->rx_errors;
513 stats->rx_dropped = vsi_stats->rx_dropped;
514 stats->rx_missed_errors = vsi_stats->rx_missed_errors;
515 stats->rx_crc_errors = vsi_stats->rx_crc_errors;
516 stats->rx_length_errors = vsi_stats->rx_length_errors;
517 }
518
519 /**
520 * i40e_vsi_reset_stats - Resets all stats of the given vsi
521 * @vsi: the VSI to have its stats reset
522 **/
i40e_vsi_reset_stats(struct i40e_vsi * vsi)523 void i40e_vsi_reset_stats(struct i40e_vsi *vsi)
524 {
525 struct rtnl_link_stats64 *ns;
526 int i;
527
528 if (!vsi)
529 return;
530
531 ns = i40e_get_vsi_stats_struct(vsi);
532 memset(ns, 0, sizeof(*ns));
533 memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets));
534 memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats));
535 memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets));
536 if (vsi->rx_rings && vsi->rx_rings[0]) {
537 for (i = 0; i < vsi->num_queue_pairs; i++) {
538 memset(&vsi->rx_rings[i]->stats, 0,
539 sizeof(vsi->rx_rings[i]->stats));
540 memset(&vsi->rx_rings[i]->rx_stats, 0,
541 sizeof(vsi->rx_rings[i]->rx_stats));
542 memset(&vsi->tx_rings[i]->stats, 0,
543 sizeof(vsi->tx_rings[i]->stats));
544 memset(&vsi->tx_rings[i]->tx_stats, 0,
545 sizeof(vsi->tx_rings[i]->tx_stats));
546 }
547 }
548 vsi->stat_offsets_loaded = false;
549 }
550
551 /**
552 * i40e_pf_reset_stats - Reset all of the stats for the given PF
553 * @pf: the PF to be reset
554 **/
i40e_pf_reset_stats(struct i40e_pf * pf)555 void i40e_pf_reset_stats(struct i40e_pf *pf)
556 {
557 struct i40e_veb *veb;
558 int i;
559
560 memset(&pf->stats, 0, sizeof(pf->stats));
561 memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets));
562 pf->stat_offsets_loaded = false;
563
564 i40e_pf_for_each_veb(pf, i, veb) {
565 memset(&veb->stats, 0, sizeof(veb->stats));
566 memset(&veb->stats_offsets, 0, sizeof(veb->stats_offsets));
567 memset(&veb->tc_stats, 0, sizeof(veb->tc_stats));
568 memset(&veb->tc_stats_offsets, 0, sizeof(veb->tc_stats_offsets));
569 veb->stat_offsets_loaded = false;
570 }
571 pf->hw_csum_rx_error = 0;
572 }
573
574 /**
575 * i40e_compute_pci_to_hw_id - compute index form PCI function.
576 * @vsi: ptr to the VSI to read from.
577 * @hw: ptr to the hardware info.
578 **/
i40e_compute_pci_to_hw_id(struct i40e_vsi * vsi,struct i40e_hw * hw)579 static u32 i40e_compute_pci_to_hw_id(struct i40e_vsi *vsi, struct i40e_hw *hw)
580 {
581 int pf_count = i40e_get_pf_count(hw);
582
583 if (vsi->type == I40E_VSI_SRIOV)
584 return (hw->port * BIT(7)) / pf_count + vsi->vf_id;
585
586 return hw->port + BIT(7);
587 }
588
589 /**
590 * i40e_stat_update64 - read and update a 64 bit stat from the chip.
591 * @hw: ptr to the hardware info.
592 * @hireg: the high 32 bit reg to read.
593 * @loreg: the low 32 bit reg to read.
594 * @offset_loaded: has the initial offset been loaded yet.
595 * @offset: ptr to current offset value.
596 * @stat: ptr to the stat.
597 *
598 * Since the device stats are not reset at PFReset, they will not
599 * be zeroed when the driver starts. We'll save the first values read
600 * and use them as offsets to be subtracted from the raw values in order
601 * to report stats that count from zero.
602 **/
i40e_stat_update64(struct i40e_hw * hw,u32 hireg,u32 loreg,bool offset_loaded,u64 * offset,u64 * stat)603 static void i40e_stat_update64(struct i40e_hw *hw, u32 hireg, u32 loreg,
604 bool offset_loaded, u64 *offset, u64 *stat)
605 {
606 u64 new_data;
607
608 new_data = rd64(hw, loreg);
609
610 if (!offset_loaded || new_data < *offset)
611 *offset = new_data;
612 *stat = new_data - *offset;
613 }
614
615 /**
616 * i40e_stat_update48 - read and update a 48 bit stat from the chip
617 * @hw: ptr to the hardware info
618 * @hireg: the high 32 bit reg to read
619 * @loreg: the low 32 bit reg to read
620 * @offset_loaded: has the initial offset been loaded yet
621 * @offset: ptr to current offset value
622 * @stat: ptr to the stat
623 *
624 * Since the device stats are not reset at PFReset, they likely will not
625 * be zeroed when the driver starts. We'll save the first values read
626 * and use them as offsets to be subtracted from the raw values in order
627 * to report stats that count from zero. In the process, we also manage
628 * the potential roll-over.
629 **/
i40e_stat_update48(struct i40e_hw * hw,u32 hireg,u32 loreg,bool offset_loaded,u64 * offset,u64 * stat)630 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
631 bool offset_loaded, u64 *offset, u64 *stat)
632 {
633 u64 new_data;
634
635 if (hw->device_id == I40E_DEV_ID_QEMU) {
636 new_data = rd32(hw, loreg);
637 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
638 } else {
639 new_data = rd64(hw, loreg);
640 }
641 if (!offset_loaded)
642 *offset = new_data;
643 if (likely(new_data >= *offset))
644 *stat = new_data - *offset;
645 else
646 *stat = (new_data + BIT_ULL(48)) - *offset;
647 *stat &= 0xFFFFFFFFFFFFULL;
648 }
649
650 /**
651 * i40e_stat_update32 - read and update a 32 bit stat from the chip
652 * @hw: ptr to the hardware info
653 * @reg: the hw reg to read
654 * @offset_loaded: has the initial offset been loaded yet
655 * @offset: ptr to current offset value
656 * @stat: ptr to the stat
657 **/
i40e_stat_update32(struct i40e_hw * hw,u32 reg,bool offset_loaded,u64 * offset,u64 * stat)658 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg,
659 bool offset_loaded, u64 *offset, u64 *stat)
660 {
661 u32 new_data;
662
663 new_data = rd32(hw, reg);
664 if (!offset_loaded)
665 *offset = new_data;
666 if (likely(new_data >= *offset))
667 *stat = (u32)(new_data - *offset);
668 else
669 *stat = (u32)((new_data + BIT_ULL(32)) - *offset);
670 }
671
672 /**
673 * i40e_stat_update_and_clear32 - read and clear hw reg, update a 32 bit stat
674 * @hw: ptr to the hardware info
675 * @reg: the hw reg to read and clear
676 * @stat: ptr to the stat
677 **/
i40e_stat_update_and_clear32(struct i40e_hw * hw,u32 reg,u64 * stat)678 static void i40e_stat_update_and_clear32(struct i40e_hw *hw, u32 reg, u64 *stat)
679 {
680 u32 new_data = rd32(hw, reg);
681
682 wr32(hw, reg, 1); /* must write a nonzero value to clear register */
683 *stat += new_data;
684 }
685
686 /**
687 * i40e_stats_update_rx_discards - update rx_discards.
688 * @vsi: ptr to the VSI to be updated.
689 * @hw: ptr to the hardware info.
690 * @stat_idx: VSI's stat_counter_idx.
691 * @offset_loaded: ptr to the VSI's stat_offsets_loaded.
692 * @stat_offset: ptr to stat_offset to store first read of specific register.
693 * @stat: ptr to VSI's stat to be updated.
694 **/
695 static void
i40e_stats_update_rx_discards(struct i40e_vsi * vsi,struct i40e_hw * hw,int stat_idx,bool offset_loaded,struct i40e_eth_stats * stat_offset,struct i40e_eth_stats * stat)696 i40e_stats_update_rx_discards(struct i40e_vsi *vsi, struct i40e_hw *hw,
697 int stat_idx, bool offset_loaded,
698 struct i40e_eth_stats *stat_offset,
699 struct i40e_eth_stats *stat)
700 {
701 i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx), offset_loaded,
702 &stat_offset->rx_discards, &stat->rx_discards);
703 i40e_stat_update64(hw,
704 I40E_GL_RXERR1H(i40e_compute_pci_to_hw_id(vsi, hw)),
705 I40E_GL_RXERR1L(i40e_compute_pci_to_hw_id(vsi, hw)),
706 offset_loaded, &stat_offset->rx_discards_other,
707 &stat->rx_discards_other);
708 }
709
710 /**
711 * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters.
712 * @vsi: the VSI to be updated
713 **/
i40e_update_eth_stats(struct i40e_vsi * vsi)714 void i40e_update_eth_stats(struct i40e_vsi *vsi)
715 {
716 int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx);
717 struct i40e_pf *pf = vsi->back;
718 struct i40e_hw *hw = &pf->hw;
719 struct i40e_eth_stats *oes;
720 struct i40e_eth_stats *es; /* device's eth stats */
721
722 es = &vsi->eth_stats;
723 oes = &vsi->eth_stats_offsets;
724
725 /* Gather up the stats that the hw collects */
726 i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
727 vsi->stat_offsets_loaded,
728 &oes->tx_errors, &es->tx_errors);
729 i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx),
730 vsi->stat_offsets_loaded,
731 &oes->rx_unknown_protocol, &es->rx_unknown_protocol);
732
733 i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
734 I40E_GLV_GORCL(stat_idx),
735 vsi->stat_offsets_loaded,
736 &oes->rx_bytes, &es->rx_bytes);
737 i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
738 I40E_GLV_UPRCL(stat_idx),
739 vsi->stat_offsets_loaded,
740 &oes->rx_unicast, &es->rx_unicast);
741 i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
742 I40E_GLV_MPRCL(stat_idx),
743 vsi->stat_offsets_loaded,
744 &oes->rx_multicast, &es->rx_multicast);
745 i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
746 I40E_GLV_BPRCL(stat_idx),
747 vsi->stat_offsets_loaded,
748 &oes->rx_broadcast, &es->rx_broadcast);
749
750 i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
751 I40E_GLV_GOTCL(stat_idx),
752 vsi->stat_offsets_loaded,
753 &oes->tx_bytes, &es->tx_bytes);
754 i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
755 I40E_GLV_UPTCL(stat_idx),
756 vsi->stat_offsets_loaded,
757 &oes->tx_unicast, &es->tx_unicast);
758 i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
759 I40E_GLV_MPTCL(stat_idx),
760 vsi->stat_offsets_loaded,
761 &oes->tx_multicast, &es->tx_multicast);
762 i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
763 I40E_GLV_BPTCL(stat_idx),
764 vsi->stat_offsets_loaded,
765 &oes->tx_broadcast, &es->tx_broadcast);
766
767 i40e_stats_update_rx_discards(vsi, hw, stat_idx,
768 vsi->stat_offsets_loaded, oes, es);
769
770 vsi->stat_offsets_loaded = true;
771 }
772
773 /**
774 * i40e_update_veb_stats - Update Switch component statistics
775 * @veb: the VEB being updated
776 **/
i40e_update_veb_stats(struct i40e_veb * veb)777 void i40e_update_veb_stats(struct i40e_veb *veb)
778 {
779 struct i40e_pf *pf = veb->pf;
780 struct i40e_hw *hw = &pf->hw;
781 struct i40e_eth_stats *oes;
782 struct i40e_eth_stats *es; /* device's eth stats */
783 struct i40e_veb_tc_stats *veb_oes;
784 struct i40e_veb_tc_stats *veb_es;
785 int i, idx = 0;
786
787 idx = veb->stats_idx;
788 es = &veb->stats;
789 oes = &veb->stats_offsets;
790 veb_es = &veb->tc_stats;
791 veb_oes = &veb->tc_stats_offsets;
792
793 /* Gather up the stats that the hw collects */
794 i40e_stat_update32(hw, I40E_GLSW_TDPC(idx),
795 veb->stat_offsets_loaded,
796 &oes->tx_discards, &es->tx_discards);
797 if (hw->revision_id > 0)
798 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx),
799 veb->stat_offsets_loaded,
800 &oes->rx_unknown_protocol,
801 &es->rx_unknown_protocol);
802 i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx),
803 veb->stat_offsets_loaded,
804 &oes->rx_bytes, &es->rx_bytes);
805 i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx),
806 veb->stat_offsets_loaded,
807 &oes->rx_unicast, &es->rx_unicast);
808 i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx),
809 veb->stat_offsets_loaded,
810 &oes->rx_multicast, &es->rx_multicast);
811 i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx),
812 veb->stat_offsets_loaded,
813 &oes->rx_broadcast, &es->rx_broadcast);
814
815 i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx),
816 veb->stat_offsets_loaded,
817 &oes->tx_bytes, &es->tx_bytes);
818 i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx),
819 veb->stat_offsets_loaded,
820 &oes->tx_unicast, &es->tx_unicast);
821 i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx),
822 veb->stat_offsets_loaded,
823 &oes->tx_multicast, &es->tx_multicast);
824 i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx),
825 veb->stat_offsets_loaded,
826 &oes->tx_broadcast, &es->tx_broadcast);
827 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
828 i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx),
829 I40E_GLVEBTC_RPCL(i, idx),
830 veb->stat_offsets_loaded,
831 &veb_oes->tc_rx_packets[i],
832 &veb_es->tc_rx_packets[i]);
833 i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx),
834 I40E_GLVEBTC_RBCL(i, idx),
835 veb->stat_offsets_loaded,
836 &veb_oes->tc_rx_bytes[i],
837 &veb_es->tc_rx_bytes[i]);
838 i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx),
839 I40E_GLVEBTC_TPCL(i, idx),
840 veb->stat_offsets_loaded,
841 &veb_oes->tc_tx_packets[i],
842 &veb_es->tc_tx_packets[i]);
843 i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx),
844 I40E_GLVEBTC_TBCL(i, idx),
845 veb->stat_offsets_loaded,
846 &veb_oes->tc_tx_bytes[i],
847 &veb_es->tc_tx_bytes[i]);
848 }
849 veb->stat_offsets_loaded = true;
850 }
851
852 /**
853 * i40e_update_vsi_stats - Update the vsi statistics counters.
854 * @vsi: the VSI to be updated
855 *
856 * There are a few instances where we store the same stat in a
857 * couple of different structs. This is partly because we have
858 * the netdev stats that need to be filled out, which is slightly
859 * different from the "eth_stats" defined by the chip and used in
860 * VF communications. We sort it out here.
861 **/
i40e_update_vsi_stats(struct i40e_vsi * vsi)862 static void i40e_update_vsi_stats(struct i40e_vsi *vsi)
863 {
864 u64 rx_page, rx_buf, rx_reuse, rx_alloc, rx_waive, rx_busy;
865 struct i40e_pf *pf = vsi->back;
866 struct rtnl_link_stats64 *ons;
867 struct rtnl_link_stats64 *ns; /* netdev stats */
868 struct i40e_eth_stats *oes;
869 struct i40e_eth_stats *es; /* device's eth stats */
870 u64 tx_restart, tx_busy;
871 struct i40e_ring *p;
872 u64 bytes, packets;
873 unsigned int start;
874 u64 tx_linearize;
875 u64 tx_force_wb;
876 u64 tx_stopped;
877 u64 rx_p, rx_b;
878 u64 tx_p, tx_b;
879 u16 q;
880
881 if (test_bit(__I40E_VSI_DOWN, vsi->state) ||
882 test_bit(__I40E_CONFIG_BUSY, pf->state))
883 return;
884
885 ns = i40e_get_vsi_stats_struct(vsi);
886 ons = &vsi->net_stats_offsets;
887 es = &vsi->eth_stats;
888 oes = &vsi->eth_stats_offsets;
889
890 /* Gather up the netdev and vsi stats that the driver collects
891 * on the fly during packet processing
892 */
893 rx_b = rx_p = 0;
894 tx_b = tx_p = 0;
895 tx_restart = tx_busy = tx_linearize = tx_force_wb = 0;
896 tx_stopped = 0;
897 rx_page = 0;
898 rx_buf = 0;
899 rx_reuse = 0;
900 rx_alloc = 0;
901 rx_waive = 0;
902 rx_busy = 0;
903 rcu_read_lock();
904 for (q = 0; q < vsi->num_queue_pairs; q++) {
905 /* locate Tx ring */
906 p = READ_ONCE(vsi->tx_rings[q]);
907 if (!p)
908 continue;
909
910 do {
911 start = u64_stats_fetch_begin(&p->syncp);
912 packets = p->stats.packets;
913 bytes = p->stats.bytes;
914 } while (u64_stats_fetch_retry(&p->syncp, start));
915 tx_b += bytes;
916 tx_p += packets;
917 tx_restart += p->tx_stats.restart_queue;
918 tx_busy += p->tx_stats.tx_busy;
919 tx_linearize += p->tx_stats.tx_linearize;
920 tx_force_wb += p->tx_stats.tx_force_wb;
921 tx_stopped += p->tx_stats.tx_stopped;
922
923 /* locate Rx ring */
924 p = READ_ONCE(vsi->rx_rings[q]);
925 if (!p)
926 continue;
927
928 do {
929 start = u64_stats_fetch_begin(&p->syncp);
930 packets = p->stats.packets;
931 bytes = p->stats.bytes;
932 } while (u64_stats_fetch_retry(&p->syncp, start));
933 rx_b += bytes;
934 rx_p += packets;
935 rx_buf += p->rx_stats.alloc_buff_failed;
936 rx_page += p->rx_stats.alloc_page_failed;
937 rx_reuse += p->rx_stats.page_reuse_count;
938 rx_alloc += p->rx_stats.page_alloc_count;
939 rx_waive += p->rx_stats.page_waive_count;
940 rx_busy += p->rx_stats.page_busy_count;
941
942 if (i40e_enabled_xdp_vsi(vsi)) {
943 /* locate XDP ring */
944 p = READ_ONCE(vsi->xdp_rings[q]);
945 if (!p)
946 continue;
947
948 do {
949 start = u64_stats_fetch_begin(&p->syncp);
950 packets = p->stats.packets;
951 bytes = p->stats.bytes;
952 } while (u64_stats_fetch_retry(&p->syncp, start));
953 tx_b += bytes;
954 tx_p += packets;
955 tx_restart += p->tx_stats.restart_queue;
956 tx_busy += p->tx_stats.tx_busy;
957 tx_linearize += p->tx_stats.tx_linearize;
958 tx_force_wb += p->tx_stats.tx_force_wb;
959 }
960 }
961 rcu_read_unlock();
962 vsi->tx_restart = tx_restart;
963 vsi->tx_busy = tx_busy;
964 vsi->tx_linearize = tx_linearize;
965 vsi->tx_force_wb = tx_force_wb;
966 vsi->tx_stopped = tx_stopped;
967 vsi->rx_page_failed = rx_page;
968 vsi->rx_buf_failed = rx_buf;
969 vsi->rx_page_reuse = rx_reuse;
970 vsi->rx_page_alloc = rx_alloc;
971 vsi->rx_page_waive = rx_waive;
972 vsi->rx_page_busy = rx_busy;
973
974 ns->rx_packets = rx_p;
975 ns->rx_bytes = rx_b;
976 ns->tx_packets = tx_p;
977 ns->tx_bytes = tx_b;
978
979 /* update netdev stats from eth stats */
980 i40e_update_eth_stats(vsi);
981 ons->tx_errors = oes->tx_errors;
982 ns->tx_errors = es->tx_errors;
983 ons->multicast = oes->rx_multicast;
984 ns->multicast = es->rx_multicast;
985 ons->rx_dropped = oes->rx_discards_other;
986 ns->rx_dropped = es->rx_discards_other;
987 ons->rx_missed_errors = oes->rx_discards;
988 ns->rx_missed_errors = es->rx_discards;
989 ons->tx_dropped = oes->tx_discards;
990 ns->tx_dropped = es->tx_discards;
991
992 /* pull in a couple PF stats if this is the main vsi */
993 if (vsi->type == I40E_VSI_MAIN) {
994 ns->rx_crc_errors = pf->stats.crc_errors;
995 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes;
996 ns->rx_length_errors = pf->stats.rx_length_errors;
997 }
998 }
999
1000 /**
1001 * i40e_update_pf_stats - Update the PF statistics counters.
1002 * @pf: the PF to be updated
1003 **/
i40e_update_pf_stats(struct i40e_pf * pf)1004 static void i40e_update_pf_stats(struct i40e_pf *pf)
1005 {
1006 struct i40e_hw_port_stats *osd = &pf->stats_offsets;
1007 struct i40e_hw_port_stats *nsd = &pf->stats;
1008 struct i40e_hw *hw = &pf->hw;
1009 u32 val;
1010 int i;
1011
1012 i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
1013 I40E_GLPRT_GORCL(hw->port),
1014 pf->stat_offsets_loaded,
1015 &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
1016 i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
1017 I40E_GLPRT_GOTCL(hw->port),
1018 pf->stat_offsets_loaded,
1019 &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
1020 i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
1021 pf->stat_offsets_loaded,
1022 &osd->eth.rx_discards,
1023 &nsd->eth.rx_discards);
1024 i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),
1025 I40E_GLPRT_UPRCL(hw->port),
1026 pf->stat_offsets_loaded,
1027 &osd->eth.rx_unicast,
1028 &nsd->eth.rx_unicast);
1029 i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
1030 I40E_GLPRT_MPRCL(hw->port),
1031 pf->stat_offsets_loaded,
1032 &osd->eth.rx_multicast,
1033 &nsd->eth.rx_multicast);
1034 i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),
1035 I40E_GLPRT_BPRCL(hw->port),
1036 pf->stat_offsets_loaded,
1037 &osd->eth.rx_broadcast,
1038 &nsd->eth.rx_broadcast);
1039 i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),
1040 I40E_GLPRT_UPTCL(hw->port),
1041 pf->stat_offsets_loaded,
1042 &osd->eth.tx_unicast,
1043 &nsd->eth.tx_unicast);
1044 i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),
1045 I40E_GLPRT_MPTCL(hw->port),
1046 pf->stat_offsets_loaded,
1047 &osd->eth.tx_multicast,
1048 &nsd->eth.tx_multicast);
1049 i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),
1050 I40E_GLPRT_BPTCL(hw->port),
1051 pf->stat_offsets_loaded,
1052 &osd->eth.tx_broadcast,
1053 &nsd->eth.tx_broadcast);
1054
1055 i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
1056 pf->stat_offsets_loaded,
1057 &osd->tx_dropped_link_down,
1058 &nsd->tx_dropped_link_down);
1059
1060 i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
1061 pf->stat_offsets_loaded,
1062 &osd->crc_errors, &nsd->crc_errors);
1063
1064 i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
1065 pf->stat_offsets_loaded,
1066 &osd->illegal_bytes, &nsd->illegal_bytes);
1067
1068 i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
1069 pf->stat_offsets_loaded,
1070 &osd->mac_local_faults,
1071 &nsd->mac_local_faults);
1072 i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
1073 pf->stat_offsets_loaded,
1074 &osd->mac_remote_faults,
1075 &nsd->mac_remote_faults);
1076
1077 i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
1078 pf->stat_offsets_loaded,
1079 &osd->rx_length_errors,
1080 &nsd->rx_length_errors);
1081
1082 i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
1083 pf->stat_offsets_loaded,
1084 &osd->link_xon_rx, &nsd->link_xon_rx);
1085 i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
1086 pf->stat_offsets_loaded,
1087 &osd->link_xon_tx, &nsd->link_xon_tx);
1088 i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
1089 pf->stat_offsets_loaded,
1090 &osd->link_xoff_rx, &nsd->link_xoff_rx);
1091 i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
1092 pf->stat_offsets_loaded,
1093 &osd->link_xoff_tx, &nsd->link_xoff_tx);
1094
1095 for (i = 0; i < 8; i++) {
1096 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),
1097 pf->stat_offsets_loaded,
1098 &osd->priority_xoff_rx[i],
1099 &nsd->priority_xoff_rx[i]);
1100 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i),
1101 pf->stat_offsets_loaded,
1102 &osd->priority_xon_rx[i],
1103 &nsd->priority_xon_rx[i]);
1104 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i),
1105 pf->stat_offsets_loaded,
1106 &osd->priority_xon_tx[i],
1107 &nsd->priority_xon_tx[i]);
1108 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),
1109 pf->stat_offsets_loaded,
1110 &osd->priority_xoff_tx[i],
1111 &nsd->priority_xoff_tx[i]);
1112 i40e_stat_update32(hw,
1113 I40E_GLPRT_RXON2OFFCNT(hw->port, i),
1114 pf->stat_offsets_loaded,
1115 &osd->priority_xon_2_xoff[i],
1116 &nsd->priority_xon_2_xoff[i]);
1117 }
1118
1119 i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
1120 I40E_GLPRT_PRC64L(hw->port),
1121 pf->stat_offsets_loaded,
1122 &osd->rx_size_64, &nsd->rx_size_64);
1123 i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
1124 I40E_GLPRT_PRC127L(hw->port),
1125 pf->stat_offsets_loaded,
1126 &osd->rx_size_127, &nsd->rx_size_127);
1127 i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
1128 I40E_GLPRT_PRC255L(hw->port),
1129 pf->stat_offsets_loaded,
1130 &osd->rx_size_255, &nsd->rx_size_255);
1131 i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
1132 I40E_GLPRT_PRC511L(hw->port),
1133 pf->stat_offsets_loaded,
1134 &osd->rx_size_511, &nsd->rx_size_511);
1135 i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
1136 I40E_GLPRT_PRC1023L(hw->port),
1137 pf->stat_offsets_loaded,
1138 &osd->rx_size_1023, &nsd->rx_size_1023);
1139 i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
1140 I40E_GLPRT_PRC1522L(hw->port),
1141 pf->stat_offsets_loaded,
1142 &osd->rx_size_1522, &nsd->rx_size_1522);
1143 i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
1144 I40E_GLPRT_PRC9522L(hw->port),
1145 pf->stat_offsets_loaded,
1146 &osd->rx_size_big, &nsd->rx_size_big);
1147
1148 i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
1149 I40E_GLPRT_PTC64L(hw->port),
1150 pf->stat_offsets_loaded,
1151 &osd->tx_size_64, &nsd->tx_size_64);
1152 i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
1153 I40E_GLPRT_PTC127L(hw->port),
1154 pf->stat_offsets_loaded,
1155 &osd->tx_size_127, &nsd->tx_size_127);
1156 i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
1157 I40E_GLPRT_PTC255L(hw->port),
1158 pf->stat_offsets_loaded,
1159 &osd->tx_size_255, &nsd->tx_size_255);
1160 i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
1161 I40E_GLPRT_PTC511L(hw->port),
1162 pf->stat_offsets_loaded,
1163 &osd->tx_size_511, &nsd->tx_size_511);
1164 i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
1165 I40E_GLPRT_PTC1023L(hw->port),
1166 pf->stat_offsets_loaded,
1167 &osd->tx_size_1023, &nsd->tx_size_1023);
1168 i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
1169 I40E_GLPRT_PTC1522L(hw->port),
1170 pf->stat_offsets_loaded,
1171 &osd->tx_size_1522, &nsd->tx_size_1522);
1172 i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
1173 I40E_GLPRT_PTC9522L(hw->port),
1174 pf->stat_offsets_loaded,
1175 &osd->tx_size_big, &nsd->tx_size_big);
1176
1177 i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
1178 pf->stat_offsets_loaded,
1179 &osd->rx_undersize, &nsd->rx_undersize);
1180 i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
1181 pf->stat_offsets_loaded,
1182 &osd->rx_fragments, &nsd->rx_fragments);
1183 i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
1184 pf->stat_offsets_loaded,
1185 &osd->rx_oversize, &nsd->rx_oversize);
1186 i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
1187 pf->stat_offsets_loaded,
1188 &osd->rx_jabber, &nsd->rx_jabber);
1189
1190 /* FDIR stats */
1191 i40e_stat_update_and_clear32(hw,
1192 I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(hw->pf_id)),
1193 &nsd->fd_atr_match);
1194 i40e_stat_update_and_clear32(hw,
1195 I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(hw->pf_id)),
1196 &nsd->fd_sb_match);
1197 i40e_stat_update_and_clear32(hw,
1198 I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(hw->pf_id)),
1199 &nsd->fd_atr_tunnel_match);
1200
1201 val = rd32(hw, I40E_PRTPM_EEE_STAT);
1202 nsd->tx_lpi_status =
1203 FIELD_GET(I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK, val);
1204 nsd->rx_lpi_status =
1205 FIELD_GET(I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK, val);
1206 i40e_stat_update32(hw, I40E_PRTPM_TLPIC,
1207 pf->stat_offsets_loaded,
1208 &osd->tx_lpi_count, &nsd->tx_lpi_count);
1209 i40e_stat_update32(hw, I40E_PRTPM_RLPIC,
1210 pf->stat_offsets_loaded,
1211 &osd->rx_lpi_count, &nsd->rx_lpi_count);
1212
1213 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) &&
1214 !test_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state))
1215 nsd->fd_sb_status = true;
1216 else
1217 nsd->fd_sb_status = false;
1218
1219 if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) &&
1220 !test_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))
1221 nsd->fd_atr_status = true;
1222 else
1223 nsd->fd_atr_status = false;
1224
1225 pf->stat_offsets_loaded = true;
1226 }
1227
1228 /**
1229 * i40e_update_stats - Update the various statistics counters.
1230 * @vsi: the VSI to be updated
1231 *
1232 * Update the various stats for this VSI and its related entities.
1233 **/
i40e_update_stats(struct i40e_vsi * vsi)1234 void i40e_update_stats(struct i40e_vsi *vsi)
1235 {
1236 struct i40e_pf *pf = vsi->back;
1237
1238 if (vsi->type == I40E_VSI_MAIN)
1239 i40e_update_pf_stats(pf);
1240
1241 i40e_update_vsi_stats(vsi);
1242 }
1243
1244 /**
1245 * i40e_count_filters - counts VSI mac filters
1246 * @vsi: the VSI to be searched
1247 *
1248 * Returns count of mac filters
1249 **/
i40e_count_filters(struct i40e_vsi * vsi)1250 int i40e_count_filters(struct i40e_vsi *vsi)
1251 {
1252 struct i40e_mac_filter *f;
1253 struct hlist_node *h;
1254 int bkt;
1255 int cnt = 0;
1256
1257 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1258 if (f->state == I40E_FILTER_NEW ||
1259 f->state == I40E_FILTER_ACTIVE)
1260 ++cnt;
1261 }
1262
1263 return cnt;
1264 }
1265
1266 /**
1267 * i40e_find_filter - Search VSI filter list for specific mac/vlan filter
1268 * @vsi: the VSI to be searched
1269 * @macaddr: the MAC address
1270 * @vlan: the vlan
1271 *
1272 * Returns ptr to the filter object or NULL
1273 **/
i40e_find_filter(struct i40e_vsi * vsi,const u8 * macaddr,s16 vlan)1274 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi,
1275 const u8 *macaddr, s16 vlan)
1276 {
1277 struct i40e_mac_filter *f;
1278 u64 key;
1279
1280 if (!vsi || !macaddr)
1281 return NULL;
1282
1283 key = i40e_addr_to_hkey(macaddr);
1284 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1285 if ((ether_addr_equal(macaddr, f->macaddr)) &&
1286 (vlan == f->vlan))
1287 return f;
1288 }
1289 return NULL;
1290 }
1291
1292 /**
1293 * i40e_find_mac - Find a mac addr in the macvlan filters list
1294 * @vsi: the VSI to be searched
1295 * @macaddr: the MAC address we are searching for
1296 *
1297 * Returns the first filter with the provided MAC address or NULL if
1298 * MAC address was not found
1299 **/
i40e_find_mac(struct i40e_vsi * vsi,const u8 * macaddr)1300 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr)
1301 {
1302 struct i40e_mac_filter *f;
1303 u64 key;
1304
1305 if (!vsi || !macaddr)
1306 return NULL;
1307
1308 key = i40e_addr_to_hkey(macaddr);
1309 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1310 if ((ether_addr_equal(macaddr, f->macaddr)))
1311 return f;
1312 }
1313 return NULL;
1314 }
1315
1316 /**
1317 * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode
1318 * @vsi: the VSI to be searched
1319 *
1320 * Returns true if VSI is in vlan mode or false otherwise
1321 **/
i40e_is_vsi_in_vlan(struct i40e_vsi * vsi)1322 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi)
1323 {
1324 /* If we have a PVID, always operate in VLAN mode */
1325 if (vsi->info.pvid)
1326 return true;
1327
1328 /* We need to operate in VLAN mode whenever we have any filters with
1329 * a VLAN other than I40E_VLAN_ALL. We could check the table each
1330 * time, incurring search cost repeatedly. However, we can notice two
1331 * things:
1332 *
1333 * 1) the only place where we can gain a VLAN filter is in
1334 * i40e_add_filter.
1335 *
1336 * 2) the only place where filters are actually removed is in
1337 * i40e_sync_filters_subtask.
1338 *
1339 * Thus, we can simply use a boolean value, has_vlan_filters which we
1340 * will set to true when we add a VLAN filter in i40e_add_filter. Then
1341 * we have to perform the full search after deleting filters in
1342 * i40e_sync_filters_subtask, but we already have to search
1343 * filters here and can perform the check at the same time. This
1344 * results in avoiding embedding a loop for VLAN mode inside another
1345 * loop over all the filters, and should maintain correctness as noted
1346 * above.
1347 */
1348 return vsi->has_vlan_filter;
1349 }
1350
1351 /**
1352 * i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary
1353 * @vsi: the VSI to configure
1354 * @tmp_add_list: list of filters ready to be added
1355 * @tmp_del_list: list of filters ready to be deleted
1356 * @vlan_filters: the number of active VLAN filters
1357 *
1358 * Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they
1359 * behave as expected. If we have any active VLAN filters remaining or about
1360 * to be added then we need to update non-VLAN filters to be marked as VLAN=0
1361 * so that they only match against untagged traffic. If we no longer have any
1362 * active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-1
1363 * so that they match against both tagged and untagged traffic. In this way,
1364 * we ensure that we correctly receive the desired traffic. This ensures that
1365 * when we have an active VLAN we will receive only untagged traffic and
1366 * traffic matching active VLANs. If we have no active VLANs then we will
1367 * operate in non-VLAN mode and receive all traffic, tagged or untagged.
1368 *
1369 * Finally, in a similar fashion, this function also corrects filters when
1370 * there is an active PVID assigned to this VSI.
1371 *
1372 * In case of memory allocation failure return -ENOMEM. Otherwise, return 0.
1373 *
1374 * This function is only expected to be called from within
1375 * i40e_sync_vsi_filters.
1376 *
1377 * NOTE: This function expects to be called while under the
1378 * mac_filter_hash_lock
1379 */
i40e_correct_mac_vlan_filters(struct i40e_vsi * vsi,struct hlist_head * tmp_add_list,struct hlist_head * tmp_del_list,int vlan_filters)1380 static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi,
1381 struct hlist_head *tmp_add_list,
1382 struct hlist_head *tmp_del_list,
1383 int vlan_filters)
1384 {
1385 s16 pvid = le16_to_cpu(vsi->info.pvid);
1386 struct i40e_mac_filter *f, *add_head;
1387 struct i40e_new_mac_filter *new;
1388 struct hlist_node *h;
1389 int bkt, new_vlan;
1390
1391 /* To determine if a particular filter needs to be replaced we
1392 * have the three following conditions:
1393 *
1394 * a) if we have a PVID assigned, then all filters which are
1395 * not marked as VLAN=PVID must be replaced with filters that
1396 * are.
1397 * b) otherwise, if we have any active VLANS, all filters
1398 * which are marked as VLAN=-1 must be replaced with
1399 * filters marked as VLAN=0
1400 * c) finally, if we do not have any active VLANS, all filters
1401 * which are marked as VLAN=0 must be replaced with filters
1402 * marked as VLAN=-1
1403 */
1404
1405 /* Update the filters about to be added in place */
1406 hlist_for_each_entry(new, tmp_add_list, hlist) {
1407 if (pvid && new->f->vlan != pvid)
1408 new->f->vlan = pvid;
1409 else if (vlan_filters && new->f->vlan == I40E_VLAN_ANY)
1410 new->f->vlan = 0;
1411 else if (!vlan_filters && new->f->vlan == 0)
1412 new->f->vlan = I40E_VLAN_ANY;
1413 }
1414
1415 /* Update the remaining active filters */
1416 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1417 /* Combine the checks for whether a filter needs to be changed
1418 * and then determine the new VLAN inside the if block, in
1419 * order to avoid duplicating code for adding the new filter
1420 * then deleting the old filter.
1421 */
1422 if ((pvid && f->vlan != pvid) ||
1423 (vlan_filters && f->vlan == I40E_VLAN_ANY) ||
1424 (!vlan_filters && f->vlan == 0)) {
1425 /* Determine the new vlan we will be adding */
1426 if (pvid)
1427 new_vlan = pvid;
1428 else if (vlan_filters)
1429 new_vlan = 0;
1430 else
1431 new_vlan = I40E_VLAN_ANY;
1432
1433 /* Create the new filter */
1434 add_head = i40e_add_filter(vsi, f->macaddr, new_vlan);
1435 if (!add_head)
1436 return -ENOMEM;
1437
1438 /* Create a temporary i40e_new_mac_filter */
1439 new = kzalloc(sizeof(*new), GFP_ATOMIC);
1440 if (!new)
1441 return -ENOMEM;
1442
1443 new->f = add_head;
1444 new->state = add_head->state;
1445
1446 /* Add the new filter to the tmp list */
1447 hlist_add_head(&new->hlist, tmp_add_list);
1448
1449 /* Put the original filter into the delete list */
1450 f->state = I40E_FILTER_REMOVE;
1451 hash_del(&f->hlist);
1452 hlist_add_head(&f->hlist, tmp_del_list);
1453 }
1454 }
1455
1456 vsi->has_vlan_filter = !!vlan_filters;
1457
1458 return 0;
1459 }
1460
1461 /**
1462 * i40e_get_vf_new_vlan - Get new vlan id on a vf
1463 * @vsi: the vsi to configure
1464 * @new_mac: new mac filter to be added
1465 * @f: existing mac filter, replaced with new_mac->f if new_mac is not NULL
1466 * @vlan_filters: the number of active VLAN filters
1467 * @trusted: flag if the VF is trusted
1468 *
1469 * Get new VLAN id based on current VLAN filters, trust, PVID
1470 * and vf-vlan-prune-disable flag.
1471 *
1472 * Returns the value of the new vlan filter or
1473 * the old value if no new filter is needed.
1474 */
i40e_get_vf_new_vlan(struct i40e_vsi * vsi,struct i40e_new_mac_filter * new_mac,struct i40e_mac_filter * f,int vlan_filters,bool trusted)1475 static s16 i40e_get_vf_new_vlan(struct i40e_vsi *vsi,
1476 struct i40e_new_mac_filter *new_mac,
1477 struct i40e_mac_filter *f,
1478 int vlan_filters,
1479 bool trusted)
1480 {
1481 s16 pvid = le16_to_cpu(vsi->info.pvid);
1482 struct i40e_pf *pf = vsi->back;
1483 bool is_any;
1484
1485 if (new_mac)
1486 f = new_mac->f;
1487
1488 if (pvid && f->vlan != pvid)
1489 return pvid;
1490
1491 is_any = (trusted ||
1492 !test_bit(I40E_FLAG_VF_VLAN_PRUNING_ENA, pf->flags));
1493
1494 if ((vlan_filters && f->vlan == I40E_VLAN_ANY) ||
1495 (!is_any && !vlan_filters && f->vlan == I40E_VLAN_ANY) ||
1496 (is_any && !vlan_filters && f->vlan == 0)) {
1497 if (is_any)
1498 return I40E_VLAN_ANY;
1499 else
1500 return 0;
1501 }
1502
1503 return f->vlan;
1504 }
1505
1506 /**
1507 * i40e_correct_vf_mac_vlan_filters - Correct non-VLAN VF filters if necessary
1508 * @vsi: the vsi to configure
1509 * @tmp_add_list: list of filters ready to be added
1510 * @tmp_del_list: list of filters ready to be deleted
1511 * @vlan_filters: the number of active VLAN filters
1512 * @trusted: flag if the VF is trusted
1513 *
1514 * Correct VF VLAN filters based on current VLAN filters, trust, PVID
1515 * and vf-vlan-prune-disable flag.
1516 *
1517 * In case of memory allocation failure return -ENOMEM. Otherwise, return 0.
1518 *
1519 * This function is only expected to be called from within
1520 * i40e_sync_vsi_filters.
1521 *
1522 * NOTE: This function expects to be called while under the
1523 * mac_filter_hash_lock
1524 */
i40e_correct_vf_mac_vlan_filters(struct i40e_vsi * vsi,struct hlist_head * tmp_add_list,struct hlist_head * tmp_del_list,int vlan_filters,bool trusted)1525 static int i40e_correct_vf_mac_vlan_filters(struct i40e_vsi *vsi,
1526 struct hlist_head *tmp_add_list,
1527 struct hlist_head *tmp_del_list,
1528 int vlan_filters,
1529 bool trusted)
1530 {
1531 struct i40e_mac_filter *f, *add_head;
1532 struct i40e_new_mac_filter *new_mac;
1533 struct hlist_node *h;
1534 int bkt, new_vlan;
1535
1536 hlist_for_each_entry(new_mac, tmp_add_list, hlist) {
1537 new_mac->f->vlan = i40e_get_vf_new_vlan(vsi, new_mac, NULL,
1538 vlan_filters, trusted);
1539 }
1540
1541 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1542 new_vlan = i40e_get_vf_new_vlan(vsi, NULL, f, vlan_filters,
1543 trusted);
1544 if (new_vlan != f->vlan) {
1545 add_head = i40e_add_filter(vsi, f->macaddr, new_vlan);
1546 if (!add_head)
1547 return -ENOMEM;
1548 /* Create a temporary i40e_new_mac_filter */
1549 new_mac = kzalloc(sizeof(*new_mac), GFP_ATOMIC);
1550 if (!new_mac)
1551 return -ENOMEM;
1552 new_mac->f = add_head;
1553 new_mac->state = add_head->state;
1554
1555 /* Add the new filter to the tmp list */
1556 hlist_add_head(&new_mac->hlist, tmp_add_list);
1557
1558 /* Put the original filter into the delete list */
1559 f->state = I40E_FILTER_REMOVE;
1560 hash_del(&f->hlist);
1561 hlist_add_head(&f->hlist, tmp_del_list);
1562 }
1563 }
1564
1565 vsi->has_vlan_filter = !!vlan_filters;
1566 return 0;
1567 }
1568
1569 /**
1570 * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM
1571 * @vsi: the PF Main VSI - inappropriate for any other VSI
1572 * @macaddr: the MAC address
1573 *
1574 * Remove whatever filter the firmware set up so the driver can manage
1575 * its own filtering intelligently.
1576 **/
i40e_rm_default_mac_filter(struct i40e_vsi * vsi,u8 * macaddr)1577 static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr)
1578 {
1579 struct i40e_aqc_remove_macvlan_element_data element;
1580 struct i40e_pf *pf = vsi->back;
1581
1582 /* Only appropriate for the PF main VSI */
1583 if (vsi->type != I40E_VSI_MAIN)
1584 return;
1585
1586 memset(&element, 0, sizeof(element));
1587 ether_addr_copy(element.mac_addr, macaddr);
1588 element.vlan_tag = 0;
1589 /* Ignore error returns, some firmware does it this way... */
1590 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
1591 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1592
1593 memset(&element, 0, sizeof(element));
1594 ether_addr_copy(element.mac_addr, macaddr);
1595 element.vlan_tag = 0;
1596 /* ...and some firmware does it this way. */
1597 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
1598 I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
1599 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1600 }
1601
1602 /**
1603 * i40e_add_filter - Add a mac/vlan filter to the VSI
1604 * @vsi: the VSI to be searched
1605 * @macaddr: the MAC address
1606 * @vlan: the vlan
1607 *
1608 * Returns ptr to the filter object or NULL when no memory available.
1609 *
1610 * NOTE: This function is expected to be called with mac_filter_hash_lock
1611 * being held.
1612 **/
i40e_add_filter(struct i40e_vsi * vsi,const u8 * macaddr,s16 vlan)1613 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi,
1614 const u8 *macaddr, s16 vlan)
1615 {
1616 struct i40e_mac_filter *f;
1617 u64 key;
1618
1619 if (!vsi || !macaddr)
1620 return NULL;
1621
1622 f = i40e_find_filter(vsi, macaddr, vlan);
1623 if (!f) {
1624 f = kzalloc(sizeof(*f), GFP_ATOMIC);
1625 if (!f)
1626 return NULL;
1627
1628 /* Update the boolean indicating if we need to function in
1629 * VLAN mode.
1630 */
1631 if (vlan >= 0)
1632 vsi->has_vlan_filter = true;
1633
1634 ether_addr_copy(f->macaddr, macaddr);
1635 f->vlan = vlan;
1636 f->state = I40E_FILTER_NEW;
1637 INIT_HLIST_NODE(&f->hlist);
1638
1639 key = i40e_addr_to_hkey(macaddr);
1640 hash_add(vsi->mac_filter_hash, &f->hlist, key);
1641
1642 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1643 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
1644 }
1645
1646 /* If we're asked to add a filter that has been marked for removal, it
1647 * is safe to simply restore it to active state. __i40e_del_filter
1648 * will have simply deleted any filters which were previously marked
1649 * NEW or FAILED, so if it is currently marked REMOVE it must have
1650 * previously been ACTIVE. Since we haven't yet run the sync filters
1651 * task, just restore this filter to the ACTIVE state so that the
1652 * sync task leaves it in place
1653 */
1654 if (f->state == I40E_FILTER_REMOVE)
1655 f->state = I40E_FILTER_ACTIVE;
1656
1657 return f;
1658 }
1659
1660 /**
1661 * __i40e_del_filter - Remove a specific filter from the VSI
1662 * @vsi: VSI to remove from
1663 * @f: the filter to remove from the list
1664 *
1665 * This function should be called instead of i40e_del_filter only if you know
1666 * the exact filter you will remove already, such as via i40e_find_filter or
1667 * i40e_find_mac.
1668 *
1669 * NOTE: This function is expected to be called with mac_filter_hash_lock
1670 * being held.
1671 * ANOTHER NOTE: This function MUST be called from within the context of
1672 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1673 * instead of list_for_each_entry().
1674 **/
__i40e_del_filter(struct i40e_vsi * vsi,struct i40e_mac_filter * f)1675 void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f)
1676 {
1677 if (!f)
1678 return;
1679
1680 /* If the filter was never added to firmware then we can just delete it
1681 * directly and we don't want to set the status to remove or else an
1682 * admin queue command will unnecessarily fire.
1683 */
1684 if ((f->state == I40E_FILTER_FAILED) ||
1685 (f->state == I40E_FILTER_NEW)) {
1686 hash_del(&f->hlist);
1687 kfree(f);
1688 } else {
1689 f->state = I40E_FILTER_REMOVE;
1690 }
1691
1692 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1693 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
1694 }
1695
1696 /**
1697 * i40e_del_filter - Remove a MAC/VLAN filter from the VSI
1698 * @vsi: the VSI to be searched
1699 * @macaddr: the MAC address
1700 * @vlan: the VLAN
1701 *
1702 * NOTE: This function is expected to be called with mac_filter_hash_lock
1703 * being held.
1704 * ANOTHER NOTE: This function MUST be called from within the context of
1705 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1706 * instead of list_for_each_entry().
1707 **/
i40e_del_filter(struct i40e_vsi * vsi,const u8 * macaddr,s16 vlan)1708 void i40e_del_filter(struct i40e_vsi *vsi, const u8 *macaddr, s16 vlan)
1709 {
1710 struct i40e_mac_filter *f;
1711
1712 if (!vsi || !macaddr)
1713 return;
1714
1715 f = i40e_find_filter(vsi, macaddr, vlan);
1716 __i40e_del_filter(vsi, f);
1717 }
1718
1719 /**
1720 * i40e_add_mac_filter - Add a MAC filter for all active VLANs
1721 * @vsi: the VSI to be searched
1722 * @macaddr: the mac address to be filtered
1723 *
1724 * If we're not in VLAN mode, just add the filter to I40E_VLAN_ANY. Otherwise,
1725 * go through all the macvlan filters and add a macvlan filter for each
1726 * unique vlan that already exists. If a PVID has been assigned, instead only
1727 * add the macaddr to that VLAN.
1728 *
1729 * Returns last filter added on success, else NULL
1730 **/
i40e_add_mac_filter(struct i40e_vsi * vsi,const u8 * macaddr)1731 struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi,
1732 const u8 *macaddr)
1733 {
1734 struct i40e_mac_filter *f, *add = NULL;
1735 struct hlist_node *h;
1736 int bkt;
1737
1738 if (vsi->info.pvid)
1739 return i40e_add_filter(vsi, macaddr,
1740 le16_to_cpu(vsi->info.pvid));
1741
1742 if (!i40e_is_vsi_in_vlan(vsi))
1743 return i40e_add_filter(vsi, macaddr, I40E_VLAN_ANY);
1744
1745 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1746 if (f->state == I40E_FILTER_REMOVE)
1747 continue;
1748 add = i40e_add_filter(vsi, macaddr, f->vlan);
1749 if (!add)
1750 return NULL;
1751 }
1752
1753 return add;
1754 }
1755
1756 /**
1757 * i40e_del_mac_filter - Remove a MAC filter from all VLANs
1758 * @vsi: the VSI to be searched
1759 * @macaddr: the mac address to be removed
1760 *
1761 * Removes a given MAC address from a VSI regardless of what VLAN it has been
1762 * associated with.
1763 *
1764 * Returns 0 for success, or error
1765 **/
i40e_del_mac_filter(struct i40e_vsi * vsi,const u8 * macaddr)1766 int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr)
1767 {
1768 struct i40e_mac_filter *f;
1769 struct hlist_node *h;
1770 bool found = false;
1771 int bkt;
1772
1773 lockdep_assert_held(&vsi->mac_filter_hash_lock);
1774 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1775 if (ether_addr_equal(macaddr, f->macaddr)) {
1776 __i40e_del_filter(vsi, f);
1777 found = true;
1778 }
1779 }
1780
1781 if (found)
1782 return 0;
1783 else
1784 return -ENOENT;
1785 }
1786
1787 /**
1788 * i40e_set_mac - NDO callback to set mac address
1789 * @netdev: network interface device structure
1790 * @p: pointer to an address structure
1791 *
1792 * Returns 0 on success, negative on failure
1793 **/
i40e_set_mac(struct net_device * netdev,void * p)1794 static int i40e_set_mac(struct net_device *netdev, void *p)
1795 {
1796 struct i40e_netdev_priv *np = netdev_priv(netdev);
1797 struct i40e_vsi *vsi = np->vsi;
1798 struct i40e_pf *pf = vsi->back;
1799 struct i40e_hw *hw = &pf->hw;
1800 struct sockaddr *addr = p;
1801
1802 if (!is_valid_ether_addr(addr->sa_data))
1803 return -EADDRNOTAVAIL;
1804
1805 if (test_bit(__I40E_DOWN, pf->state) ||
1806 test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
1807 return -EADDRNOTAVAIL;
1808
1809 if (ether_addr_equal(hw->mac.addr, addr->sa_data))
1810 netdev_info(netdev, "returning to hw mac address %pM\n",
1811 hw->mac.addr);
1812 else
1813 netdev_info(netdev, "set new mac address %pM\n", addr->sa_data);
1814
1815 /* Copy the address first, so that we avoid a possible race with
1816 * .set_rx_mode().
1817 * - Remove old address from MAC filter
1818 * - Copy new address
1819 * - Add new address to MAC filter
1820 */
1821 spin_lock_bh(&vsi->mac_filter_hash_lock);
1822 i40e_del_mac_filter(vsi, netdev->dev_addr);
1823 eth_hw_addr_set(netdev, addr->sa_data);
1824 i40e_add_mac_filter(vsi, netdev->dev_addr);
1825 spin_unlock_bh(&vsi->mac_filter_hash_lock);
1826
1827 if (vsi->type == I40E_VSI_MAIN) {
1828 int ret;
1829
1830 ret = i40e_aq_mac_address_write(hw, I40E_AQC_WRITE_TYPE_LAA_WOL,
1831 addr->sa_data, NULL);
1832 if (ret)
1833 netdev_info(netdev, "Ignoring error from firmware on LAA update, status %pe, AQ ret %s\n",
1834 ERR_PTR(ret),
1835 i40e_aq_str(hw, hw->aq.asq_last_status));
1836 }
1837
1838 /* schedule our worker thread which will take care of
1839 * applying the new filter changes
1840 */
1841 i40e_service_event_schedule(pf);
1842 return 0;
1843 }
1844
1845 /**
1846 * i40e_config_rss_aq - Prepare for RSS using AQ commands
1847 * @vsi: vsi structure
1848 * @seed: RSS hash seed
1849 * @lut: pointer to lookup table of lut_size
1850 * @lut_size: size of the lookup table
1851 **/
i40e_config_rss_aq(struct i40e_vsi * vsi,const u8 * seed,u8 * lut,u16 lut_size)1852 static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
1853 u8 *lut, u16 lut_size)
1854 {
1855 struct i40e_pf *pf = vsi->back;
1856 struct i40e_hw *hw = &pf->hw;
1857 int ret = 0;
1858
1859 if (seed) {
1860 struct i40e_aqc_get_set_rss_key_data *seed_dw =
1861 (struct i40e_aqc_get_set_rss_key_data *)seed;
1862 ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw);
1863 if (ret) {
1864 dev_info(&pf->pdev->dev,
1865 "Cannot set RSS key, err %pe aq_err %s\n",
1866 ERR_PTR(ret),
1867 i40e_aq_str(hw, hw->aq.asq_last_status));
1868 return ret;
1869 }
1870 }
1871 if (lut) {
1872 bool pf_lut = vsi->type == I40E_VSI_MAIN;
1873
1874 ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
1875 if (ret) {
1876 dev_info(&pf->pdev->dev,
1877 "Cannot set RSS lut, err %pe aq_err %s\n",
1878 ERR_PTR(ret),
1879 i40e_aq_str(hw, hw->aq.asq_last_status));
1880 return ret;
1881 }
1882 }
1883 return ret;
1884 }
1885
1886 /**
1887 * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used
1888 * @vsi: VSI structure
1889 **/
i40e_vsi_config_rss(struct i40e_vsi * vsi)1890 static int i40e_vsi_config_rss(struct i40e_vsi *vsi)
1891 {
1892 struct i40e_pf *pf = vsi->back;
1893 u8 seed[I40E_HKEY_ARRAY_SIZE];
1894 u8 *lut;
1895 int ret;
1896
1897 if (!test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps))
1898 return 0;
1899 if (!vsi->rss_size)
1900 vsi->rss_size = min_t(int, pf->alloc_rss_size,
1901 vsi->num_queue_pairs);
1902 if (!vsi->rss_size)
1903 return -EINVAL;
1904 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
1905 if (!lut)
1906 return -ENOMEM;
1907
1908 /* Use the user configured hash keys and lookup table if there is one,
1909 * otherwise use default
1910 */
1911 if (vsi->rss_lut_user)
1912 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
1913 else
1914 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
1915 if (vsi->rss_hkey_user)
1916 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
1917 else
1918 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
1919 ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size);
1920 kfree(lut);
1921 return ret;
1922 }
1923
1924 /**
1925 * i40e_vsi_setup_queue_map_mqprio - Prepares mqprio based tc_config
1926 * @vsi: the VSI being configured,
1927 * @ctxt: VSI context structure
1928 * @enabled_tc: number of traffic classes to enable
1929 *
1930 * Prepares VSI tc_config to have queue configurations based on MQPRIO options.
1931 **/
i40e_vsi_setup_queue_map_mqprio(struct i40e_vsi * vsi,struct i40e_vsi_context * ctxt,u8 enabled_tc)1932 static int i40e_vsi_setup_queue_map_mqprio(struct i40e_vsi *vsi,
1933 struct i40e_vsi_context *ctxt,
1934 u8 enabled_tc)
1935 {
1936 u16 qcount = 0, max_qcount, qmap, sections = 0;
1937 int i, override_q, pow, num_qps, ret;
1938 u8 netdev_tc = 0, offset = 0;
1939
1940 if (vsi->type != I40E_VSI_MAIN)
1941 return -EINVAL;
1942 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
1943 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
1944 vsi->tc_config.numtc = vsi->mqprio_qopt.qopt.num_tc;
1945 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
1946 num_qps = vsi->mqprio_qopt.qopt.count[0];
1947
1948 /* find the next higher power-of-2 of num queue pairs */
1949 pow = ilog2(num_qps);
1950 if (!is_power_of_2(num_qps))
1951 pow++;
1952 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
1953 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
1954
1955 /* Setup queue offset/count for all TCs for given VSI */
1956 max_qcount = vsi->mqprio_qopt.qopt.count[0];
1957 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1958 /* See if the given TC is enabled for the given VSI */
1959 if (vsi->tc_config.enabled_tc & BIT(i)) {
1960 offset = vsi->mqprio_qopt.qopt.offset[i];
1961 qcount = vsi->mqprio_qopt.qopt.count[i];
1962 if (qcount > max_qcount)
1963 max_qcount = qcount;
1964 vsi->tc_config.tc_info[i].qoffset = offset;
1965 vsi->tc_config.tc_info[i].qcount = qcount;
1966 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
1967 } else {
1968 /* TC is not enabled so set the offset to
1969 * default queue and allocate one queue
1970 * for the given TC.
1971 */
1972 vsi->tc_config.tc_info[i].qoffset = 0;
1973 vsi->tc_config.tc_info[i].qcount = 1;
1974 vsi->tc_config.tc_info[i].netdev_tc = 0;
1975 }
1976 }
1977
1978 /* Set actual Tx/Rx queue pairs */
1979 vsi->num_queue_pairs = offset + qcount;
1980
1981 /* Setup queue TC[0].qmap for given VSI context */
1982 ctxt->info.tc_mapping[0] = cpu_to_le16(qmap);
1983 ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
1984 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
1985 ctxt->info.valid_sections |= cpu_to_le16(sections);
1986
1987 /* Reconfigure RSS for main VSI with max queue count */
1988 vsi->rss_size = max_qcount;
1989 ret = i40e_vsi_config_rss(vsi);
1990 if (ret) {
1991 dev_info(&vsi->back->pdev->dev,
1992 "Failed to reconfig rss for num_queues (%u)\n",
1993 max_qcount);
1994 return ret;
1995 }
1996 vsi->reconfig_rss = true;
1997 dev_dbg(&vsi->back->pdev->dev,
1998 "Reconfigured rss with num_queues (%u)\n", max_qcount);
1999
2000 /* Find queue count available for channel VSIs and starting offset
2001 * for channel VSIs
2002 */
2003 override_q = vsi->mqprio_qopt.qopt.count[0];
2004 if (override_q && override_q < vsi->num_queue_pairs) {
2005 vsi->cnt_q_avail = vsi->num_queue_pairs - override_q;
2006 vsi->next_base_queue = override_q;
2007 }
2008 return 0;
2009 }
2010
2011 /**
2012 * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc
2013 * @vsi: the VSI being setup
2014 * @ctxt: VSI context structure
2015 * @enabled_tc: Enabled TCs bitmap
2016 * @is_add: True if called before Add VSI
2017 *
2018 * Setup VSI queue mapping for enabled traffic classes.
2019 **/
i40e_vsi_setup_queue_map(struct i40e_vsi * vsi,struct i40e_vsi_context * ctxt,u8 enabled_tc,bool is_add)2020 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
2021 struct i40e_vsi_context *ctxt,
2022 u8 enabled_tc,
2023 bool is_add)
2024 {
2025 struct i40e_pf *pf = vsi->back;
2026 u16 num_tc_qps = 0;
2027 u16 sections = 0;
2028 u8 netdev_tc = 0;
2029 u16 numtc = 1;
2030 u16 qcount;
2031 u8 offset;
2032 u16 qmap;
2033 int i;
2034
2035 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
2036 offset = 0;
2037 /* zero out queue mapping, it will get updated on the end of the function */
2038 memset(ctxt->info.queue_mapping, 0, sizeof(ctxt->info.queue_mapping));
2039
2040 if (vsi->type == I40E_VSI_MAIN) {
2041 /* This code helps add more queue to the VSI if we have
2042 * more cores than RSS can support, the higher cores will
2043 * be served by ATR or other filters. Furthermore, the
2044 * non-zero req_queue_pairs says that user requested a new
2045 * queue count via ethtool's set_channels, so use this
2046 * value for queues distribution across traffic classes
2047 * We need at least one queue pair for the interface
2048 * to be usable as we see in else statement.
2049 */
2050 if (vsi->req_queue_pairs > 0)
2051 vsi->num_queue_pairs = vsi->req_queue_pairs;
2052 else if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
2053 vsi->num_queue_pairs = pf->num_lan_msix;
2054 else
2055 vsi->num_queue_pairs = 1;
2056 }
2057
2058 /* Number of queues per enabled TC */
2059 if (vsi->type == I40E_VSI_MAIN ||
2060 (vsi->type == I40E_VSI_SRIOV && vsi->num_queue_pairs != 0))
2061 num_tc_qps = vsi->num_queue_pairs;
2062 else
2063 num_tc_qps = vsi->alloc_queue_pairs;
2064
2065 if (enabled_tc && test_bit(I40E_FLAG_DCB_ENA, vsi->back->flags)) {
2066 /* Find numtc from enabled TC bitmap */
2067 for (i = 0, numtc = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
2068 if (enabled_tc & BIT(i)) /* TC is enabled */
2069 numtc++;
2070 }
2071 if (!numtc) {
2072 dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n");
2073 numtc = 1;
2074 }
2075 num_tc_qps = num_tc_qps / numtc;
2076 num_tc_qps = min_t(int, num_tc_qps,
2077 i40e_pf_get_max_q_per_tc(pf));
2078 }
2079
2080 vsi->tc_config.numtc = numtc;
2081 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
2082
2083 /* Do not allow use more TC queue pairs than MSI-X vectors exist */
2084 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
2085 num_tc_qps = min_t(int, num_tc_qps, pf->num_lan_msix);
2086
2087 /* Setup queue offset/count for all TCs for given VSI */
2088 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
2089 /* See if the given TC is enabled for the given VSI */
2090 if (vsi->tc_config.enabled_tc & BIT(i)) {
2091 /* TC is enabled */
2092 int pow, num_qps;
2093
2094 switch (vsi->type) {
2095 case I40E_VSI_MAIN:
2096 if ((!test_bit(I40E_FLAG_FD_SB_ENA,
2097 pf->flags) &&
2098 !test_bit(I40E_FLAG_FD_ATR_ENA,
2099 pf->flags)) ||
2100 vsi->tc_config.enabled_tc != 1) {
2101 qcount = min_t(int, pf->alloc_rss_size,
2102 num_tc_qps);
2103 break;
2104 }
2105 fallthrough;
2106 case I40E_VSI_FDIR:
2107 case I40E_VSI_SRIOV:
2108 case I40E_VSI_VMDQ2:
2109 default:
2110 qcount = num_tc_qps;
2111 WARN_ON(i != 0);
2112 break;
2113 }
2114 vsi->tc_config.tc_info[i].qoffset = offset;
2115 vsi->tc_config.tc_info[i].qcount = qcount;
2116
2117 /* find the next higher power-of-2 of num queue pairs */
2118 num_qps = qcount;
2119 pow = 0;
2120 while (num_qps && (BIT_ULL(pow) < qcount)) {
2121 pow++;
2122 num_qps >>= 1;
2123 }
2124
2125 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
2126 qmap =
2127 (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
2128 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
2129
2130 offset += qcount;
2131 } else {
2132 /* TC is not enabled so set the offset to
2133 * default queue and allocate one queue
2134 * for the given TC.
2135 */
2136 vsi->tc_config.tc_info[i].qoffset = 0;
2137 vsi->tc_config.tc_info[i].qcount = 1;
2138 vsi->tc_config.tc_info[i].netdev_tc = 0;
2139
2140 qmap = 0;
2141 }
2142 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap);
2143 }
2144 /* Do not change previously set num_queue_pairs for PFs and VFs*/
2145 if ((vsi->type == I40E_VSI_MAIN && numtc != 1) ||
2146 (vsi->type == I40E_VSI_SRIOV && vsi->num_queue_pairs == 0) ||
2147 (vsi->type != I40E_VSI_MAIN && vsi->type != I40E_VSI_SRIOV))
2148 vsi->num_queue_pairs = offset;
2149
2150 /* Scheduler section valid can only be set for ADD VSI */
2151 if (is_add) {
2152 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
2153
2154 ctxt->info.up_enable_bits = enabled_tc;
2155 }
2156 if (vsi->type == I40E_VSI_SRIOV) {
2157 ctxt->info.mapping_flags |=
2158 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
2159 for (i = 0; i < vsi->num_queue_pairs; i++)
2160 ctxt->info.queue_mapping[i] =
2161 cpu_to_le16(vsi->base_queue + i);
2162 } else {
2163 ctxt->info.mapping_flags |=
2164 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
2165 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
2166 }
2167 ctxt->info.valid_sections |= cpu_to_le16(sections);
2168 }
2169
2170 /**
2171 * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address
2172 * @netdev: the netdevice
2173 * @addr: address to add
2174 *
2175 * Called by __dev_(mc|uc)_sync when an address needs to be added. We call
2176 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
2177 */
i40e_addr_sync(struct net_device * netdev,const u8 * addr)2178 static int i40e_addr_sync(struct net_device *netdev, const u8 *addr)
2179 {
2180 struct i40e_netdev_priv *np = netdev_priv(netdev);
2181 struct i40e_vsi *vsi = np->vsi;
2182
2183 if (i40e_add_mac_filter(vsi, addr))
2184 return 0;
2185 else
2186 return -ENOMEM;
2187 }
2188
2189 /**
2190 * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address
2191 * @netdev: the netdevice
2192 * @addr: address to add
2193 *
2194 * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call
2195 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
2196 */
i40e_addr_unsync(struct net_device * netdev,const u8 * addr)2197 static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr)
2198 {
2199 struct i40e_netdev_priv *np = netdev_priv(netdev);
2200 struct i40e_vsi *vsi = np->vsi;
2201
2202 /* Under some circumstances, we might receive a request to delete
2203 * our own device address from our uc list. Because we store the
2204 * device address in the VSI's MAC/VLAN filter list, we need to ignore
2205 * such requests and not delete our device address from this list.
2206 */
2207 if (ether_addr_equal(addr, netdev->dev_addr))
2208 return 0;
2209
2210 i40e_del_mac_filter(vsi, addr);
2211
2212 return 0;
2213 }
2214
2215 /**
2216 * i40e_set_rx_mode - NDO callback to set the netdev filters
2217 * @netdev: network interface device structure
2218 **/
i40e_set_rx_mode(struct net_device * netdev)2219 static void i40e_set_rx_mode(struct net_device *netdev)
2220 {
2221 struct i40e_netdev_priv *np = netdev_priv(netdev);
2222 struct i40e_vsi *vsi = np->vsi;
2223
2224 spin_lock_bh(&vsi->mac_filter_hash_lock);
2225
2226 __dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
2227 __dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
2228
2229 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2230
2231 /* check for other flag changes */
2232 if (vsi->current_netdev_flags != vsi->netdev->flags) {
2233 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2234 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
2235 }
2236 }
2237
2238 /**
2239 * i40e_undo_del_filter_entries - Undo the changes made to MAC filter entries
2240 * @vsi: Pointer to VSI struct
2241 * @from: Pointer to list which contains MAC filter entries - changes to
2242 * those entries needs to be undone.
2243 *
2244 * MAC filter entries from this list were slated for deletion.
2245 **/
i40e_undo_del_filter_entries(struct i40e_vsi * vsi,struct hlist_head * from)2246 static void i40e_undo_del_filter_entries(struct i40e_vsi *vsi,
2247 struct hlist_head *from)
2248 {
2249 struct i40e_mac_filter *f;
2250 struct hlist_node *h;
2251
2252 hlist_for_each_entry_safe(f, h, from, hlist) {
2253 u64 key = i40e_addr_to_hkey(f->macaddr);
2254
2255 /* Move the element back into MAC filter list*/
2256 hlist_del(&f->hlist);
2257 hash_add(vsi->mac_filter_hash, &f->hlist, key);
2258 }
2259 }
2260
2261 /**
2262 * i40e_undo_add_filter_entries - Undo the changes made to MAC filter entries
2263 * @vsi: Pointer to vsi struct
2264 * @from: Pointer to list which contains MAC filter entries - changes to
2265 * those entries needs to be undone.
2266 *
2267 * MAC filter entries from this list were slated for addition.
2268 **/
i40e_undo_add_filter_entries(struct i40e_vsi * vsi,struct hlist_head * from)2269 static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi,
2270 struct hlist_head *from)
2271 {
2272 struct i40e_new_mac_filter *new;
2273 struct hlist_node *h;
2274
2275 hlist_for_each_entry_safe(new, h, from, hlist) {
2276 /* We can simply free the wrapper structure */
2277 hlist_del(&new->hlist);
2278 netdev_hw_addr_refcnt(new->f, vsi->netdev, -1);
2279 kfree(new);
2280 }
2281 }
2282
2283 /**
2284 * i40e_next_filter - Get the next non-broadcast filter from a list
2285 * @next: pointer to filter in list
2286 *
2287 * Returns the next non-broadcast filter in the list. Required so that we
2288 * ignore broadcast filters within the list, since these are not handled via
2289 * the normal firmware update path.
2290 */
2291 static
i40e_next_filter(struct i40e_new_mac_filter * next)2292 struct i40e_new_mac_filter *i40e_next_filter(struct i40e_new_mac_filter *next)
2293 {
2294 hlist_for_each_entry_continue(next, hlist) {
2295 if (!is_broadcast_ether_addr(next->f->macaddr))
2296 return next;
2297 }
2298
2299 return NULL;
2300 }
2301
2302 /**
2303 * i40e_update_filter_state - Update filter state based on return data
2304 * from firmware
2305 * @count: Number of filters added
2306 * @add_list: return data from fw
2307 * @add_head: pointer to first filter in current batch
2308 *
2309 * MAC filter entries from list were slated to be added to device. Returns
2310 * number of successful filters. Note that 0 does NOT mean success!
2311 **/
2312 static int
i40e_update_filter_state(int count,struct i40e_aqc_add_macvlan_element_data * add_list,struct i40e_new_mac_filter * add_head)2313 i40e_update_filter_state(int count,
2314 struct i40e_aqc_add_macvlan_element_data *add_list,
2315 struct i40e_new_mac_filter *add_head)
2316 {
2317 int retval = 0;
2318 int i;
2319
2320 for (i = 0; i < count; i++) {
2321 /* Always check status of each filter. We don't need to check
2322 * the firmware return status because we pre-set the filter
2323 * status to I40E_AQC_MM_ERR_NO_RES when sending the filter
2324 * request to the adminq. Thus, if it no longer matches then
2325 * we know the filter is active.
2326 */
2327 if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) {
2328 add_head->state = I40E_FILTER_FAILED;
2329 } else {
2330 add_head->state = I40E_FILTER_ACTIVE;
2331 retval++;
2332 }
2333
2334 add_head = i40e_next_filter(add_head);
2335 if (!add_head)
2336 break;
2337 }
2338
2339 return retval;
2340 }
2341
2342 /**
2343 * i40e_aqc_del_filters - Request firmware to delete a set of filters
2344 * @vsi: ptr to the VSI
2345 * @vsi_name: name to display in messages
2346 * @list: the list of filters to send to firmware
2347 * @num_del: the number of filters to delete
2348 * @retval: Set to -EIO on failure to delete
2349 *
2350 * Send a request to firmware via AdminQ to delete a set of filters. Uses
2351 * *retval instead of a return value so that success does not force ret_val to
2352 * be set to 0. This ensures that a sequence of calls to this function
2353 * preserve the previous value of *retval on successful delete.
2354 */
2355 static
i40e_aqc_del_filters(struct i40e_vsi * vsi,const char * vsi_name,struct i40e_aqc_remove_macvlan_element_data * list,int num_del,int * retval)2356 void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name,
2357 struct i40e_aqc_remove_macvlan_element_data *list,
2358 int num_del, int *retval)
2359 {
2360 struct i40e_hw *hw = &vsi->back->hw;
2361 enum i40e_admin_queue_err aq_status;
2362 int aq_ret;
2363
2364 aq_ret = i40e_aq_remove_macvlan_v2(hw, vsi->seid, list, num_del, NULL,
2365 &aq_status);
2366
2367 /* Explicitly ignore and do not report when firmware returns ENOENT */
2368 if (aq_ret && !(aq_status == I40E_AQ_RC_ENOENT)) {
2369 *retval = -EIO;
2370 dev_info(&vsi->back->pdev->dev,
2371 "ignoring delete macvlan error on %s, err %pe, aq_err %s\n",
2372 vsi_name, ERR_PTR(aq_ret),
2373 i40e_aq_str(hw, aq_status));
2374 }
2375 }
2376
2377 /**
2378 * i40e_aqc_add_filters - Request firmware to add a set of filters
2379 * @vsi: ptr to the VSI
2380 * @vsi_name: name to display in messages
2381 * @list: the list of filters to send to firmware
2382 * @add_head: Position in the add hlist
2383 * @num_add: the number of filters to add
2384 *
2385 * Send a request to firmware via AdminQ to add a chunk of filters. Will set
2386 * __I40E_VSI_OVERFLOW_PROMISC bit in vsi->state if the firmware has run out of
2387 * space for more filters.
2388 */
2389 static
i40e_aqc_add_filters(struct i40e_vsi * vsi,const char * vsi_name,struct i40e_aqc_add_macvlan_element_data * list,struct i40e_new_mac_filter * add_head,int num_add)2390 void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name,
2391 struct i40e_aqc_add_macvlan_element_data *list,
2392 struct i40e_new_mac_filter *add_head,
2393 int num_add)
2394 {
2395 struct i40e_hw *hw = &vsi->back->hw;
2396 enum i40e_admin_queue_err aq_status;
2397 int fcnt;
2398
2399 i40e_aq_add_macvlan_v2(hw, vsi->seid, list, num_add, NULL, &aq_status);
2400 fcnt = i40e_update_filter_state(num_add, list, add_head);
2401
2402 if (fcnt != num_add) {
2403 if (vsi->type == I40E_VSI_MAIN) {
2404 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2405 dev_warn(&vsi->back->pdev->dev,
2406 "Error %s adding RX filters on %s, promiscuous mode forced on\n",
2407 i40e_aq_str(hw, aq_status), vsi_name);
2408 } else if (vsi->type == I40E_VSI_SRIOV ||
2409 vsi->type == I40E_VSI_VMDQ1 ||
2410 vsi->type == I40E_VSI_VMDQ2) {
2411 dev_warn(&vsi->back->pdev->dev,
2412 "Error %s adding RX filters on %s, please set promiscuous on manually for %s\n",
2413 i40e_aq_str(hw, aq_status), vsi_name,
2414 vsi_name);
2415 } else {
2416 dev_warn(&vsi->back->pdev->dev,
2417 "Error %s adding RX filters on %s, incorrect VSI type: %i.\n",
2418 i40e_aq_str(hw, aq_status), vsi_name,
2419 vsi->type);
2420 }
2421 }
2422 }
2423
2424 /**
2425 * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags
2426 * @vsi: pointer to the VSI
2427 * @vsi_name: the VSI name
2428 * @f: filter data
2429 *
2430 * This function sets or clears the promiscuous broadcast flags for VLAN
2431 * filters in order to properly receive broadcast frames. Assumes that only
2432 * broadcast filters are passed.
2433 *
2434 * Returns status indicating success or failure;
2435 **/
2436 static int
i40e_aqc_broadcast_filter(struct i40e_vsi * vsi,const char * vsi_name,struct i40e_mac_filter * f)2437 i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name,
2438 struct i40e_mac_filter *f)
2439 {
2440 bool enable = f->state == I40E_FILTER_NEW;
2441 struct i40e_hw *hw = &vsi->back->hw;
2442 int aq_ret;
2443
2444 if (f->vlan == I40E_VLAN_ANY) {
2445 aq_ret = i40e_aq_set_vsi_broadcast(hw,
2446 vsi->seid,
2447 enable,
2448 NULL);
2449 } else {
2450 aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw,
2451 vsi->seid,
2452 enable,
2453 f->vlan,
2454 NULL);
2455 }
2456
2457 if (aq_ret) {
2458 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2459 dev_warn(&vsi->back->pdev->dev,
2460 "Error %s, forcing overflow promiscuous on %s\n",
2461 i40e_aq_str(hw, hw->aq.asq_last_status),
2462 vsi_name);
2463 }
2464
2465 return aq_ret;
2466 }
2467
2468 /**
2469 * i40e_set_promiscuous - set promiscuous mode
2470 * @pf: board private structure
2471 * @promisc: promisc on or off
2472 *
2473 * There are different ways of setting promiscuous mode on a PF depending on
2474 * what state/environment we're in. This identifies and sets it appropriately.
2475 * Returns 0 on success.
2476 **/
i40e_set_promiscuous(struct i40e_pf * pf,bool promisc)2477 static int i40e_set_promiscuous(struct i40e_pf *pf, bool promisc)
2478 {
2479 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);
2480 struct i40e_hw *hw = &pf->hw;
2481 int aq_ret;
2482
2483 if (vsi->type == I40E_VSI_MAIN &&
2484 i40e_pf_get_main_veb(pf) &&
2485 !test_bit(I40E_FLAG_MFP_ENA, pf->flags)) {
2486 /* set defport ON for Main VSI instead of true promisc
2487 * this way we will get all unicast/multicast and VLAN
2488 * promisc behavior but will not get VF or VMDq traffic
2489 * replicated on the Main VSI.
2490 */
2491 if (promisc)
2492 aq_ret = i40e_aq_set_default_vsi(hw,
2493 vsi->seid,
2494 NULL);
2495 else
2496 aq_ret = i40e_aq_clear_default_vsi(hw,
2497 vsi->seid,
2498 NULL);
2499 if (aq_ret) {
2500 dev_info(&pf->pdev->dev,
2501 "Set default VSI failed, err %pe, aq_err %s\n",
2502 ERR_PTR(aq_ret),
2503 i40e_aq_str(hw, hw->aq.asq_last_status));
2504 }
2505 } else {
2506 aq_ret = i40e_aq_set_vsi_unicast_promiscuous(
2507 hw,
2508 vsi->seid,
2509 promisc, NULL,
2510 true);
2511 if (aq_ret) {
2512 dev_info(&pf->pdev->dev,
2513 "set unicast promisc failed, err %pe, aq_err %s\n",
2514 ERR_PTR(aq_ret),
2515 i40e_aq_str(hw, hw->aq.asq_last_status));
2516 }
2517 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(
2518 hw,
2519 vsi->seid,
2520 promisc, NULL);
2521 if (aq_ret) {
2522 dev_info(&pf->pdev->dev,
2523 "set multicast promisc failed, err %pe, aq_err %s\n",
2524 ERR_PTR(aq_ret),
2525 i40e_aq_str(hw, hw->aq.asq_last_status));
2526 }
2527 }
2528
2529 if (!aq_ret)
2530 pf->cur_promisc = promisc;
2531
2532 return aq_ret;
2533 }
2534
2535 /**
2536 * i40e_sync_vsi_filters - Update the VSI filter list to the HW
2537 * @vsi: ptr to the VSI
2538 *
2539 * Push any outstanding VSI filter changes through the AdminQ.
2540 *
2541 * Returns 0 or error value
2542 **/
i40e_sync_vsi_filters(struct i40e_vsi * vsi)2543 int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
2544 {
2545 struct hlist_head tmp_add_list, tmp_del_list;
2546 struct i40e_mac_filter *f;
2547 struct i40e_new_mac_filter *new, *add_head = NULL;
2548 struct i40e_hw *hw = &vsi->back->hw;
2549 bool old_overflow, new_overflow;
2550 unsigned int failed_filters = 0;
2551 unsigned int vlan_filters = 0;
2552 char vsi_name[16] = "PF";
2553 int filter_list_len = 0;
2554 u32 changed_flags = 0;
2555 struct hlist_node *h;
2556 struct i40e_pf *pf;
2557 int num_add = 0;
2558 int num_del = 0;
2559 int aq_ret = 0;
2560 int retval = 0;
2561 u16 cmd_flags;
2562 int list_size;
2563 int bkt;
2564
2565 /* empty array typed pointers, kcalloc later */
2566 struct i40e_aqc_add_macvlan_element_data *add_list;
2567 struct i40e_aqc_remove_macvlan_element_data *del_list;
2568
2569 while (test_and_set_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state))
2570 usleep_range(1000, 2000);
2571 pf = vsi->back;
2572
2573 old_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2574
2575 if (vsi->netdev) {
2576 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;
2577 vsi->current_netdev_flags = vsi->netdev->flags;
2578 }
2579
2580 INIT_HLIST_HEAD(&tmp_add_list);
2581 INIT_HLIST_HEAD(&tmp_del_list);
2582
2583 if (vsi->type == I40E_VSI_SRIOV)
2584 snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id);
2585 else if (vsi->type != I40E_VSI_MAIN)
2586 snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid);
2587
2588 if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) {
2589 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED;
2590
2591 spin_lock_bh(&vsi->mac_filter_hash_lock);
2592 /* Create a list of filters to delete. */
2593 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2594 if (f->state == I40E_FILTER_REMOVE) {
2595 /* Move the element into temporary del_list */
2596 hash_del(&f->hlist);
2597 hlist_add_head(&f->hlist, &tmp_del_list);
2598
2599 /* Avoid counting removed filters */
2600 continue;
2601 }
2602 if (f->state == I40E_FILTER_NEW) {
2603 /* Create a temporary i40e_new_mac_filter */
2604 new = kzalloc(sizeof(*new), GFP_ATOMIC);
2605 if (!new)
2606 goto err_no_memory_locked;
2607
2608 /* Store pointer to the real filter */
2609 new->f = f;
2610 new->state = f->state;
2611
2612 /* Add it to the hash list */
2613 hlist_add_head(&new->hlist, &tmp_add_list);
2614 }
2615
2616 /* Count the number of active (current and new) VLAN
2617 * filters we have now. Does not count filters which
2618 * are marked for deletion.
2619 */
2620 if (f->vlan > 0)
2621 vlan_filters++;
2622 }
2623
2624 if (vsi->type != I40E_VSI_SRIOV)
2625 retval = i40e_correct_mac_vlan_filters
2626 (vsi, &tmp_add_list, &tmp_del_list,
2627 vlan_filters);
2628 else if (pf->vf)
2629 retval = i40e_correct_vf_mac_vlan_filters
2630 (vsi, &tmp_add_list, &tmp_del_list,
2631 vlan_filters, pf->vf[vsi->vf_id].trusted);
2632
2633 hlist_for_each_entry(new, &tmp_add_list, hlist)
2634 netdev_hw_addr_refcnt(new->f, vsi->netdev, 1);
2635
2636 if (retval)
2637 goto err_no_memory_locked;
2638
2639 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2640 }
2641
2642 /* Now process 'del_list' outside the lock */
2643 if (!hlist_empty(&tmp_del_list)) {
2644 filter_list_len = hw->aq.asq_buf_size /
2645 sizeof(struct i40e_aqc_remove_macvlan_element_data);
2646 list_size = filter_list_len *
2647 sizeof(struct i40e_aqc_remove_macvlan_element_data);
2648 del_list = kzalloc(list_size, GFP_ATOMIC);
2649 if (!del_list)
2650 goto err_no_memory;
2651
2652 hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) {
2653 cmd_flags = 0;
2654
2655 /* handle broadcast filters by updating the broadcast
2656 * promiscuous flag and release filter list.
2657 */
2658 if (is_broadcast_ether_addr(f->macaddr)) {
2659 i40e_aqc_broadcast_filter(vsi, vsi_name, f);
2660
2661 hlist_del(&f->hlist);
2662 kfree(f);
2663 continue;
2664 }
2665
2666 /* add to delete list */
2667 ether_addr_copy(del_list[num_del].mac_addr, f->macaddr);
2668 if (f->vlan == I40E_VLAN_ANY) {
2669 del_list[num_del].vlan_tag = 0;
2670 cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
2671 } else {
2672 del_list[num_del].vlan_tag =
2673 cpu_to_le16((u16)(f->vlan));
2674 }
2675
2676 cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
2677 del_list[num_del].flags = cmd_flags;
2678 num_del++;
2679
2680 /* flush a full buffer */
2681 if (num_del == filter_list_len) {
2682 i40e_aqc_del_filters(vsi, vsi_name, del_list,
2683 num_del, &retval);
2684 memset(del_list, 0, list_size);
2685 num_del = 0;
2686 }
2687 /* Release memory for MAC filter entries which were
2688 * synced up with HW.
2689 */
2690 hlist_del(&f->hlist);
2691 kfree(f);
2692 }
2693
2694 if (num_del) {
2695 i40e_aqc_del_filters(vsi, vsi_name, del_list,
2696 num_del, &retval);
2697 }
2698
2699 kfree(del_list);
2700 del_list = NULL;
2701 }
2702
2703 if (!hlist_empty(&tmp_add_list)) {
2704 /* Do all the adds now. */
2705 filter_list_len = hw->aq.asq_buf_size /
2706 sizeof(struct i40e_aqc_add_macvlan_element_data);
2707 list_size = filter_list_len *
2708 sizeof(struct i40e_aqc_add_macvlan_element_data);
2709 add_list = kzalloc(list_size, GFP_ATOMIC);
2710 if (!add_list)
2711 goto err_no_memory;
2712
2713 num_add = 0;
2714 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2715 /* handle broadcast filters by updating the broadcast
2716 * promiscuous flag instead of adding a MAC filter.
2717 */
2718 if (is_broadcast_ether_addr(new->f->macaddr)) {
2719 if (i40e_aqc_broadcast_filter(vsi, vsi_name,
2720 new->f))
2721 new->state = I40E_FILTER_FAILED;
2722 else
2723 new->state = I40E_FILTER_ACTIVE;
2724 continue;
2725 }
2726
2727 /* add to add array */
2728 if (num_add == 0)
2729 add_head = new;
2730 cmd_flags = 0;
2731 ether_addr_copy(add_list[num_add].mac_addr,
2732 new->f->macaddr);
2733 if (new->f->vlan == I40E_VLAN_ANY) {
2734 add_list[num_add].vlan_tag = 0;
2735 cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
2736 } else {
2737 add_list[num_add].vlan_tag =
2738 cpu_to_le16((u16)(new->f->vlan));
2739 }
2740 add_list[num_add].queue_number = 0;
2741 /* set invalid match method for later detection */
2742 add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES;
2743 cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
2744 add_list[num_add].flags = cpu_to_le16(cmd_flags);
2745 num_add++;
2746
2747 /* flush a full buffer */
2748 if (num_add == filter_list_len) {
2749 i40e_aqc_add_filters(vsi, vsi_name, add_list,
2750 add_head, num_add);
2751 memset(add_list, 0, list_size);
2752 num_add = 0;
2753 }
2754 }
2755 if (num_add) {
2756 i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head,
2757 num_add);
2758 }
2759 /* Now move all of the filters from the temp add list back to
2760 * the VSI's list.
2761 */
2762 spin_lock_bh(&vsi->mac_filter_hash_lock);
2763 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2764 /* Only update the state if we're still NEW */
2765 if (new->f->state == I40E_FILTER_NEW)
2766 new->f->state = new->state;
2767 hlist_del(&new->hlist);
2768 netdev_hw_addr_refcnt(new->f, vsi->netdev, -1);
2769 kfree(new);
2770 }
2771 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2772 kfree(add_list);
2773 add_list = NULL;
2774 }
2775
2776 /* Determine the number of active and failed filters. */
2777 spin_lock_bh(&vsi->mac_filter_hash_lock);
2778 vsi->active_filters = 0;
2779 hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) {
2780 if (f->state == I40E_FILTER_ACTIVE)
2781 vsi->active_filters++;
2782 else if (f->state == I40E_FILTER_FAILED)
2783 failed_filters++;
2784 }
2785 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2786
2787 /* Check if we are able to exit overflow promiscuous mode. We can
2788 * safely exit if we didn't just enter, we no longer have any failed
2789 * filters, and we have reduced filters below the threshold value.
2790 */
2791 if (old_overflow && !failed_filters &&
2792 vsi->active_filters < vsi->promisc_threshold) {
2793 dev_info(&pf->pdev->dev,
2794 "filter logjam cleared on %s, leaving overflow promiscuous mode\n",
2795 vsi_name);
2796 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2797 vsi->promisc_threshold = 0;
2798 }
2799
2800 /* if the VF is not trusted do not do promisc */
2801 if (vsi->type == I40E_VSI_SRIOV && pf->vf &&
2802 !pf->vf[vsi->vf_id].trusted) {
2803 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2804 goto out;
2805 }
2806
2807 new_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2808
2809 /* If we are entering overflow promiscuous, we need to calculate a new
2810 * threshold for when we are safe to exit
2811 */
2812 if (!old_overflow && new_overflow)
2813 vsi->promisc_threshold = (vsi->active_filters * 3) / 4;
2814
2815 /* check for changes in promiscuous modes */
2816 if (changed_flags & IFF_ALLMULTI) {
2817 bool cur_multipromisc;
2818
2819 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI);
2820 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw,
2821 vsi->seid,
2822 cur_multipromisc,
2823 NULL);
2824 if (aq_ret) {
2825 retval = i40e_aq_rc_to_posix(aq_ret,
2826 hw->aq.asq_last_status);
2827 dev_info(&pf->pdev->dev,
2828 "set multi promisc failed on %s, err %pe aq_err %s\n",
2829 vsi_name,
2830 ERR_PTR(aq_ret),
2831 i40e_aq_str(hw, hw->aq.asq_last_status));
2832 } else {
2833 dev_info(&pf->pdev->dev, "%s allmulti mode.\n",
2834 cur_multipromisc ? "entering" : "leaving");
2835 }
2836 }
2837
2838 if ((changed_flags & IFF_PROMISC) || old_overflow != new_overflow) {
2839 bool cur_promisc;
2840
2841 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) ||
2842 new_overflow);
2843 aq_ret = i40e_set_promiscuous(pf, cur_promisc);
2844 if (aq_ret) {
2845 retval = i40e_aq_rc_to_posix(aq_ret,
2846 hw->aq.asq_last_status);
2847 dev_info(&pf->pdev->dev,
2848 "Setting promiscuous %s failed on %s, err %pe aq_err %s\n",
2849 cur_promisc ? "on" : "off",
2850 vsi_name,
2851 ERR_PTR(aq_ret),
2852 i40e_aq_str(hw, hw->aq.asq_last_status));
2853 }
2854 }
2855 out:
2856 /* if something went wrong then set the changed flag so we try again */
2857 if (retval)
2858 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2859
2860 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);
2861 return retval;
2862
2863 err_no_memory:
2864 /* Restore elements on the temporary add and delete lists */
2865 spin_lock_bh(&vsi->mac_filter_hash_lock);
2866 err_no_memory_locked:
2867 i40e_undo_del_filter_entries(vsi, &tmp_del_list);
2868 i40e_undo_add_filter_entries(vsi, &tmp_add_list);
2869 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2870
2871 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2872 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);
2873 return -ENOMEM;
2874 }
2875
2876 /**
2877 * i40e_sync_filters_subtask - Sync the VSI filter list with HW
2878 * @pf: board private structure
2879 **/
i40e_sync_filters_subtask(struct i40e_pf * pf)2880 static void i40e_sync_filters_subtask(struct i40e_pf *pf)
2881 {
2882 struct i40e_vsi *vsi;
2883 int v;
2884
2885 if (!pf)
2886 return;
2887 if (!test_and_clear_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state))
2888 return;
2889 if (test_bit(__I40E_VF_DISABLE, pf->state)) {
2890 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state);
2891 return;
2892 }
2893
2894 i40e_pf_for_each_vsi(pf, v, vsi) {
2895 if ((vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) &&
2896 !test_bit(__I40E_VSI_RELEASING, vsi->state)) {
2897 int ret = i40e_sync_vsi_filters(vsi);
2898
2899 if (ret) {
2900 /* come back and try again later */
2901 set_bit(__I40E_MACVLAN_SYNC_PENDING,
2902 pf->state);
2903 break;
2904 }
2905 }
2906 }
2907 }
2908
2909 /**
2910 * i40e_calculate_vsi_rx_buf_len - Calculates buffer length
2911 *
2912 * @vsi: VSI to calculate rx_buf_len from
2913 */
i40e_calculate_vsi_rx_buf_len(struct i40e_vsi * vsi)2914 static u16 i40e_calculate_vsi_rx_buf_len(struct i40e_vsi *vsi)
2915 {
2916 if (!vsi->netdev || test_bit(I40E_FLAG_LEGACY_RX_ENA, vsi->back->flags))
2917 return SKB_WITH_OVERHEAD(I40E_RXBUFFER_2048);
2918
2919 return PAGE_SIZE < 8192 ? I40E_RXBUFFER_3072 : I40E_RXBUFFER_2048;
2920 }
2921
2922 /**
2923 * i40e_max_vsi_frame_size - returns the maximum allowed frame size for VSI
2924 * @vsi: the vsi
2925 * @xdp_prog: XDP program
2926 **/
i40e_max_vsi_frame_size(struct i40e_vsi * vsi,struct bpf_prog * xdp_prog)2927 static int i40e_max_vsi_frame_size(struct i40e_vsi *vsi,
2928 struct bpf_prog *xdp_prog)
2929 {
2930 u16 rx_buf_len = i40e_calculate_vsi_rx_buf_len(vsi);
2931 u16 chain_len;
2932
2933 if (xdp_prog && !xdp_prog->aux->xdp_has_frags)
2934 chain_len = 1;
2935 else
2936 chain_len = I40E_MAX_CHAINED_RX_BUFFERS;
2937
2938 return min_t(u16, rx_buf_len * chain_len, I40E_MAX_RXBUFFER);
2939 }
2940
2941 /**
2942 * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit
2943 * @netdev: network interface device structure
2944 * @new_mtu: new value for maximum frame size
2945 *
2946 * Returns 0 on success, negative on failure
2947 **/
i40e_change_mtu(struct net_device * netdev,int new_mtu)2948 static int i40e_change_mtu(struct net_device *netdev, int new_mtu)
2949 {
2950 struct i40e_netdev_priv *np = netdev_priv(netdev);
2951 struct i40e_vsi *vsi = np->vsi;
2952 struct i40e_pf *pf = vsi->back;
2953 int frame_size;
2954
2955 frame_size = i40e_max_vsi_frame_size(vsi, vsi->xdp_prog);
2956 if (new_mtu > frame_size - I40E_PACKET_HDR_PAD) {
2957 netdev_err(netdev, "Error changing mtu to %d, Max is %d\n",
2958 new_mtu, frame_size - I40E_PACKET_HDR_PAD);
2959 return -EINVAL;
2960 }
2961
2962 netdev_dbg(netdev, "changing MTU from %d to %d\n",
2963 netdev->mtu, new_mtu);
2964 WRITE_ONCE(netdev->mtu, new_mtu);
2965 if (netif_running(netdev))
2966 i40e_vsi_reinit_locked(vsi);
2967 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
2968 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);
2969 return 0;
2970 }
2971
2972 /**
2973 * i40e_ioctl - Access the hwtstamp interface
2974 * @netdev: network interface device structure
2975 * @ifr: interface request data
2976 * @cmd: ioctl command
2977 **/
i40e_ioctl(struct net_device * netdev,struct ifreq * ifr,int cmd)2978 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2979 {
2980 struct i40e_netdev_priv *np = netdev_priv(netdev);
2981 struct i40e_pf *pf = np->vsi->back;
2982
2983 switch (cmd) {
2984 case SIOCGHWTSTAMP:
2985 return i40e_ptp_get_ts_config(pf, ifr);
2986 case SIOCSHWTSTAMP:
2987 return i40e_ptp_set_ts_config(pf, ifr);
2988 default:
2989 return -EOPNOTSUPP;
2990 }
2991 }
2992
2993 /**
2994 * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI
2995 * @vsi: the vsi being adjusted
2996 **/
i40e_vlan_stripping_enable(struct i40e_vsi * vsi)2997 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi)
2998 {
2999 struct i40e_vsi_context ctxt;
3000 int ret;
3001
3002 /* Don't modify stripping options if a port VLAN is active */
3003 if (vsi->info.pvid)
3004 return;
3005
3006 if ((vsi->info.valid_sections &
3007 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
3008 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0))
3009 return; /* already enabled */
3010
3011 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
3012 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
3013 I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
3014
3015 ctxt.seid = vsi->seid;
3016 ctxt.info = vsi->info;
3017 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
3018 if (ret) {
3019 dev_info(&vsi->back->pdev->dev,
3020 "update vlan stripping failed, err %pe aq_err %s\n",
3021 ERR_PTR(ret),
3022 i40e_aq_str(&vsi->back->hw,
3023 vsi->back->hw.aq.asq_last_status));
3024 }
3025 }
3026
3027 /**
3028 * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI
3029 * @vsi: the vsi being adjusted
3030 **/
i40e_vlan_stripping_disable(struct i40e_vsi * vsi)3031 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi)
3032 {
3033 struct i40e_vsi_context ctxt;
3034 int ret;
3035
3036 /* Don't modify stripping options if a port VLAN is active */
3037 if (vsi->info.pvid)
3038 return;
3039
3040 if ((vsi->info.valid_sections &
3041 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
3042 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) ==
3043 I40E_AQ_VSI_PVLAN_EMOD_MASK))
3044 return; /* already disabled */
3045
3046 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
3047 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
3048 I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
3049
3050 ctxt.seid = vsi->seid;
3051 ctxt.info = vsi->info;
3052 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
3053 if (ret) {
3054 dev_info(&vsi->back->pdev->dev,
3055 "update vlan stripping failed, err %pe aq_err %s\n",
3056 ERR_PTR(ret),
3057 i40e_aq_str(&vsi->back->hw,
3058 vsi->back->hw.aq.asq_last_status));
3059 }
3060 }
3061
3062 /**
3063 * i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address
3064 * @vsi: the vsi being configured
3065 * @vid: vlan id to be added (0 = untagged only , -1 = any)
3066 *
3067 * This is a helper function for adding a new MAC/VLAN filter with the
3068 * specified VLAN for each existing MAC address already in the hash table.
3069 * This function does *not* perform any accounting to update filters based on
3070 * VLAN mode.
3071 *
3072 * NOTE: this function expects to be called while under the
3073 * mac_filter_hash_lock
3074 **/
i40e_add_vlan_all_mac(struct i40e_vsi * vsi,s16 vid)3075 int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
3076 {
3077 struct i40e_mac_filter *f, *add_f;
3078 struct hlist_node *h;
3079 int bkt;
3080
3081 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
3082 /* If we're asked to add a filter that has been marked for
3083 * removal, it is safe to simply restore it to active state.
3084 * __i40e_del_filter will have simply deleted any filters which
3085 * were previously marked NEW or FAILED, so if it is currently
3086 * marked REMOVE it must have previously been ACTIVE. Since we
3087 * haven't yet run the sync filters task, just restore this
3088 * filter to the ACTIVE state so that the sync task leaves it
3089 * in place.
3090 */
3091 if (f->state == I40E_FILTER_REMOVE && f->vlan == vid) {
3092 f->state = I40E_FILTER_ACTIVE;
3093 continue;
3094 } else if (f->state == I40E_FILTER_REMOVE) {
3095 continue;
3096 }
3097 add_f = i40e_add_filter(vsi, f->macaddr, vid);
3098 if (!add_f) {
3099 dev_info(&vsi->back->pdev->dev,
3100 "Could not add vlan filter %d for %pM\n",
3101 vid, f->macaddr);
3102 return -ENOMEM;
3103 }
3104 }
3105
3106 return 0;
3107 }
3108
3109 /**
3110 * i40e_vsi_add_vlan - Add VSI membership for given VLAN
3111 * @vsi: the VSI being configured
3112 * @vid: VLAN id to be added
3113 **/
i40e_vsi_add_vlan(struct i40e_vsi * vsi,u16 vid)3114 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid)
3115 {
3116 int err;
3117
3118 if (vsi->info.pvid)
3119 return -EINVAL;
3120
3121 /* The network stack will attempt to add VID=0, with the intention to
3122 * receive priority tagged packets with a VLAN of 0. Our HW receives
3123 * these packets by default when configured to receive untagged
3124 * packets, so we don't need to add a filter for this case.
3125 * Additionally, HW interprets adding a VID=0 filter as meaning to
3126 * receive *only* tagged traffic and stops receiving untagged traffic.
3127 * Thus, we do not want to actually add a filter for VID=0
3128 */
3129 if (!vid)
3130 return 0;
3131
3132 /* Locked once because all functions invoked below iterates list*/
3133 spin_lock_bh(&vsi->mac_filter_hash_lock);
3134 err = i40e_add_vlan_all_mac(vsi, vid);
3135 spin_unlock_bh(&vsi->mac_filter_hash_lock);
3136 if (err)
3137 return err;
3138
3139 /* schedule our worker thread which will take care of
3140 * applying the new filter changes
3141 */
3142 i40e_service_event_schedule(vsi->back);
3143 return 0;
3144 }
3145
3146 /**
3147 * i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN
3148 * @vsi: the vsi being configured
3149 * @vid: vlan id to be removed (0 = untagged only , -1 = any)
3150 *
3151 * This function should be used to remove all VLAN filters which match the
3152 * given VID. It does not schedule the service event and does not take the
3153 * mac_filter_hash_lock so it may be combined with other operations under
3154 * a single invocation of the mac_filter_hash_lock.
3155 *
3156 * NOTE: this function expects to be called while under the
3157 * mac_filter_hash_lock
3158 */
i40e_rm_vlan_all_mac(struct i40e_vsi * vsi,s16 vid)3159 void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
3160 {
3161 struct i40e_mac_filter *f;
3162 struct hlist_node *h;
3163 int bkt;
3164
3165 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
3166 if (f->vlan == vid)
3167 __i40e_del_filter(vsi, f);
3168 }
3169 }
3170
3171 /**
3172 * i40e_vsi_kill_vlan - Remove VSI membership for given VLAN
3173 * @vsi: the VSI being configured
3174 * @vid: VLAN id to be removed
3175 **/
i40e_vsi_kill_vlan(struct i40e_vsi * vsi,u16 vid)3176 void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid)
3177 {
3178 if (!vid || vsi->info.pvid)
3179 return;
3180
3181 spin_lock_bh(&vsi->mac_filter_hash_lock);
3182 i40e_rm_vlan_all_mac(vsi, vid);
3183 spin_unlock_bh(&vsi->mac_filter_hash_lock);
3184
3185 /* schedule our worker thread which will take care of
3186 * applying the new filter changes
3187 */
3188 i40e_service_event_schedule(vsi->back);
3189 }
3190
3191 /**
3192 * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload
3193 * @netdev: network interface to be adjusted
3194 * @proto: unused protocol value
3195 * @vid: vlan id to be added
3196 *
3197 * net_device_ops implementation for adding vlan ids
3198 **/
i40e_vlan_rx_add_vid(struct net_device * netdev,__always_unused __be16 proto,u16 vid)3199 static int i40e_vlan_rx_add_vid(struct net_device *netdev,
3200 __always_unused __be16 proto, u16 vid)
3201 {
3202 struct i40e_netdev_priv *np = netdev_priv(netdev);
3203 struct i40e_vsi *vsi = np->vsi;
3204 int ret = 0;
3205
3206 if (vid >= VLAN_N_VID)
3207 return -EINVAL;
3208
3209 ret = i40e_vsi_add_vlan(vsi, vid);
3210 if (!ret)
3211 set_bit(vid, vsi->active_vlans);
3212
3213 return ret;
3214 }
3215
3216 /**
3217 * i40e_vlan_rx_add_vid_up - Add a vlan id filter to HW offload in UP path
3218 * @netdev: network interface to be adjusted
3219 * @proto: unused protocol value
3220 * @vid: vlan id to be added
3221 **/
i40e_vlan_rx_add_vid_up(struct net_device * netdev,__always_unused __be16 proto,u16 vid)3222 static void i40e_vlan_rx_add_vid_up(struct net_device *netdev,
3223 __always_unused __be16 proto, u16 vid)
3224 {
3225 struct i40e_netdev_priv *np = netdev_priv(netdev);
3226 struct i40e_vsi *vsi = np->vsi;
3227
3228 if (vid >= VLAN_N_VID)
3229 return;
3230 set_bit(vid, vsi->active_vlans);
3231 }
3232
3233 /**
3234 * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload
3235 * @netdev: network interface to be adjusted
3236 * @proto: unused protocol value
3237 * @vid: vlan id to be removed
3238 *
3239 * net_device_ops implementation for removing vlan ids
3240 **/
i40e_vlan_rx_kill_vid(struct net_device * netdev,__always_unused __be16 proto,u16 vid)3241 static int i40e_vlan_rx_kill_vid(struct net_device *netdev,
3242 __always_unused __be16 proto, u16 vid)
3243 {
3244 struct i40e_netdev_priv *np = netdev_priv(netdev);
3245 struct i40e_vsi *vsi = np->vsi;
3246
3247 /* return code is ignored as there is nothing a user
3248 * can do about failure to remove and a log message was
3249 * already printed from the other function
3250 */
3251 i40e_vsi_kill_vlan(vsi, vid);
3252
3253 clear_bit(vid, vsi->active_vlans);
3254
3255 return 0;
3256 }
3257
3258 /**
3259 * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up
3260 * @vsi: the vsi being brought back up
3261 **/
i40e_restore_vlan(struct i40e_vsi * vsi)3262 static void i40e_restore_vlan(struct i40e_vsi *vsi)
3263 {
3264 u16 vid;
3265
3266 if (!vsi->netdev)
3267 return;
3268
3269 if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
3270 i40e_vlan_stripping_enable(vsi);
3271 else
3272 i40e_vlan_stripping_disable(vsi);
3273
3274 for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID)
3275 i40e_vlan_rx_add_vid_up(vsi->netdev, htons(ETH_P_8021Q),
3276 vid);
3277 }
3278
3279 /**
3280 * i40e_vsi_add_pvid - Add pvid for the VSI
3281 * @vsi: the vsi being adjusted
3282 * @vid: the vlan id to set as a PVID
3283 **/
i40e_vsi_add_pvid(struct i40e_vsi * vsi,u16 vid)3284 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid)
3285 {
3286 struct i40e_vsi_context ctxt;
3287 int ret;
3288
3289 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
3290 vsi->info.pvid = cpu_to_le16(vid);
3291 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED |
3292 I40E_AQ_VSI_PVLAN_INSERT_PVID |
3293 I40E_AQ_VSI_PVLAN_EMOD_STR;
3294
3295 ctxt.seid = vsi->seid;
3296 ctxt.info = vsi->info;
3297 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
3298 if (ret) {
3299 dev_info(&vsi->back->pdev->dev,
3300 "add pvid failed, err %pe aq_err %s\n",
3301 ERR_PTR(ret),
3302 i40e_aq_str(&vsi->back->hw,
3303 vsi->back->hw.aq.asq_last_status));
3304 return -ENOENT;
3305 }
3306
3307 return 0;
3308 }
3309
3310 /**
3311 * i40e_vsi_remove_pvid - Remove the pvid from the VSI
3312 * @vsi: the vsi being adjusted
3313 *
3314 * Just use the vlan_rx_register() service to put it back to normal
3315 **/
i40e_vsi_remove_pvid(struct i40e_vsi * vsi)3316 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi)
3317 {
3318 vsi->info.pvid = 0;
3319
3320 i40e_vlan_stripping_disable(vsi);
3321 }
3322
3323 /**
3324 * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources
3325 * @vsi: ptr to the VSI
3326 *
3327 * If this function returns with an error, then it's possible one or
3328 * more of the rings is populated (while the rest are not). It is the
3329 * callers duty to clean those orphaned rings.
3330 *
3331 * Return 0 on success, negative on failure
3332 **/
i40e_vsi_setup_tx_resources(struct i40e_vsi * vsi)3333 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi)
3334 {
3335 int i, err = 0;
3336
3337 for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3338 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]);
3339
3340 if (!i40e_enabled_xdp_vsi(vsi))
3341 return err;
3342
3343 for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3344 err = i40e_setup_tx_descriptors(vsi->xdp_rings[i]);
3345
3346 return err;
3347 }
3348
3349 /**
3350 * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues
3351 * @vsi: ptr to the VSI
3352 *
3353 * Free VSI's transmit software resources
3354 **/
i40e_vsi_free_tx_resources(struct i40e_vsi * vsi)3355 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi)
3356 {
3357 int i;
3358
3359 if (vsi->tx_rings) {
3360 for (i = 0; i < vsi->num_queue_pairs; i++)
3361 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc)
3362 i40e_free_tx_resources(vsi->tx_rings[i]);
3363 }
3364
3365 if (vsi->xdp_rings) {
3366 for (i = 0; i < vsi->num_queue_pairs; i++)
3367 if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc)
3368 i40e_free_tx_resources(vsi->xdp_rings[i]);
3369 }
3370 }
3371
3372 /**
3373 * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources
3374 * @vsi: ptr to the VSI
3375 *
3376 * If this function returns with an error, then it's possible one or
3377 * more of the rings is populated (while the rest are not). It is the
3378 * callers duty to clean those orphaned rings.
3379 *
3380 * Return 0 on success, negative on failure
3381 **/
i40e_vsi_setup_rx_resources(struct i40e_vsi * vsi)3382 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi)
3383 {
3384 int i, err = 0;
3385
3386 for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3387 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]);
3388 return err;
3389 }
3390
3391 /**
3392 * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues
3393 * @vsi: ptr to the VSI
3394 *
3395 * Free all receive software resources
3396 **/
i40e_vsi_free_rx_resources(struct i40e_vsi * vsi)3397 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi)
3398 {
3399 int i;
3400
3401 if (!vsi->rx_rings)
3402 return;
3403
3404 for (i = 0; i < vsi->num_queue_pairs; i++)
3405 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc)
3406 i40e_free_rx_resources(vsi->rx_rings[i]);
3407 }
3408
3409 /**
3410 * i40e_config_xps_tx_ring - Configure XPS for a Tx ring
3411 * @ring: The Tx ring to configure
3412 *
3413 * This enables/disables XPS for a given Tx descriptor ring
3414 * based on the TCs enabled for the VSI that ring belongs to.
3415 **/
i40e_config_xps_tx_ring(struct i40e_ring * ring)3416 static void i40e_config_xps_tx_ring(struct i40e_ring *ring)
3417 {
3418 int cpu;
3419
3420 if (!ring->q_vector || !ring->netdev || ring->ch)
3421 return;
3422
3423 /* We only initialize XPS once, so as not to overwrite user settings */
3424 if (test_and_set_bit(__I40E_TX_XPS_INIT_DONE, ring->state))
3425 return;
3426
3427 cpu = cpumask_local_spread(ring->q_vector->v_idx, -1);
3428 netif_set_xps_queue(ring->netdev, get_cpu_mask(cpu),
3429 ring->queue_index);
3430 }
3431
3432 /**
3433 * i40e_xsk_pool - Retrieve the AF_XDP buffer pool if XDP and ZC is enabled
3434 * @ring: The Tx or Rx ring
3435 *
3436 * Returns the AF_XDP buffer pool or NULL.
3437 **/
i40e_xsk_pool(struct i40e_ring * ring)3438 static struct xsk_buff_pool *i40e_xsk_pool(struct i40e_ring *ring)
3439 {
3440 bool xdp_on = i40e_enabled_xdp_vsi(ring->vsi);
3441 int qid = ring->queue_index;
3442
3443 if (ring_is_xdp(ring))
3444 qid -= ring->vsi->alloc_queue_pairs;
3445
3446 if (!xdp_on || !test_bit(qid, ring->vsi->af_xdp_zc_qps))
3447 return NULL;
3448
3449 return xsk_get_pool_from_qid(ring->vsi->netdev, qid);
3450 }
3451
3452 /**
3453 * i40e_configure_tx_ring - Configure a transmit ring context and rest
3454 * @ring: The Tx ring to configure
3455 *
3456 * Configure the Tx descriptor ring in the HMC context.
3457 **/
i40e_configure_tx_ring(struct i40e_ring * ring)3458 static int i40e_configure_tx_ring(struct i40e_ring *ring)
3459 {
3460 struct i40e_vsi *vsi = ring->vsi;
3461 u16 pf_q = vsi->base_queue + ring->queue_index;
3462 struct i40e_hw *hw = &vsi->back->hw;
3463 struct i40e_hmc_obj_txq tx_ctx;
3464 u32 qtx_ctl = 0;
3465 int err = 0;
3466
3467 if (ring_is_xdp(ring))
3468 ring->xsk_pool = i40e_xsk_pool(ring);
3469
3470 /* some ATR related tx ring init */
3471 if (test_bit(I40E_FLAG_FD_ATR_ENA, vsi->back->flags)) {
3472 ring->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE;
3473 ring->atr_count = 0;
3474 } else {
3475 ring->atr_sample_rate = 0;
3476 }
3477
3478 /* configure XPS */
3479 i40e_config_xps_tx_ring(ring);
3480
3481 /* clear the context structure first */
3482 memset(&tx_ctx, 0, sizeof(tx_ctx));
3483
3484 tx_ctx.new_context = 1;
3485 tx_ctx.base = (ring->dma / 128);
3486 tx_ctx.qlen = ring->count;
3487 if (test_bit(I40E_FLAG_FD_SB_ENA, vsi->back->flags) ||
3488 test_bit(I40E_FLAG_FD_ATR_ENA, vsi->back->flags))
3489 tx_ctx.fd_ena = 1;
3490 if (test_bit(I40E_FLAG_PTP_ENA, vsi->back->flags))
3491 tx_ctx.timesync_ena = 1;
3492 /* FDIR VSI tx ring can still use RS bit and writebacks */
3493 if (vsi->type != I40E_VSI_FDIR)
3494 tx_ctx.head_wb_ena = 1;
3495 tx_ctx.head_wb_addr = ring->dma +
3496 (ring->count * sizeof(struct i40e_tx_desc));
3497
3498 /* As part of VSI creation/update, FW allocates certain
3499 * Tx arbitration queue sets for each TC enabled for
3500 * the VSI. The FW returns the handles to these queue
3501 * sets as part of the response buffer to Add VSI,
3502 * Update VSI, etc. AQ commands. It is expected that
3503 * these queue set handles be associated with the Tx
3504 * queues by the driver as part of the TX queue context
3505 * initialization. This has to be done regardless of
3506 * DCB as by default everything is mapped to TC0.
3507 */
3508
3509 if (ring->ch)
3510 tx_ctx.rdylist =
3511 le16_to_cpu(ring->ch->info.qs_handle[ring->dcb_tc]);
3512
3513 else
3514 tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]);
3515
3516 tx_ctx.rdylist_act = 0;
3517
3518 /* clear the context in the HMC */
3519 err = i40e_clear_lan_tx_queue_context(hw, pf_q);
3520 if (err) {
3521 dev_info(&vsi->back->pdev->dev,
3522 "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n",
3523 ring->queue_index, pf_q, err);
3524 return -ENOMEM;
3525 }
3526
3527 /* set the context in the HMC */
3528 err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx);
3529 if (err) {
3530 dev_info(&vsi->back->pdev->dev,
3531 "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n",
3532 ring->queue_index, pf_q, err);
3533 return -ENOMEM;
3534 }
3535
3536 /* Now associate this queue with this PCI function */
3537 if (ring->ch) {
3538 if (ring->ch->type == I40E_VSI_VMDQ2)
3539 qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
3540 else
3541 return -EINVAL;
3542
3543 qtx_ctl |= FIELD_PREP(I40E_QTX_CTL_VFVM_INDX_MASK,
3544 ring->ch->vsi_number);
3545 } else {
3546 if (vsi->type == I40E_VSI_VMDQ2) {
3547 qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
3548 qtx_ctl |= FIELD_PREP(I40E_QTX_CTL_VFVM_INDX_MASK,
3549 vsi->id);
3550 } else {
3551 qtx_ctl = I40E_QTX_CTL_PF_QUEUE;
3552 }
3553 }
3554
3555 qtx_ctl |= FIELD_PREP(I40E_QTX_CTL_PF_INDX_MASK, hw->pf_id);
3556 wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl);
3557 i40e_flush(hw);
3558
3559 /* cache tail off for easier writes later */
3560 ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q);
3561
3562 return 0;
3563 }
3564
3565 /**
3566 * i40e_rx_offset - Return expected offset into page to access data
3567 * @rx_ring: Ring we are requesting offset of
3568 *
3569 * Returns the offset value for ring into the data buffer.
3570 */
i40e_rx_offset(struct i40e_ring * rx_ring)3571 static unsigned int i40e_rx_offset(struct i40e_ring *rx_ring)
3572 {
3573 return ring_uses_build_skb(rx_ring) ? I40E_SKB_PAD : 0;
3574 }
3575
3576 /**
3577 * i40e_configure_rx_ring - Configure a receive ring context
3578 * @ring: The Rx ring to configure
3579 *
3580 * Configure the Rx descriptor ring in the HMC context.
3581 **/
i40e_configure_rx_ring(struct i40e_ring * ring)3582 static int i40e_configure_rx_ring(struct i40e_ring *ring)
3583 {
3584 struct i40e_vsi *vsi = ring->vsi;
3585 u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len;
3586 u16 pf_q = vsi->base_queue + ring->queue_index;
3587 struct i40e_hw *hw = &vsi->back->hw;
3588 struct i40e_hmc_obj_rxq rx_ctx;
3589 int err = 0;
3590 bool ok;
3591
3592 bitmap_zero(ring->state, __I40E_RING_STATE_NBITS);
3593
3594 /* clear the context structure first */
3595 memset(&rx_ctx, 0, sizeof(rx_ctx));
3596
3597 ring->rx_buf_len = vsi->rx_buf_len;
3598
3599 /* XDP RX-queue info only needed for RX rings exposed to XDP */
3600 if (ring->vsi->type != I40E_VSI_MAIN)
3601 goto skip;
3602
3603 if (!xdp_rxq_info_is_reg(&ring->xdp_rxq)) {
3604 err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev,
3605 ring->queue_index,
3606 ring->q_vector->napi.napi_id,
3607 ring->rx_buf_len);
3608 if (err)
3609 return err;
3610 }
3611
3612 ring->xsk_pool = i40e_xsk_pool(ring);
3613 if (ring->xsk_pool) {
3614 xdp_rxq_info_unreg(&ring->xdp_rxq);
3615 ring->rx_buf_len = xsk_pool_get_rx_frame_size(ring->xsk_pool);
3616 err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev,
3617 ring->queue_index,
3618 ring->q_vector->napi.napi_id,
3619 ring->rx_buf_len);
3620 if (err)
3621 return err;
3622 err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
3623 MEM_TYPE_XSK_BUFF_POOL,
3624 NULL);
3625 if (err)
3626 return err;
3627 dev_info(&vsi->back->pdev->dev,
3628 "Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n",
3629 ring->queue_index);
3630
3631 } else {
3632 err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
3633 MEM_TYPE_PAGE_SHARED,
3634 NULL);
3635 if (err)
3636 return err;
3637 }
3638
3639 skip:
3640 xdp_init_buff(&ring->xdp, i40e_rx_pg_size(ring) / 2, &ring->xdp_rxq);
3641
3642 rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len,
3643 BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT));
3644
3645 rx_ctx.base = (ring->dma / 128);
3646 rx_ctx.qlen = ring->count;
3647
3648 /* use 16 byte descriptors */
3649 rx_ctx.dsize = 0;
3650
3651 /* descriptor type is always zero
3652 * rx_ctx.dtype = 0;
3653 */
3654 rx_ctx.hsplit_0 = 0;
3655
3656 rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len);
3657 if (hw->revision_id == 0)
3658 rx_ctx.lrxqthresh = 0;
3659 else
3660 rx_ctx.lrxqthresh = 1;
3661 rx_ctx.crcstrip = 1;
3662 rx_ctx.l2tsel = 1;
3663 /* this controls whether VLAN is stripped from inner headers */
3664 rx_ctx.showiv = 0;
3665 /* set the prefena field to 1 because the manual says to */
3666 rx_ctx.prefena = 1;
3667
3668 /* clear the context in the HMC */
3669 err = i40e_clear_lan_rx_queue_context(hw, pf_q);
3670 if (err) {
3671 dev_info(&vsi->back->pdev->dev,
3672 "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3673 ring->queue_index, pf_q, err);
3674 return -ENOMEM;
3675 }
3676
3677 /* set the context in the HMC */
3678 err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx);
3679 if (err) {
3680 dev_info(&vsi->back->pdev->dev,
3681 "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3682 ring->queue_index, pf_q, err);
3683 return -ENOMEM;
3684 }
3685
3686 /* configure Rx buffer alignment */
3687 if (!vsi->netdev || test_bit(I40E_FLAG_LEGACY_RX_ENA, vsi->back->flags)) {
3688 if (I40E_2K_TOO_SMALL_WITH_PADDING) {
3689 dev_info(&vsi->back->pdev->dev,
3690 "2k Rx buffer is too small to fit standard MTU and skb_shared_info\n");
3691 return -EOPNOTSUPP;
3692 }
3693 clear_ring_build_skb_enabled(ring);
3694 } else {
3695 set_ring_build_skb_enabled(ring);
3696 }
3697
3698 ring->rx_offset = i40e_rx_offset(ring);
3699
3700 /* cache tail for quicker writes, and clear the reg before use */
3701 ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q);
3702 writel(0, ring->tail);
3703
3704 if (ring->xsk_pool) {
3705 xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq);
3706 ok = i40e_alloc_rx_buffers_zc(ring, I40E_DESC_UNUSED(ring));
3707 } else {
3708 ok = !i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring));
3709 }
3710 if (!ok) {
3711 /* Log this in case the user has forgotten to give the kernel
3712 * any buffers, even later in the application.
3713 */
3714 dev_info(&vsi->back->pdev->dev,
3715 "Failed to allocate some buffers on %sRx ring %d (pf_q %d)\n",
3716 ring->xsk_pool ? "AF_XDP ZC enabled " : "",
3717 ring->queue_index, pf_q);
3718 }
3719
3720 return 0;
3721 }
3722
3723 /**
3724 * i40e_vsi_configure_tx - Configure the VSI for Tx
3725 * @vsi: VSI structure describing this set of rings and resources
3726 *
3727 * Configure the Tx VSI for operation.
3728 **/
i40e_vsi_configure_tx(struct i40e_vsi * vsi)3729 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi)
3730 {
3731 int err = 0;
3732 u16 i;
3733
3734 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3735 err = i40e_configure_tx_ring(vsi->tx_rings[i]);
3736
3737 if (err || !i40e_enabled_xdp_vsi(vsi))
3738 return err;
3739
3740 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3741 err = i40e_configure_tx_ring(vsi->xdp_rings[i]);
3742
3743 return err;
3744 }
3745
3746 /**
3747 * i40e_vsi_configure_rx - Configure the VSI for Rx
3748 * @vsi: the VSI being configured
3749 *
3750 * Configure the Rx VSI for operation.
3751 **/
i40e_vsi_configure_rx(struct i40e_vsi * vsi)3752 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi)
3753 {
3754 int err = 0;
3755 u16 i;
3756
3757 vsi->max_frame = i40e_max_vsi_frame_size(vsi, vsi->xdp_prog);
3758 vsi->rx_buf_len = i40e_calculate_vsi_rx_buf_len(vsi);
3759
3760 #if (PAGE_SIZE < 8192)
3761 if (vsi->netdev && !I40E_2K_TOO_SMALL_WITH_PADDING &&
3762 vsi->netdev->mtu <= ETH_DATA_LEN) {
3763 vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
3764 vsi->max_frame = vsi->rx_buf_len;
3765 }
3766 #endif
3767
3768 /* set up individual rings */
3769 for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3770 err = i40e_configure_rx_ring(vsi->rx_rings[i]);
3771
3772 return err;
3773 }
3774
3775 /**
3776 * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC
3777 * @vsi: ptr to the VSI
3778 **/
i40e_vsi_config_dcb_rings(struct i40e_vsi * vsi)3779 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi)
3780 {
3781 struct i40e_ring *tx_ring, *rx_ring;
3782 u16 qoffset, qcount;
3783 int i, n;
3784
3785 if (!test_bit(I40E_FLAG_DCB_ENA, vsi->back->flags)) {
3786 /* Reset the TC information */
3787 for (i = 0; i < vsi->num_queue_pairs; i++) {
3788 rx_ring = vsi->rx_rings[i];
3789 tx_ring = vsi->tx_rings[i];
3790 rx_ring->dcb_tc = 0;
3791 tx_ring->dcb_tc = 0;
3792 }
3793 return;
3794 }
3795
3796 for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) {
3797 if (!(vsi->tc_config.enabled_tc & BIT_ULL(n)))
3798 continue;
3799
3800 qoffset = vsi->tc_config.tc_info[n].qoffset;
3801 qcount = vsi->tc_config.tc_info[n].qcount;
3802 for (i = qoffset; i < (qoffset + qcount); i++) {
3803 rx_ring = vsi->rx_rings[i];
3804 tx_ring = vsi->tx_rings[i];
3805 rx_ring->dcb_tc = n;
3806 tx_ring->dcb_tc = n;
3807 }
3808 }
3809 }
3810
3811 /**
3812 * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI
3813 * @vsi: ptr to the VSI
3814 **/
i40e_set_vsi_rx_mode(struct i40e_vsi * vsi)3815 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi)
3816 {
3817 if (vsi->netdev)
3818 i40e_set_rx_mode(vsi->netdev);
3819 }
3820
3821 /**
3822 * i40e_reset_fdir_filter_cnt - Reset flow director filter counters
3823 * @pf: Pointer to the targeted PF
3824 *
3825 * Set all flow director counters to 0.
3826 */
i40e_reset_fdir_filter_cnt(struct i40e_pf * pf)3827 static void i40e_reset_fdir_filter_cnt(struct i40e_pf *pf)
3828 {
3829 pf->fd_tcp4_filter_cnt = 0;
3830 pf->fd_udp4_filter_cnt = 0;
3831 pf->fd_sctp4_filter_cnt = 0;
3832 pf->fd_ip4_filter_cnt = 0;
3833 pf->fd_tcp6_filter_cnt = 0;
3834 pf->fd_udp6_filter_cnt = 0;
3835 pf->fd_sctp6_filter_cnt = 0;
3836 pf->fd_ip6_filter_cnt = 0;
3837 }
3838
3839 /**
3840 * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters
3841 * @vsi: Pointer to the targeted VSI
3842 *
3843 * This function replays the hlist on the hw where all the SB Flow Director
3844 * filters were saved.
3845 **/
i40e_fdir_filter_restore(struct i40e_vsi * vsi)3846 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi)
3847 {
3848 struct i40e_fdir_filter *filter;
3849 struct i40e_pf *pf = vsi->back;
3850 struct hlist_node *node;
3851
3852 if (!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags))
3853 return;
3854
3855 /* Reset FDir counters as we're replaying all existing filters */
3856 i40e_reset_fdir_filter_cnt(pf);
3857
3858 hlist_for_each_entry_safe(filter, node,
3859 &pf->fdir_filter_list, fdir_node) {
3860 i40e_add_del_fdir(vsi, filter, true);
3861 }
3862 }
3863
3864 /**
3865 * i40e_vsi_configure - Set up the VSI for action
3866 * @vsi: the VSI being configured
3867 **/
i40e_vsi_configure(struct i40e_vsi * vsi)3868 static int i40e_vsi_configure(struct i40e_vsi *vsi)
3869 {
3870 int err;
3871
3872 i40e_set_vsi_rx_mode(vsi);
3873 i40e_restore_vlan(vsi);
3874 i40e_vsi_config_dcb_rings(vsi);
3875 err = i40e_vsi_configure_tx(vsi);
3876 if (!err)
3877 err = i40e_vsi_configure_rx(vsi);
3878
3879 return err;
3880 }
3881
3882 /**
3883 * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW
3884 * @vsi: the VSI being configured
3885 **/
i40e_vsi_configure_msix(struct i40e_vsi * vsi)3886 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi)
3887 {
3888 bool has_xdp = i40e_enabled_xdp_vsi(vsi);
3889 struct i40e_pf *pf = vsi->back;
3890 struct i40e_hw *hw = &pf->hw;
3891 u16 vector;
3892 int i, q;
3893 u32 qp;
3894
3895 /* The interrupt indexing is offset by 1 in the PFINT_ITRn
3896 * and PFINT_LNKLSTn registers, e.g.:
3897 * PFINT_ITRn[0..n-1] gets msix-1..msix-n (qpair interrupts)
3898 */
3899 qp = vsi->base_queue;
3900 vector = vsi->base_vector;
3901 for (i = 0; i < vsi->num_q_vectors; i++, vector++) {
3902 struct i40e_q_vector *q_vector = vsi->q_vectors[i];
3903
3904 q_vector->rx.next_update = jiffies + 1;
3905 q_vector->rx.target_itr =
3906 ITR_TO_REG(vsi->rx_rings[i]->itr_setting);
3907 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1),
3908 q_vector->rx.target_itr >> 1);
3909 q_vector->rx.current_itr = q_vector->rx.target_itr;
3910
3911 q_vector->tx.next_update = jiffies + 1;
3912 q_vector->tx.target_itr =
3913 ITR_TO_REG(vsi->tx_rings[i]->itr_setting);
3914 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1),
3915 q_vector->tx.target_itr >> 1);
3916 q_vector->tx.current_itr = q_vector->tx.target_itr;
3917
3918 /* Set ITR for software interrupts triggered after exiting
3919 * busy-loop polling.
3920 */
3921 wr32(hw, I40E_PFINT_ITRN(I40E_SW_ITR, vector - 1),
3922 I40E_ITR_20K);
3923
3924 wr32(hw, I40E_PFINT_RATEN(vector - 1),
3925 i40e_intrl_usec_to_reg(vsi->int_rate_limit));
3926
3927 /* begin of linked list for RX queue assigned to this vector */
3928 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp);
3929 for (q = 0; q < q_vector->num_ringpairs; q++) {
3930 u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp;
3931 u32 val;
3932
3933 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
3934 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
3935 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
3936 (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
3937 (I40E_QUEUE_TYPE_TX <<
3938 I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
3939
3940 wr32(hw, I40E_QINT_RQCTL(qp), val);
3941
3942 if (has_xdp) {
3943 /* TX queue with next queue set to TX */
3944 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3945 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3946 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3947 (qp << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
3948 (I40E_QUEUE_TYPE_TX <<
3949 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3950
3951 wr32(hw, I40E_QINT_TQCTL(nextqp), val);
3952 }
3953 /* TX queue with next RX or end of linked list */
3954 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3955 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3956 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3957 ((qp + 1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
3958 (I40E_QUEUE_TYPE_RX <<
3959 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3960
3961 /* Terminate the linked list */
3962 if (q == (q_vector->num_ringpairs - 1))
3963 val |= (I40E_QUEUE_END_OF_LIST <<
3964 I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
3965
3966 wr32(hw, I40E_QINT_TQCTL(qp), val);
3967 qp++;
3968 }
3969 }
3970
3971 i40e_flush(hw);
3972 }
3973
3974 /**
3975 * i40e_enable_misc_int_causes - enable the non-queue interrupts
3976 * @pf: pointer to private device data structure
3977 **/
i40e_enable_misc_int_causes(struct i40e_pf * pf)3978 static void i40e_enable_misc_int_causes(struct i40e_pf *pf)
3979 {
3980 struct i40e_hw *hw = &pf->hw;
3981 u32 val;
3982
3983 /* clear things first */
3984 wr32(hw, I40E_PFINT_ICR0_ENA, 0); /* disable all */
3985 rd32(hw, I40E_PFINT_ICR0); /* read to clear */
3986
3987 val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK |
3988 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK |
3989 I40E_PFINT_ICR0_ENA_GRST_MASK |
3990 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK |
3991 I40E_PFINT_ICR0_ENA_GPIO_MASK |
3992 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK |
3993 I40E_PFINT_ICR0_ENA_VFLR_MASK |
3994 I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
3995
3996 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags))
3997 val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
3998
3999 if (test_bit(I40E_FLAG_PTP_ENA, pf->flags))
4000 val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
4001
4002 wr32(hw, I40E_PFINT_ICR0_ENA, val);
4003
4004 /* SW_ITR_IDX = 0, but don't change INTENA */
4005 wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
4006 I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
4007
4008 /* OTHER_ITR_IDX = 0 */
4009 wr32(hw, I40E_PFINT_STAT_CTL0, 0);
4010 }
4011
4012 /**
4013 * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW
4014 * @vsi: the VSI being configured
4015 **/
i40e_configure_msi_and_legacy(struct i40e_vsi * vsi)4016 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi)
4017 {
4018 u32 nextqp = i40e_enabled_xdp_vsi(vsi) ? vsi->alloc_queue_pairs : 0;
4019 struct i40e_q_vector *q_vector = vsi->q_vectors[0];
4020 struct i40e_pf *pf = vsi->back;
4021 struct i40e_hw *hw = &pf->hw;
4022
4023 /* set the ITR configuration */
4024 q_vector->rx.next_update = jiffies + 1;
4025 q_vector->rx.target_itr = ITR_TO_REG(vsi->rx_rings[0]->itr_setting);
4026 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.target_itr >> 1);
4027 q_vector->rx.current_itr = q_vector->rx.target_itr;
4028 q_vector->tx.next_update = jiffies + 1;
4029 q_vector->tx.target_itr = ITR_TO_REG(vsi->tx_rings[0]->itr_setting);
4030 wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.target_itr >> 1);
4031 q_vector->tx.current_itr = q_vector->tx.target_itr;
4032
4033 i40e_enable_misc_int_causes(pf);
4034
4035 /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
4036 wr32(hw, I40E_PFINT_LNKLST0, 0);
4037
4038 /* Associate the queue pair to the vector and enable the queue
4039 * interrupt RX queue in linked list with next queue set to TX
4040 */
4041 wr32(hw, I40E_QINT_RQCTL(0), I40E_QINT_RQCTL_VAL(nextqp, 0, TX));
4042
4043 if (i40e_enabled_xdp_vsi(vsi)) {
4044 /* TX queue in linked list with next queue set to TX */
4045 wr32(hw, I40E_QINT_TQCTL(nextqp),
4046 I40E_QINT_TQCTL_VAL(nextqp, 0, TX));
4047 }
4048
4049 /* last TX queue so the next RX queue doesn't matter */
4050 wr32(hw, I40E_QINT_TQCTL(0),
4051 I40E_QINT_TQCTL_VAL(I40E_QUEUE_END_OF_LIST, 0, RX));
4052 i40e_flush(hw);
4053 }
4054
4055 /**
4056 * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0
4057 * @pf: board private structure
4058 **/
i40e_irq_dynamic_disable_icr0(struct i40e_pf * pf)4059 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf)
4060 {
4061 struct i40e_hw *hw = &pf->hw;
4062
4063 wr32(hw, I40E_PFINT_DYN_CTL0,
4064 I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
4065 i40e_flush(hw);
4066 }
4067
4068 /**
4069 * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0
4070 * @pf: board private structure
4071 **/
i40e_irq_dynamic_enable_icr0(struct i40e_pf * pf)4072 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf)
4073 {
4074 struct i40e_hw *hw = &pf->hw;
4075 u32 val;
4076
4077 val = I40E_PFINT_DYN_CTL0_INTENA_MASK |
4078 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
4079 (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
4080
4081 wr32(hw, I40E_PFINT_DYN_CTL0, val);
4082 i40e_flush(hw);
4083 }
4084
4085 /**
4086 * i40e_msix_clean_rings - MSIX mode Interrupt Handler
4087 * @irq: interrupt number
4088 * @data: pointer to a q_vector
4089 **/
i40e_msix_clean_rings(int irq,void * data)4090 static irqreturn_t i40e_msix_clean_rings(int irq, void *data)
4091 {
4092 struct i40e_q_vector *q_vector = data;
4093
4094 if (!q_vector->tx.ring && !q_vector->rx.ring)
4095 return IRQ_HANDLED;
4096
4097 napi_schedule_irqoff(&q_vector->napi);
4098
4099 return IRQ_HANDLED;
4100 }
4101
4102 /**
4103 * i40e_irq_affinity_notify - Callback for affinity changes
4104 * @notify: context as to what irq was changed
4105 * @mask: the new affinity mask
4106 *
4107 * This is a callback function used by the irq_set_affinity_notifier function
4108 * so that we may register to receive changes to the irq affinity masks.
4109 **/
i40e_irq_affinity_notify(struct irq_affinity_notify * notify,const cpumask_t * mask)4110 static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify,
4111 const cpumask_t *mask)
4112 {
4113 struct i40e_q_vector *q_vector =
4114 container_of(notify, struct i40e_q_vector, affinity_notify);
4115
4116 cpumask_copy(&q_vector->affinity_mask, mask);
4117 }
4118
4119 /**
4120 * i40e_irq_affinity_release - Callback for affinity notifier release
4121 * @ref: internal core kernel usage
4122 *
4123 * This is a callback function used by the irq_set_affinity_notifier function
4124 * to inform the current notification subscriber that they will no longer
4125 * receive notifications.
4126 **/
i40e_irq_affinity_release(struct kref * ref)4127 static void i40e_irq_affinity_release(struct kref *ref) {}
4128
4129 /**
4130 * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts
4131 * @vsi: the VSI being configured
4132 * @basename: name for the vector
4133 *
4134 * Allocates MSI-X vectors and requests interrupts from the kernel.
4135 **/
i40e_vsi_request_irq_msix(struct i40e_vsi * vsi,char * basename)4136 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename)
4137 {
4138 int q_vectors = vsi->num_q_vectors;
4139 struct i40e_pf *pf = vsi->back;
4140 int base = vsi->base_vector;
4141 int rx_int_idx = 0;
4142 int tx_int_idx = 0;
4143 int vector, err;
4144 int irq_num;
4145 int cpu;
4146
4147 for (vector = 0; vector < q_vectors; vector++) {
4148 struct i40e_q_vector *q_vector = vsi->q_vectors[vector];
4149
4150 irq_num = pf->msix_entries[base + vector].vector;
4151
4152 if (q_vector->tx.ring && q_vector->rx.ring) {
4153 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
4154 "%s-%s-%d", basename, "TxRx", rx_int_idx++);
4155 tx_int_idx++;
4156 } else if (q_vector->rx.ring) {
4157 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
4158 "%s-%s-%d", basename, "rx", rx_int_idx++);
4159 } else if (q_vector->tx.ring) {
4160 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
4161 "%s-%s-%d", basename, "tx", tx_int_idx++);
4162 } else {
4163 /* skip this unused q_vector */
4164 continue;
4165 }
4166 err = request_irq(irq_num,
4167 vsi->irq_handler,
4168 0,
4169 q_vector->name,
4170 q_vector);
4171 if (err) {
4172 dev_info(&pf->pdev->dev,
4173 "MSIX request_irq failed, error: %d\n", err);
4174 goto free_queue_irqs;
4175 }
4176
4177 /* register for affinity change notifications */
4178 q_vector->irq_num = irq_num;
4179 q_vector->affinity_notify.notify = i40e_irq_affinity_notify;
4180 q_vector->affinity_notify.release = i40e_irq_affinity_release;
4181 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);
4182 /* Spread affinity hints out across online CPUs.
4183 *
4184 * get_cpu_mask returns a static constant mask with
4185 * a permanent lifetime so it's ok to pass to
4186 * irq_update_affinity_hint without making a copy.
4187 */
4188 cpu = cpumask_local_spread(q_vector->v_idx, -1);
4189 irq_update_affinity_hint(irq_num, get_cpu_mask(cpu));
4190 }
4191
4192 vsi->irqs_ready = true;
4193 return 0;
4194
4195 free_queue_irqs:
4196 while (vector) {
4197 vector--;
4198 irq_num = pf->msix_entries[base + vector].vector;
4199 irq_set_affinity_notifier(irq_num, NULL);
4200 irq_update_affinity_hint(irq_num, NULL);
4201 free_irq(irq_num, &vsi->q_vectors[vector]);
4202 }
4203 return err;
4204 }
4205
4206 /**
4207 * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI
4208 * @vsi: the VSI being un-configured
4209 **/
i40e_vsi_disable_irq(struct i40e_vsi * vsi)4210 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi)
4211 {
4212 struct i40e_pf *pf = vsi->back;
4213 struct i40e_hw *hw = &pf->hw;
4214 int base = vsi->base_vector;
4215 int i;
4216
4217 /* disable interrupt causation from each queue */
4218 for (i = 0; i < vsi->num_queue_pairs; i++) {
4219 u32 val;
4220
4221 val = rd32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx));
4222 val &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK;
4223 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), val);
4224
4225 val = rd32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx));
4226 val &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK;
4227 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), val);
4228
4229 if (!i40e_enabled_xdp_vsi(vsi))
4230 continue;
4231 wr32(hw, I40E_QINT_TQCTL(vsi->xdp_rings[i]->reg_idx), 0);
4232 }
4233
4234 /* disable each interrupt */
4235 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
4236 for (i = vsi->base_vector;
4237 i < (vsi->num_q_vectors + vsi->base_vector); i++)
4238 wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0);
4239
4240 i40e_flush(hw);
4241 for (i = 0; i < vsi->num_q_vectors; i++)
4242 synchronize_irq(pf->msix_entries[i + base].vector);
4243 } else {
4244 /* Legacy and MSI mode - this stops all interrupt handling */
4245 wr32(hw, I40E_PFINT_ICR0_ENA, 0);
4246 wr32(hw, I40E_PFINT_DYN_CTL0, 0);
4247 i40e_flush(hw);
4248 synchronize_irq(pf->pdev->irq);
4249 }
4250 }
4251
4252 /**
4253 * i40e_vsi_enable_irq - Enable IRQ for the given VSI
4254 * @vsi: the VSI being configured
4255 **/
i40e_vsi_enable_irq(struct i40e_vsi * vsi)4256 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi)
4257 {
4258 struct i40e_pf *pf = vsi->back;
4259 int i;
4260
4261 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
4262 for (i = 0; i < vsi->num_q_vectors; i++)
4263 i40e_irq_dynamic_enable(vsi, i);
4264 } else {
4265 i40e_irq_dynamic_enable_icr0(pf);
4266 }
4267
4268 i40e_flush(&pf->hw);
4269 return 0;
4270 }
4271
4272 /**
4273 * i40e_free_misc_vector - Free the vector that handles non-queue events
4274 * @pf: board private structure
4275 **/
i40e_free_misc_vector(struct i40e_pf * pf)4276 static void i40e_free_misc_vector(struct i40e_pf *pf)
4277 {
4278 /* Disable ICR 0 */
4279 wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0);
4280 i40e_flush(&pf->hw);
4281
4282 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags) && pf->msix_entries) {
4283 free_irq(pf->msix_entries[0].vector, pf);
4284 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);
4285 }
4286 }
4287
4288 /**
4289 * i40e_intr - MSI/Legacy and non-queue interrupt handler
4290 * @irq: interrupt number
4291 * @data: pointer to a q_vector
4292 *
4293 * This is the handler used for all MSI/Legacy interrupts, and deals
4294 * with both queue and non-queue interrupts. This is also used in
4295 * MSIX mode to handle the non-queue interrupts.
4296 **/
i40e_intr(int irq,void * data)4297 static irqreturn_t i40e_intr(int irq, void *data)
4298 {
4299 struct i40e_pf *pf = (struct i40e_pf *)data;
4300 struct i40e_hw *hw = &pf->hw;
4301 irqreturn_t ret = IRQ_NONE;
4302 u32 icr0, icr0_remaining;
4303 u32 val, ena_mask;
4304
4305 icr0 = rd32(hw, I40E_PFINT_ICR0);
4306 ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA);
4307
4308 /* if sharing a legacy IRQ, we might get called w/o an intr pending */
4309 if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0)
4310 goto enable_intr;
4311
4312 /* if interrupt but no bits showing, must be SWINT */
4313 if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) ||
4314 (icr0 & I40E_PFINT_ICR0_SWINT_MASK))
4315 pf->sw_int_count++;
4316
4317 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags) &&
4318 (icr0 & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) {
4319 ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
4320 dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n");
4321 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
4322 }
4323
4324 /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */
4325 if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) {
4326 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);
4327 struct i40e_q_vector *q_vector = vsi->q_vectors[0];
4328
4329 /* We do not have a way to disarm Queue causes while leaving
4330 * interrupt enabled for all other causes, ideally
4331 * interrupt should be disabled while we are in NAPI but
4332 * this is not a performance path and napi_schedule()
4333 * can deal with rescheduling.
4334 */
4335 if (!test_bit(__I40E_DOWN, pf->state))
4336 napi_schedule_irqoff(&q_vector->napi);
4337 }
4338
4339 if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
4340 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
4341 set_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);
4342 i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n");
4343 }
4344
4345 if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
4346 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
4347 set_bit(__I40E_MDD_EVENT_PENDING, pf->state);
4348 }
4349
4350 if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
4351 /* disable any further VFLR event notifications */
4352 if (test_bit(__I40E_VF_RESETS_DISABLED, pf->state)) {
4353 u32 reg = rd32(hw, I40E_PFINT_ICR0_ENA);
4354
4355 reg &= ~I40E_PFINT_ICR0_VFLR_MASK;
4356 wr32(hw, I40E_PFINT_ICR0_ENA, reg);
4357 } else {
4358 ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
4359 set_bit(__I40E_VFLR_EVENT_PENDING, pf->state);
4360 }
4361 }
4362
4363 if (icr0 & I40E_PFINT_ICR0_GRST_MASK) {
4364 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
4365 set_bit(__I40E_RESET_INTR_RECEIVED, pf->state);
4366 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK;
4367 val = rd32(hw, I40E_GLGEN_RSTAT);
4368 val = FIELD_GET(I40E_GLGEN_RSTAT_RESET_TYPE_MASK, val);
4369 if (val == I40E_RESET_CORER) {
4370 pf->corer_count++;
4371 } else if (val == I40E_RESET_GLOBR) {
4372 pf->globr_count++;
4373 } else if (val == I40E_RESET_EMPR) {
4374 pf->empr_count++;
4375 set_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state);
4376 }
4377 }
4378
4379 if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) {
4380 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK;
4381 dev_info(&pf->pdev->dev, "HMC error interrupt\n");
4382 dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n",
4383 rd32(hw, I40E_PFHMC_ERRORINFO),
4384 rd32(hw, I40E_PFHMC_ERRORDATA));
4385 }
4386
4387 if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) {
4388 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0);
4389
4390 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_EVENT0_MASK)
4391 schedule_work(&pf->ptp_extts0_work);
4392
4393 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK)
4394 i40e_ptp_tx_hwtstamp(pf);
4395
4396 icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
4397 }
4398
4399 /* If a critical error is pending we have no choice but to reset the
4400 * device.
4401 * Report and mask out any remaining unexpected interrupts.
4402 */
4403 icr0_remaining = icr0 & ena_mask;
4404 if (icr0_remaining) {
4405 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n",
4406 icr0_remaining);
4407 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) ||
4408 (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) ||
4409 (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) {
4410 dev_info(&pf->pdev->dev, "device will be reset\n");
4411 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
4412 i40e_service_event_schedule(pf);
4413 }
4414 ena_mask &= ~icr0_remaining;
4415 }
4416 ret = IRQ_HANDLED;
4417
4418 enable_intr:
4419 /* re-enable interrupt causes */
4420 wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask);
4421 if (!test_bit(__I40E_DOWN, pf->state) ||
4422 test_bit(__I40E_RECOVERY_MODE, pf->state)) {
4423 i40e_service_event_schedule(pf);
4424 i40e_irq_dynamic_enable_icr0(pf);
4425 }
4426
4427 return ret;
4428 }
4429
4430 /**
4431 * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes
4432 * @tx_ring: tx ring to clean
4433 * @budget: how many cleans we're allowed
4434 *
4435 * Returns true if there's any budget left (e.g. the clean is finished)
4436 **/
i40e_clean_fdir_tx_irq(struct i40e_ring * tx_ring,int budget)4437 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget)
4438 {
4439 struct i40e_vsi *vsi = tx_ring->vsi;
4440 u16 i = tx_ring->next_to_clean;
4441 struct i40e_tx_buffer *tx_buf;
4442 struct i40e_tx_desc *tx_desc;
4443
4444 tx_buf = &tx_ring->tx_bi[i];
4445 tx_desc = I40E_TX_DESC(tx_ring, i);
4446 i -= tx_ring->count;
4447
4448 do {
4449 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
4450
4451 /* if next_to_watch is not set then there is no work pending */
4452 if (!eop_desc)
4453 break;
4454
4455 /* prevent any other reads prior to eop_desc */
4456 smp_rmb();
4457
4458 /* if the descriptor isn't done, no work yet to do */
4459 if (!(eop_desc->cmd_type_offset_bsz &
4460 cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE)))
4461 break;
4462
4463 /* clear next_to_watch to prevent false hangs */
4464 tx_buf->next_to_watch = NULL;
4465
4466 tx_desc->buffer_addr = 0;
4467 tx_desc->cmd_type_offset_bsz = 0;
4468 /* move past filter desc */
4469 tx_buf++;
4470 tx_desc++;
4471 i++;
4472 if (unlikely(!i)) {
4473 i -= tx_ring->count;
4474 tx_buf = tx_ring->tx_bi;
4475 tx_desc = I40E_TX_DESC(tx_ring, 0);
4476 }
4477 /* unmap skb header data */
4478 dma_unmap_single(tx_ring->dev,
4479 dma_unmap_addr(tx_buf, dma),
4480 dma_unmap_len(tx_buf, len),
4481 DMA_TO_DEVICE);
4482 if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB)
4483 kfree(tx_buf->raw_buf);
4484
4485 tx_buf->raw_buf = NULL;
4486 tx_buf->tx_flags = 0;
4487 tx_buf->next_to_watch = NULL;
4488 dma_unmap_len_set(tx_buf, len, 0);
4489 tx_desc->buffer_addr = 0;
4490 tx_desc->cmd_type_offset_bsz = 0;
4491
4492 /* move us past the eop_desc for start of next FD desc */
4493 tx_buf++;
4494 tx_desc++;
4495 i++;
4496 if (unlikely(!i)) {
4497 i -= tx_ring->count;
4498 tx_buf = tx_ring->tx_bi;
4499 tx_desc = I40E_TX_DESC(tx_ring, 0);
4500 }
4501
4502 /* update budget accounting */
4503 budget--;
4504 } while (likely(budget));
4505
4506 i += tx_ring->count;
4507 tx_ring->next_to_clean = i;
4508
4509 if (test_bit(I40E_FLAG_MSIX_ENA, vsi->back->flags))
4510 i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx);
4511
4512 return budget > 0;
4513 }
4514
4515 /**
4516 * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring
4517 * @irq: interrupt number
4518 * @data: pointer to a q_vector
4519 **/
i40e_fdir_clean_ring(int irq,void * data)4520 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data)
4521 {
4522 struct i40e_q_vector *q_vector = data;
4523 struct i40e_vsi *vsi;
4524
4525 if (!q_vector->tx.ring)
4526 return IRQ_HANDLED;
4527
4528 vsi = q_vector->tx.ring->vsi;
4529 i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit);
4530
4531 return IRQ_HANDLED;
4532 }
4533
4534 /**
4535 * i40e_map_vector_to_qp - Assigns the queue pair to the vector
4536 * @vsi: the VSI being configured
4537 * @v_idx: vector index
4538 * @qp_idx: queue pair index
4539 **/
i40e_map_vector_to_qp(struct i40e_vsi * vsi,int v_idx,int qp_idx)4540 static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx)
4541 {
4542 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
4543 struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx];
4544 struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx];
4545
4546 tx_ring->q_vector = q_vector;
4547 tx_ring->next = q_vector->tx.ring;
4548 q_vector->tx.ring = tx_ring;
4549 q_vector->tx.count++;
4550
4551 /* Place XDP Tx ring in the same q_vector ring list as regular Tx */
4552 if (i40e_enabled_xdp_vsi(vsi)) {
4553 struct i40e_ring *xdp_ring = vsi->xdp_rings[qp_idx];
4554
4555 xdp_ring->q_vector = q_vector;
4556 xdp_ring->next = q_vector->tx.ring;
4557 q_vector->tx.ring = xdp_ring;
4558 q_vector->tx.count++;
4559 }
4560
4561 rx_ring->q_vector = q_vector;
4562 rx_ring->next = q_vector->rx.ring;
4563 q_vector->rx.ring = rx_ring;
4564 q_vector->rx.count++;
4565 }
4566
4567 /**
4568 * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors
4569 * @vsi: the VSI being configured
4570 *
4571 * This function maps descriptor rings to the queue-specific vectors
4572 * we were allotted through the MSI-X enabling code. Ideally, we'd have
4573 * one vector per queue pair, but on a constrained vector budget, we
4574 * group the queue pairs as "efficiently" as possible.
4575 **/
i40e_vsi_map_rings_to_vectors(struct i40e_vsi * vsi)4576 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi)
4577 {
4578 int qp_remaining = vsi->num_queue_pairs;
4579 int q_vectors = vsi->num_q_vectors;
4580 int num_ringpairs;
4581 int v_start = 0;
4582 int qp_idx = 0;
4583
4584 /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to
4585 * group them so there are multiple queues per vector.
4586 * It is also important to go through all the vectors available to be
4587 * sure that if we don't use all the vectors, that the remaining vectors
4588 * are cleared. This is especially important when decreasing the
4589 * number of queues in use.
4590 */
4591 for (; v_start < q_vectors; v_start++) {
4592 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start];
4593
4594 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start);
4595
4596 q_vector->num_ringpairs = num_ringpairs;
4597 q_vector->reg_idx = q_vector->v_idx + vsi->base_vector - 1;
4598
4599 q_vector->rx.count = 0;
4600 q_vector->tx.count = 0;
4601 q_vector->rx.ring = NULL;
4602 q_vector->tx.ring = NULL;
4603
4604 while (num_ringpairs--) {
4605 i40e_map_vector_to_qp(vsi, v_start, qp_idx);
4606 qp_idx++;
4607 qp_remaining--;
4608 }
4609 }
4610 }
4611
4612 /**
4613 * i40e_vsi_request_irq - Request IRQ from the OS
4614 * @vsi: the VSI being configured
4615 * @basename: name for the vector
4616 **/
i40e_vsi_request_irq(struct i40e_vsi * vsi,char * basename)4617 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename)
4618 {
4619 struct i40e_pf *pf = vsi->back;
4620 int err;
4621
4622 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
4623 err = i40e_vsi_request_irq_msix(vsi, basename);
4624 else if (test_bit(I40E_FLAG_MSI_ENA, pf->flags))
4625 err = request_irq(pf->pdev->irq, i40e_intr, 0,
4626 pf->int_name, pf);
4627 else
4628 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED,
4629 pf->int_name, pf);
4630
4631 if (err)
4632 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err);
4633
4634 return err;
4635 }
4636
4637 #ifdef CONFIG_NET_POLL_CONTROLLER
4638 /**
4639 * i40e_netpoll - A Polling 'interrupt' handler
4640 * @netdev: network interface device structure
4641 *
4642 * This is used by netconsole to send skbs without having to re-enable
4643 * interrupts. It's not called while the normal interrupt routine is executing.
4644 **/
i40e_netpoll(struct net_device * netdev)4645 static void i40e_netpoll(struct net_device *netdev)
4646 {
4647 struct i40e_netdev_priv *np = netdev_priv(netdev);
4648 struct i40e_vsi *vsi = np->vsi;
4649 struct i40e_pf *pf = vsi->back;
4650 int i;
4651
4652 /* if interface is down do nothing */
4653 if (test_bit(__I40E_VSI_DOWN, vsi->state))
4654 return;
4655
4656 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
4657 for (i = 0; i < vsi->num_q_vectors; i++)
4658 i40e_msix_clean_rings(0, vsi->q_vectors[i]);
4659 } else {
4660 i40e_intr(pf->pdev->irq, netdev);
4661 }
4662 }
4663 #endif
4664
4665 #define I40E_QTX_ENA_WAIT_COUNT 50
4666
4667 /**
4668 * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled
4669 * @pf: the PF being configured
4670 * @pf_q: the PF queue
4671 * @enable: enable or disable state of the queue
4672 *
4673 * This routine will wait for the given Tx queue of the PF to reach the
4674 * enabled or disabled state.
4675 * Returns -ETIMEDOUT in case of failing to reach the requested state after
4676 * multiple retries; else will return 0 in case of success.
4677 **/
i40e_pf_txq_wait(struct i40e_pf * pf,int pf_q,bool enable)4678 static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4679 {
4680 int i;
4681 u32 tx_reg;
4682
4683 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4684 tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q));
4685 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
4686 break;
4687
4688 usleep_range(10, 20);
4689 }
4690 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4691 return -ETIMEDOUT;
4692
4693 return 0;
4694 }
4695
4696 /**
4697 * i40e_control_tx_q - Start or stop a particular Tx queue
4698 * @pf: the PF structure
4699 * @pf_q: the PF queue to configure
4700 * @enable: start or stop the queue
4701 *
4702 * This function enables or disables a single queue. Note that any delay
4703 * required after the operation is expected to be handled by the caller of
4704 * this function.
4705 **/
i40e_control_tx_q(struct i40e_pf * pf,int pf_q,bool enable)4706 static void i40e_control_tx_q(struct i40e_pf *pf, int pf_q, bool enable)
4707 {
4708 struct i40e_hw *hw = &pf->hw;
4709 u32 tx_reg;
4710 int i;
4711
4712 /* warn the TX unit of coming changes */
4713 i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable);
4714 if (!enable)
4715 usleep_range(10, 20);
4716
4717 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
4718 tx_reg = rd32(hw, I40E_QTX_ENA(pf_q));
4719 if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) ==
4720 ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1))
4721 break;
4722 usleep_range(1000, 2000);
4723 }
4724
4725 /* Skip if the queue is already in the requested state */
4726 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
4727 return;
4728
4729 /* turn on/off the queue */
4730 if (enable) {
4731 wr32(hw, I40E_QTX_HEAD(pf_q), 0);
4732 tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK;
4733 } else {
4734 tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
4735 }
4736
4737 wr32(hw, I40E_QTX_ENA(pf_q), tx_reg);
4738 }
4739
4740 /**
4741 * i40e_control_wait_tx_q - Start/stop Tx queue and wait for completion
4742 * @seid: VSI SEID
4743 * @pf: the PF structure
4744 * @pf_q: the PF queue to configure
4745 * @is_xdp: true if the queue is used for XDP
4746 * @enable: start or stop the queue
4747 **/
i40e_control_wait_tx_q(int seid,struct i40e_pf * pf,int pf_q,bool is_xdp,bool enable)4748 int i40e_control_wait_tx_q(int seid, struct i40e_pf *pf, int pf_q,
4749 bool is_xdp, bool enable)
4750 {
4751 int ret;
4752
4753 i40e_control_tx_q(pf, pf_q, enable);
4754
4755 /* wait for the change to finish */
4756 ret = i40e_pf_txq_wait(pf, pf_q, enable);
4757 if (ret) {
4758 dev_info(&pf->pdev->dev,
4759 "VSI seid %d %sTx ring %d %sable timeout\n",
4760 seid, (is_xdp ? "XDP " : ""), pf_q,
4761 (enable ? "en" : "dis"));
4762 }
4763
4764 return ret;
4765 }
4766
4767 /**
4768 * i40e_vsi_enable_tx - Start a VSI's rings
4769 * @vsi: the VSI being configured
4770 **/
i40e_vsi_enable_tx(struct i40e_vsi * vsi)4771 static int i40e_vsi_enable_tx(struct i40e_vsi *vsi)
4772 {
4773 struct i40e_pf *pf = vsi->back;
4774 int i, pf_q, ret = 0;
4775
4776 pf_q = vsi->base_queue;
4777 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4778 ret = i40e_control_wait_tx_q(vsi->seid, pf,
4779 pf_q,
4780 false /*is xdp*/, true);
4781 if (ret)
4782 break;
4783
4784 if (!i40e_enabled_xdp_vsi(vsi))
4785 continue;
4786
4787 ret = i40e_control_wait_tx_q(vsi->seid, pf,
4788 pf_q + vsi->alloc_queue_pairs,
4789 true /*is xdp*/, true);
4790 if (ret)
4791 break;
4792 }
4793 return ret;
4794 }
4795
4796 /**
4797 * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled
4798 * @pf: the PF being configured
4799 * @pf_q: the PF queue
4800 * @enable: enable or disable state of the queue
4801 *
4802 * This routine will wait for the given Rx queue of the PF to reach the
4803 * enabled or disabled state.
4804 * Returns -ETIMEDOUT in case of failing to reach the requested state after
4805 * multiple retries; else will return 0 in case of success.
4806 **/
i40e_pf_rxq_wait(struct i40e_pf * pf,int pf_q,bool enable)4807 static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4808 {
4809 int i;
4810 u32 rx_reg;
4811
4812 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4813 rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q));
4814 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4815 break;
4816
4817 usleep_range(10, 20);
4818 }
4819 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4820 return -ETIMEDOUT;
4821
4822 return 0;
4823 }
4824
4825 /**
4826 * i40e_control_rx_q - Start or stop a particular Rx queue
4827 * @pf: the PF structure
4828 * @pf_q: the PF queue to configure
4829 * @enable: start or stop the queue
4830 *
4831 * This function enables or disables a single queue. Note that
4832 * any delay required after the operation is expected to be
4833 * handled by the caller of this function.
4834 **/
i40e_control_rx_q(struct i40e_pf * pf,int pf_q,bool enable)4835 static void i40e_control_rx_q(struct i40e_pf *pf, int pf_q, bool enable)
4836 {
4837 struct i40e_hw *hw = &pf->hw;
4838 u32 rx_reg;
4839 int i;
4840
4841 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
4842 rx_reg = rd32(hw, I40E_QRX_ENA(pf_q));
4843 if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) ==
4844 ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1))
4845 break;
4846 usleep_range(1000, 2000);
4847 }
4848
4849 /* Skip if the queue is already in the requested state */
4850 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4851 return;
4852
4853 /* turn on/off the queue */
4854 if (enable)
4855 rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK;
4856 else
4857 rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
4858
4859 wr32(hw, I40E_QRX_ENA(pf_q), rx_reg);
4860 }
4861
4862 /**
4863 * i40e_control_wait_rx_q
4864 * @pf: the PF structure
4865 * @pf_q: queue being configured
4866 * @enable: start or stop the rings
4867 *
4868 * This function enables or disables a single queue along with waiting
4869 * for the change to finish. The caller of this function should handle
4870 * the delays needed in the case of disabling queues.
4871 **/
i40e_control_wait_rx_q(struct i40e_pf * pf,int pf_q,bool enable)4872 int i40e_control_wait_rx_q(struct i40e_pf *pf, int pf_q, bool enable)
4873 {
4874 int ret = 0;
4875
4876 i40e_control_rx_q(pf, pf_q, enable);
4877
4878 /* wait for the change to finish */
4879 ret = i40e_pf_rxq_wait(pf, pf_q, enable);
4880 if (ret)
4881 return ret;
4882
4883 return ret;
4884 }
4885
4886 /**
4887 * i40e_vsi_enable_rx - Start a VSI's rings
4888 * @vsi: the VSI being configured
4889 **/
i40e_vsi_enable_rx(struct i40e_vsi * vsi)4890 static int i40e_vsi_enable_rx(struct i40e_vsi *vsi)
4891 {
4892 struct i40e_pf *pf = vsi->back;
4893 int i, pf_q, ret = 0;
4894
4895 pf_q = vsi->base_queue;
4896 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4897 ret = i40e_control_wait_rx_q(pf, pf_q, true);
4898 if (ret) {
4899 dev_info(&pf->pdev->dev,
4900 "VSI seid %d Rx ring %d enable timeout\n",
4901 vsi->seid, pf_q);
4902 break;
4903 }
4904 }
4905
4906 return ret;
4907 }
4908
4909 /**
4910 * i40e_vsi_start_rings - Start a VSI's rings
4911 * @vsi: the VSI being configured
4912 **/
i40e_vsi_start_rings(struct i40e_vsi * vsi)4913 int i40e_vsi_start_rings(struct i40e_vsi *vsi)
4914 {
4915 int ret = 0;
4916
4917 /* do rx first for enable and last for disable */
4918 ret = i40e_vsi_enable_rx(vsi);
4919 if (ret)
4920 return ret;
4921 ret = i40e_vsi_enable_tx(vsi);
4922
4923 return ret;
4924 }
4925
4926 #define I40E_DISABLE_TX_GAP_MSEC 50
4927
4928 /**
4929 * i40e_vsi_stop_rings - Stop a VSI's rings
4930 * @vsi: the VSI being configured
4931 **/
i40e_vsi_stop_rings(struct i40e_vsi * vsi)4932 void i40e_vsi_stop_rings(struct i40e_vsi *vsi)
4933 {
4934 struct i40e_pf *pf = vsi->back;
4935 u32 pf_q, tx_q_end, rx_q_end;
4936
4937 /* When port TX is suspended, don't wait */
4938 if (test_bit(__I40E_PORT_SUSPENDED, vsi->back->state))
4939 return i40e_vsi_stop_rings_no_wait(vsi);
4940
4941 tx_q_end = vsi->base_queue +
4942 vsi->alloc_queue_pairs * (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
4943 for (pf_q = vsi->base_queue; pf_q < tx_q_end; pf_q++)
4944 i40e_pre_tx_queue_cfg(&pf->hw, pf_q, false);
4945
4946 rx_q_end = vsi->base_queue + vsi->num_queue_pairs;
4947 for (pf_q = vsi->base_queue; pf_q < rx_q_end; pf_q++)
4948 i40e_control_rx_q(pf, pf_q, false);
4949
4950 msleep(I40E_DISABLE_TX_GAP_MSEC);
4951 for (pf_q = vsi->base_queue; pf_q < tx_q_end; pf_q++)
4952 wr32(&pf->hw, I40E_QTX_ENA(pf_q), 0);
4953
4954 i40e_vsi_wait_queues_disabled(vsi);
4955 }
4956
4957 /**
4958 * i40e_vsi_stop_rings_no_wait - Stop a VSI's rings and do not delay
4959 * @vsi: the VSI being shutdown
4960 *
4961 * This function stops all the rings for a VSI but does not delay to verify
4962 * that rings have been disabled. It is expected that the caller is shutting
4963 * down multiple VSIs at once and will delay together for all the VSIs after
4964 * initiating the shutdown. This is particularly useful for shutting down lots
4965 * of VFs together. Otherwise, a large delay can be incurred while configuring
4966 * each VSI in serial.
4967 **/
i40e_vsi_stop_rings_no_wait(struct i40e_vsi * vsi)4968 void i40e_vsi_stop_rings_no_wait(struct i40e_vsi *vsi)
4969 {
4970 struct i40e_pf *pf = vsi->back;
4971 int i, pf_q;
4972
4973 pf_q = vsi->base_queue;
4974 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4975 i40e_control_tx_q(pf, pf_q, false);
4976 i40e_control_rx_q(pf, pf_q, false);
4977 }
4978 }
4979
4980 /**
4981 * i40e_vsi_free_irq - Free the irq association with the OS
4982 * @vsi: the VSI being configured
4983 **/
i40e_vsi_free_irq(struct i40e_vsi * vsi)4984 static void i40e_vsi_free_irq(struct i40e_vsi *vsi)
4985 {
4986 struct i40e_pf *pf = vsi->back;
4987 struct i40e_hw *hw = &pf->hw;
4988 int base = vsi->base_vector;
4989 u32 val, qp;
4990 int i;
4991
4992 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
4993 if (!vsi->q_vectors)
4994 return;
4995
4996 if (!vsi->irqs_ready)
4997 return;
4998
4999 vsi->irqs_ready = false;
5000 for (i = 0; i < vsi->num_q_vectors; i++) {
5001 int irq_num;
5002 u16 vector;
5003
5004 vector = i + base;
5005 irq_num = pf->msix_entries[vector].vector;
5006
5007 /* free only the irqs that were actually requested */
5008 if (!vsi->q_vectors[i] ||
5009 !vsi->q_vectors[i]->num_ringpairs)
5010 continue;
5011
5012 /* clear the affinity notifier in the IRQ descriptor */
5013 irq_set_affinity_notifier(irq_num, NULL);
5014 /* remove our suggested affinity mask for this IRQ */
5015 irq_update_affinity_hint(irq_num, NULL);
5016 free_irq(irq_num, vsi->q_vectors[i]);
5017
5018 /* Tear down the interrupt queue link list
5019 *
5020 * We know that they come in pairs and always
5021 * the Rx first, then the Tx. To clear the
5022 * link list, stick the EOL value into the
5023 * next_q field of the registers.
5024 */
5025 val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1));
5026 qp = FIELD_GET(I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK,
5027 val);
5028 val |= I40E_QUEUE_END_OF_LIST
5029 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
5030 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val);
5031
5032 while (qp != I40E_QUEUE_END_OF_LIST) {
5033 u32 next;
5034
5035 val = rd32(hw, I40E_QINT_RQCTL(qp));
5036
5037 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK |
5038 I40E_QINT_RQCTL_MSIX0_INDX_MASK |
5039 I40E_QINT_RQCTL_CAUSE_ENA_MASK |
5040 I40E_QINT_RQCTL_INTEVENT_MASK);
5041
5042 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
5043 I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
5044
5045 wr32(hw, I40E_QINT_RQCTL(qp), val);
5046
5047 val = rd32(hw, I40E_QINT_TQCTL(qp));
5048
5049 next = FIELD_GET(I40E_QINT_TQCTL_NEXTQ_INDX_MASK,
5050 val);
5051
5052 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK |
5053 I40E_QINT_TQCTL_MSIX0_INDX_MASK |
5054 I40E_QINT_TQCTL_CAUSE_ENA_MASK |
5055 I40E_QINT_TQCTL_INTEVENT_MASK);
5056
5057 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
5058 I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
5059
5060 wr32(hw, I40E_QINT_TQCTL(qp), val);
5061 qp = next;
5062 }
5063 }
5064 } else {
5065 free_irq(pf->pdev->irq, pf);
5066
5067 val = rd32(hw, I40E_PFINT_LNKLST0);
5068 qp = FIELD_GET(I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK, val);
5069 val |= I40E_QUEUE_END_OF_LIST
5070 << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT;
5071 wr32(hw, I40E_PFINT_LNKLST0, val);
5072
5073 val = rd32(hw, I40E_QINT_RQCTL(qp));
5074 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK |
5075 I40E_QINT_RQCTL_MSIX0_INDX_MASK |
5076 I40E_QINT_RQCTL_CAUSE_ENA_MASK |
5077 I40E_QINT_RQCTL_INTEVENT_MASK);
5078
5079 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
5080 I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
5081
5082 wr32(hw, I40E_QINT_RQCTL(qp), val);
5083
5084 val = rd32(hw, I40E_QINT_TQCTL(qp));
5085
5086 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK |
5087 I40E_QINT_TQCTL_MSIX0_INDX_MASK |
5088 I40E_QINT_TQCTL_CAUSE_ENA_MASK |
5089 I40E_QINT_TQCTL_INTEVENT_MASK);
5090
5091 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
5092 I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
5093
5094 wr32(hw, I40E_QINT_TQCTL(qp), val);
5095 }
5096 }
5097
5098 /**
5099 * i40e_free_q_vector - Free memory allocated for specific interrupt vector
5100 * @vsi: the VSI being configured
5101 * @v_idx: Index of vector to be freed
5102 *
5103 * This function frees the memory allocated to the q_vector. In addition if
5104 * NAPI is enabled it will delete any references to the NAPI struct prior
5105 * to freeing the q_vector.
5106 **/
i40e_free_q_vector(struct i40e_vsi * vsi,int v_idx)5107 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx)
5108 {
5109 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
5110 struct i40e_ring *ring;
5111
5112 if (!q_vector)
5113 return;
5114
5115 /* disassociate q_vector from rings */
5116 i40e_for_each_ring(ring, q_vector->tx)
5117 ring->q_vector = NULL;
5118
5119 i40e_for_each_ring(ring, q_vector->rx)
5120 ring->q_vector = NULL;
5121
5122 /* only VSI w/ an associated netdev is set up w/ NAPI */
5123 if (vsi->netdev)
5124 netif_napi_del(&q_vector->napi);
5125
5126 vsi->q_vectors[v_idx] = NULL;
5127
5128 kfree_rcu(q_vector, rcu);
5129 }
5130
5131 /**
5132 * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors
5133 * @vsi: the VSI being un-configured
5134 *
5135 * This frees the memory allocated to the q_vectors and
5136 * deletes references to the NAPI struct.
5137 **/
i40e_vsi_free_q_vectors(struct i40e_vsi * vsi)5138 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi)
5139 {
5140 int v_idx;
5141
5142 for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++)
5143 i40e_free_q_vector(vsi, v_idx);
5144 }
5145
5146 /**
5147 * i40e_reset_interrupt_capability - Disable interrupt setup in OS
5148 * @pf: board private structure
5149 **/
i40e_reset_interrupt_capability(struct i40e_pf * pf)5150 static void i40e_reset_interrupt_capability(struct i40e_pf *pf)
5151 {
5152 /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */
5153 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
5154 pci_disable_msix(pf->pdev);
5155 kfree(pf->msix_entries);
5156 pf->msix_entries = NULL;
5157 kfree(pf->irq_pile);
5158 pf->irq_pile = NULL;
5159 } else if (test_bit(I40E_FLAG_MSI_ENA, pf->flags)) {
5160 pci_disable_msi(pf->pdev);
5161 }
5162 clear_bit(I40E_FLAG_MSI_ENA, pf->flags);
5163 clear_bit(I40E_FLAG_MSIX_ENA, pf->flags);
5164 }
5165
5166 /**
5167 * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings
5168 * @pf: board private structure
5169 *
5170 * We go through and clear interrupt specific resources and reset the structure
5171 * to pre-load conditions
5172 **/
i40e_clear_interrupt_scheme(struct i40e_pf * pf)5173 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf)
5174 {
5175 struct i40e_vsi *vsi;
5176 int i;
5177
5178 if (test_bit(__I40E_MISC_IRQ_REQUESTED, pf->state))
5179 i40e_free_misc_vector(pf);
5180
5181 i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector,
5182 I40E_IWARP_IRQ_PILE_ID);
5183
5184 i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1);
5185
5186 i40e_pf_for_each_vsi(pf, i, vsi)
5187 i40e_vsi_free_q_vectors(vsi);
5188
5189 i40e_reset_interrupt_capability(pf);
5190 }
5191
5192 /**
5193 * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI
5194 * @vsi: the VSI being configured
5195 **/
i40e_napi_enable_all(struct i40e_vsi * vsi)5196 static void i40e_napi_enable_all(struct i40e_vsi *vsi)
5197 {
5198 int q_idx;
5199
5200 if (!vsi->netdev)
5201 return;
5202
5203 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
5204 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
5205
5206 if (q_vector->rx.ring || q_vector->tx.ring)
5207 napi_enable(&q_vector->napi);
5208 }
5209 }
5210
5211 /**
5212 * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI
5213 * @vsi: the VSI being configured
5214 **/
i40e_napi_disable_all(struct i40e_vsi * vsi)5215 static void i40e_napi_disable_all(struct i40e_vsi *vsi)
5216 {
5217 int q_idx;
5218
5219 if (!vsi->netdev)
5220 return;
5221
5222 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
5223 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
5224
5225 if (q_vector->rx.ring || q_vector->tx.ring)
5226 napi_disable(&q_vector->napi);
5227 }
5228 }
5229
5230 /**
5231 * i40e_vsi_close - Shut down a VSI
5232 * @vsi: the vsi to be quelled
5233 **/
i40e_vsi_close(struct i40e_vsi * vsi)5234 static void i40e_vsi_close(struct i40e_vsi *vsi)
5235 {
5236 struct i40e_pf *pf = vsi->back;
5237 if (!test_and_set_bit(__I40E_VSI_DOWN, vsi->state))
5238 i40e_down(vsi);
5239 i40e_vsi_free_irq(vsi);
5240 i40e_vsi_free_tx_resources(vsi);
5241 i40e_vsi_free_rx_resources(vsi);
5242 vsi->current_netdev_flags = 0;
5243 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
5244 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
5245 set_bit(__I40E_CLIENT_RESET, pf->state);
5246 }
5247
5248 /**
5249 * i40e_quiesce_vsi - Pause a given VSI
5250 * @vsi: the VSI being paused
5251 **/
i40e_quiesce_vsi(struct i40e_vsi * vsi)5252 static void i40e_quiesce_vsi(struct i40e_vsi *vsi)
5253 {
5254 if (test_bit(__I40E_VSI_DOWN, vsi->state))
5255 return;
5256
5257 set_bit(__I40E_VSI_NEEDS_RESTART, vsi->state);
5258 if (vsi->netdev && netif_running(vsi->netdev))
5259 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev);
5260 else
5261 i40e_vsi_close(vsi);
5262 }
5263
5264 /**
5265 * i40e_unquiesce_vsi - Resume a given VSI
5266 * @vsi: the VSI being resumed
5267 **/
i40e_unquiesce_vsi(struct i40e_vsi * vsi)5268 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi)
5269 {
5270 if (!test_and_clear_bit(__I40E_VSI_NEEDS_RESTART, vsi->state))
5271 return;
5272
5273 if (vsi->netdev && netif_running(vsi->netdev))
5274 vsi->netdev->netdev_ops->ndo_open(vsi->netdev);
5275 else
5276 i40e_vsi_open(vsi); /* this clears the DOWN bit */
5277 }
5278
5279 /**
5280 * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF
5281 * @pf: the PF
5282 **/
i40e_pf_quiesce_all_vsi(struct i40e_pf * pf)5283 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf)
5284 {
5285 struct i40e_vsi *vsi;
5286 int v;
5287
5288 i40e_pf_for_each_vsi(pf, v, vsi)
5289 i40e_quiesce_vsi(vsi);
5290 }
5291
5292 /**
5293 * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF
5294 * @pf: the PF
5295 **/
i40e_pf_unquiesce_all_vsi(struct i40e_pf * pf)5296 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf)
5297 {
5298 struct i40e_vsi *vsi;
5299 int v;
5300
5301 i40e_pf_for_each_vsi(pf, v, vsi)
5302 i40e_unquiesce_vsi(vsi);
5303 }
5304
5305 /**
5306 * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled
5307 * @vsi: the VSI being configured
5308 *
5309 * Wait until all queues on a given VSI have been disabled.
5310 **/
i40e_vsi_wait_queues_disabled(struct i40e_vsi * vsi)5311 int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi)
5312 {
5313 struct i40e_pf *pf = vsi->back;
5314 int i, pf_q, ret;
5315
5316 pf_q = vsi->base_queue;
5317 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
5318 /* Check and wait for the Tx queue */
5319 ret = i40e_pf_txq_wait(pf, pf_q, false);
5320 if (ret) {
5321 dev_info(&pf->pdev->dev,
5322 "VSI seid %d Tx ring %d disable timeout\n",
5323 vsi->seid, pf_q);
5324 return ret;
5325 }
5326
5327 if (!i40e_enabled_xdp_vsi(vsi))
5328 goto wait_rx;
5329
5330 /* Check and wait for the XDP Tx queue */
5331 ret = i40e_pf_txq_wait(pf, pf_q + vsi->alloc_queue_pairs,
5332 false);
5333 if (ret) {
5334 dev_info(&pf->pdev->dev,
5335 "VSI seid %d XDP Tx ring %d disable timeout\n",
5336 vsi->seid, pf_q);
5337 return ret;
5338 }
5339 wait_rx:
5340 /* Check and wait for the Rx queue */
5341 ret = i40e_pf_rxq_wait(pf, pf_q, false);
5342 if (ret) {
5343 dev_info(&pf->pdev->dev,
5344 "VSI seid %d Rx ring %d disable timeout\n",
5345 vsi->seid, pf_q);
5346 return ret;
5347 }
5348 }
5349
5350 return 0;
5351 }
5352
5353 #ifdef CONFIG_I40E_DCB
5354 /**
5355 * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled
5356 * @pf: the PF
5357 *
5358 * This function waits for the queues to be in disabled state for all the
5359 * VSIs that are managed by this PF.
5360 **/
i40e_pf_wait_queues_disabled(struct i40e_pf * pf)5361 static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf)
5362 {
5363 struct i40e_vsi *vsi;
5364 int v, ret = 0;
5365
5366 i40e_pf_for_each_vsi(pf, v, vsi) {
5367 ret = i40e_vsi_wait_queues_disabled(vsi);
5368 if (ret)
5369 break;
5370 }
5371
5372 return ret;
5373 }
5374
5375 #endif
5376
5377 /**
5378 * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP
5379 * @pf: pointer to PF
5380 *
5381 * Get TC map for ISCSI PF type that will include iSCSI TC
5382 * and LAN TC.
5383 **/
i40e_get_iscsi_tc_map(struct i40e_pf * pf)5384 static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf)
5385 {
5386 struct i40e_dcb_app_priority_table app;
5387 struct i40e_hw *hw = &pf->hw;
5388 u8 enabled_tc = 1; /* TC0 is always enabled */
5389 u8 tc, i;
5390 /* Get the iSCSI APP TLV */
5391 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5392
5393 for (i = 0; i < dcbcfg->numapps; i++) {
5394 app = dcbcfg->app[i];
5395 if (app.selector == I40E_APP_SEL_TCPIP &&
5396 app.protocolid == I40E_APP_PROTOID_ISCSI) {
5397 tc = dcbcfg->etscfg.prioritytable[app.priority];
5398 enabled_tc |= BIT(tc);
5399 break;
5400 }
5401 }
5402
5403 return enabled_tc;
5404 }
5405
5406 /**
5407 * i40e_dcb_get_num_tc - Get the number of TCs from DCBx config
5408 * @dcbcfg: the corresponding DCBx configuration structure
5409 *
5410 * Return the number of TCs from given DCBx configuration
5411 **/
i40e_dcb_get_num_tc(struct i40e_dcbx_config * dcbcfg)5412 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg)
5413 {
5414 int i, tc_unused = 0;
5415 u8 num_tc = 0;
5416 u8 ret = 0;
5417
5418 /* Scan the ETS Config Priority Table to find
5419 * traffic class enabled for a given priority
5420 * and create a bitmask of enabled TCs
5421 */
5422 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++)
5423 num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]);
5424
5425 /* Now scan the bitmask to check for
5426 * contiguous TCs starting with TC0
5427 */
5428 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5429 if (num_tc & BIT(i)) {
5430 if (!tc_unused) {
5431 ret++;
5432 } else {
5433 pr_err("Non-contiguous TC - Disabling DCB\n");
5434 return 1;
5435 }
5436 } else {
5437 tc_unused = 1;
5438 }
5439 }
5440
5441 /* There is always at least TC0 */
5442 if (!ret)
5443 ret = 1;
5444
5445 return ret;
5446 }
5447
5448 /**
5449 * i40e_dcb_get_enabled_tc - Get enabled traffic classes
5450 * @dcbcfg: the corresponding DCBx configuration structure
5451 *
5452 * Query the current DCB configuration and return the number of
5453 * traffic classes enabled from the given DCBX config
5454 **/
i40e_dcb_get_enabled_tc(struct i40e_dcbx_config * dcbcfg)5455 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg)
5456 {
5457 u8 num_tc = i40e_dcb_get_num_tc(dcbcfg);
5458 u8 enabled_tc = 1;
5459 u8 i;
5460
5461 for (i = 0; i < num_tc; i++)
5462 enabled_tc |= BIT(i);
5463
5464 return enabled_tc;
5465 }
5466
5467 /**
5468 * i40e_mqprio_get_enabled_tc - Get enabled traffic classes
5469 * @pf: PF being queried
5470 *
5471 * Query the current MQPRIO configuration and return the number of
5472 * traffic classes enabled.
5473 **/
i40e_mqprio_get_enabled_tc(struct i40e_pf * pf)5474 static u8 i40e_mqprio_get_enabled_tc(struct i40e_pf *pf)
5475 {
5476 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);
5477 u8 num_tc = vsi->mqprio_qopt.qopt.num_tc;
5478 u8 enabled_tc = 1, i;
5479
5480 for (i = 1; i < num_tc; i++)
5481 enabled_tc |= BIT(i);
5482 return enabled_tc;
5483 }
5484
5485 /**
5486 * i40e_pf_get_num_tc - Get enabled traffic classes for PF
5487 * @pf: PF being queried
5488 *
5489 * Return number of traffic classes enabled for the given PF
5490 **/
i40e_pf_get_num_tc(struct i40e_pf * pf)5491 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf)
5492 {
5493 u8 i, enabled_tc = 1;
5494 u8 num_tc = 0;
5495
5496 if (i40e_is_tc_mqprio_enabled(pf)) {
5497 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);
5498
5499 return vsi->mqprio_qopt.qopt.num_tc;
5500 }
5501
5502 /* If neither MQPRIO nor DCB is enabled, then always use single TC */
5503 if (!test_bit(I40E_FLAG_DCB_ENA, pf->flags))
5504 return 1;
5505
5506 /* SFP mode will be enabled for all TCs on port */
5507 if (!test_bit(I40E_FLAG_MFP_ENA, pf->flags))
5508 return i40e_dcb_get_num_tc(&pf->hw.local_dcbx_config);
5509
5510 /* MFP mode return count of enabled TCs for this PF */
5511 if (pf->hw.func_caps.iscsi)
5512 enabled_tc = i40e_get_iscsi_tc_map(pf);
5513 else
5514 return 1; /* Only TC0 */
5515
5516 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5517 if (enabled_tc & BIT(i))
5518 num_tc++;
5519 }
5520 return num_tc;
5521 }
5522
5523 /**
5524 * i40e_pf_get_tc_map - Get bitmap for enabled traffic classes
5525 * @pf: PF being queried
5526 *
5527 * Return a bitmap for enabled traffic classes for this PF.
5528 **/
i40e_pf_get_tc_map(struct i40e_pf * pf)5529 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf)
5530 {
5531 if (i40e_is_tc_mqprio_enabled(pf))
5532 return i40e_mqprio_get_enabled_tc(pf);
5533
5534 /* If neither MQPRIO nor DCB is enabled for this PF then just return
5535 * default TC
5536 */
5537 if (!test_bit(I40E_FLAG_DCB_ENA, pf->flags))
5538 return I40E_DEFAULT_TRAFFIC_CLASS;
5539
5540 /* SFP mode we want PF to be enabled for all TCs */
5541 if (!test_bit(I40E_FLAG_MFP_ENA, pf->flags))
5542 return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config);
5543
5544 /* MFP enabled and iSCSI PF type */
5545 if (pf->hw.func_caps.iscsi)
5546 return i40e_get_iscsi_tc_map(pf);
5547 else
5548 return I40E_DEFAULT_TRAFFIC_CLASS;
5549 }
5550
5551 /**
5552 * i40e_vsi_get_bw_info - Query VSI BW Information
5553 * @vsi: the VSI being queried
5554 *
5555 * Returns 0 on success, negative value on failure
5556 **/
i40e_vsi_get_bw_info(struct i40e_vsi * vsi)5557 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi)
5558 {
5559 struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0};
5560 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
5561 struct i40e_pf *pf = vsi->back;
5562 struct i40e_hw *hw = &pf->hw;
5563 u32 tc_bw_max;
5564 int ret;
5565 int i;
5566
5567 /* Get the VSI level BW configuration */
5568 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL);
5569 if (ret) {
5570 dev_info(&pf->pdev->dev,
5571 "couldn't get PF vsi bw config, err %pe aq_err %s\n",
5572 ERR_PTR(ret),
5573 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5574 return -EINVAL;
5575 }
5576
5577 /* Get the VSI level BW configuration per TC */
5578 ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config,
5579 NULL);
5580 if (ret) {
5581 dev_info(&pf->pdev->dev,
5582 "couldn't get PF vsi ets bw config, err %pe aq_err %s\n",
5583 ERR_PTR(ret),
5584 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5585 return -EINVAL;
5586 }
5587
5588 if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) {
5589 dev_info(&pf->pdev->dev,
5590 "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n",
5591 bw_config.tc_valid_bits,
5592 bw_ets_config.tc_valid_bits);
5593 /* Still continuing */
5594 }
5595
5596 vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit);
5597 vsi->bw_max_quanta = bw_config.max_bw;
5598 tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) |
5599 (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16);
5600 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5601 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i];
5602 vsi->bw_ets_limit_credits[i] =
5603 le16_to_cpu(bw_ets_config.credits[i]);
5604 /* 3 bits out of 4 for each TC */
5605 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7);
5606 }
5607
5608 return 0;
5609 }
5610
5611 /**
5612 * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC
5613 * @vsi: the VSI being configured
5614 * @enabled_tc: TC bitmap
5615 * @bw_share: BW shared credits per TC
5616 *
5617 * Returns 0 on success, negative value on failure
5618 **/
i40e_vsi_configure_bw_alloc(struct i40e_vsi * vsi,u8 enabled_tc,u8 * bw_share)5619 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc,
5620 u8 *bw_share)
5621 {
5622 struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
5623 struct i40e_pf *pf = vsi->back;
5624 int ret;
5625 int i;
5626
5627 /* There is no need to reset BW when mqprio mode is on. */
5628 if (i40e_is_tc_mqprio_enabled(pf))
5629 return 0;
5630 if (!vsi->mqprio_qopt.qopt.hw && !test_bit(I40E_FLAG_DCB_ENA, pf->flags)) {
5631 ret = i40e_set_bw_limit(vsi, vsi->seid, 0);
5632 if (ret)
5633 dev_info(&pf->pdev->dev,
5634 "Failed to reset tx rate for vsi->seid %u\n",
5635 vsi->seid);
5636 return ret;
5637 }
5638 memset(&bw_data, 0, sizeof(bw_data));
5639 bw_data.tc_valid_bits = enabled_tc;
5640 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5641 bw_data.tc_bw_credits[i] = bw_share[i];
5642
5643 ret = i40e_aq_config_vsi_tc_bw(&pf->hw, vsi->seid, &bw_data, NULL);
5644 if (ret) {
5645 dev_info(&pf->pdev->dev,
5646 "AQ command Config VSI BW allocation per TC failed = %d\n",
5647 pf->hw.aq.asq_last_status);
5648 return -EINVAL;
5649 }
5650
5651 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5652 vsi->info.qs_handle[i] = bw_data.qs_handles[i];
5653
5654 return 0;
5655 }
5656
5657 /**
5658 * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration
5659 * @vsi: the VSI being configured
5660 * @enabled_tc: TC map to be enabled
5661 *
5662 **/
i40e_vsi_config_netdev_tc(struct i40e_vsi * vsi,u8 enabled_tc)5663 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc)
5664 {
5665 struct net_device *netdev = vsi->netdev;
5666 struct i40e_pf *pf = vsi->back;
5667 struct i40e_hw *hw = &pf->hw;
5668 u8 netdev_tc = 0;
5669 int i;
5670 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5671
5672 if (!netdev)
5673 return;
5674
5675 if (!enabled_tc) {
5676 netdev_reset_tc(netdev);
5677 return;
5678 }
5679
5680 /* Set up actual enabled TCs on the VSI */
5681 if (netdev_set_num_tc(netdev, vsi->tc_config.numtc))
5682 return;
5683
5684 /* set per TC queues for the VSI */
5685 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5686 /* Only set TC queues for enabled tcs
5687 *
5688 * e.g. For a VSI that has TC0 and TC3 enabled the
5689 * enabled_tc bitmap would be 0x00001001; the driver
5690 * will set the numtc for netdev as 2 that will be
5691 * referenced by the netdev layer as TC 0 and 1.
5692 */
5693 if (vsi->tc_config.enabled_tc & BIT(i))
5694 netdev_set_tc_queue(netdev,
5695 vsi->tc_config.tc_info[i].netdev_tc,
5696 vsi->tc_config.tc_info[i].qcount,
5697 vsi->tc_config.tc_info[i].qoffset);
5698 }
5699
5700 if (i40e_is_tc_mqprio_enabled(pf))
5701 return;
5702
5703 /* Assign UP2TC map for the VSI */
5704 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
5705 /* Get the actual TC# for the UP */
5706 u8 ets_tc = dcbcfg->etscfg.prioritytable[i];
5707 /* Get the mapped netdev TC# for the UP */
5708 netdev_tc = vsi->tc_config.tc_info[ets_tc].netdev_tc;
5709 netdev_set_prio_tc_map(netdev, i, netdev_tc);
5710 }
5711 }
5712
5713 /**
5714 * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map
5715 * @vsi: the VSI being configured
5716 * @ctxt: the ctxt buffer returned from AQ VSI update param command
5717 **/
i40e_vsi_update_queue_map(struct i40e_vsi * vsi,struct i40e_vsi_context * ctxt)5718 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi,
5719 struct i40e_vsi_context *ctxt)
5720 {
5721 /* copy just the sections touched not the entire info
5722 * since not all sections are valid as returned by
5723 * update vsi params
5724 */
5725 vsi->info.mapping_flags = ctxt->info.mapping_flags;
5726 memcpy(&vsi->info.queue_mapping,
5727 &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping));
5728 memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping,
5729 sizeof(vsi->info.tc_mapping));
5730 }
5731
5732 /**
5733 * i40e_update_adq_vsi_queues - update queue mapping for ADq VSI
5734 * @vsi: the VSI being reconfigured
5735 * @vsi_offset: offset from main VF VSI
5736 */
i40e_update_adq_vsi_queues(struct i40e_vsi * vsi,int vsi_offset)5737 int i40e_update_adq_vsi_queues(struct i40e_vsi *vsi, int vsi_offset)
5738 {
5739 struct i40e_vsi_context ctxt = {};
5740 struct i40e_pf *pf;
5741 struct i40e_hw *hw;
5742 int ret;
5743
5744 if (!vsi)
5745 return -EINVAL;
5746 pf = vsi->back;
5747 hw = &pf->hw;
5748
5749 ctxt.seid = vsi->seid;
5750 ctxt.pf_num = hw->pf_id;
5751 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id + vsi_offset;
5752 ctxt.uplink_seid = vsi->uplink_seid;
5753 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
5754 ctxt.flags = I40E_AQ_VSI_TYPE_VF;
5755 ctxt.info = vsi->info;
5756
5757 i40e_vsi_setup_queue_map(vsi, &ctxt, vsi->tc_config.enabled_tc,
5758 false);
5759 if (vsi->reconfig_rss) {
5760 vsi->rss_size = min_t(int, pf->alloc_rss_size,
5761 vsi->num_queue_pairs);
5762 ret = i40e_vsi_config_rss(vsi);
5763 if (ret) {
5764 dev_info(&pf->pdev->dev, "Failed to reconfig rss for num_queues\n");
5765 return ret;
5766 }
5767 vsi->reconfig_rss = false;
5768 }
5769
5770 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
5771 if (ret) {
5772 dev_info(&pf->pdev->dev, "Update vsi config failed, err %pe aq_err %s\n",
5773 ERR_PTR(ret),
5774 i40e_aq_str(hw, hw->aq.asq_last_status));
5775 return ret;
5776 }
5777 /* update the local VSI info with updated queue map */
5778 i40e_vsi_update_queue_map(vsi, &ctxt);
5779 vsi->info.valid_sections = 0;
5780
5781 return ret;
5782 }
5783
5784 /**
5785 * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map
5786 * @vsi: VSI to be configured
5787 * @enabled_tc: TC bitmap
5788 *
5789 * This configures a particular VSI for TCs that are mapped to the
5790 * given TC bitmap. It uses default bandwidth share for TCs across
5791 * VSIs to configure TC for a particular VSI.
5792 *
5793 * NOTE:
5794 * It is expected that the VSI queues have been quisced before calling
5795 * this function.
5796 **/
i40e_vsi_config_tc(struct i40e_vsi * vsi,u8 enabled_tc)5797 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc)
5798 {
5799 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
5800 struct i40e_pf *pf = vsi->back;
5801 struct i40e_hw *hw = &pf->hw;
5802 struct i40e_vsi_context ctxt;
5803 int ret = 0;
5804 int i;
5805
5806 /* Check if enabled_tc is same as existing or new TCs */
5807 if (vsi->tc_config.enabled_tc == enabled_tc &&
5808 vsi->mqprio_qopt.mode != TC_MQPRIO_MODE_CHANNEL)
5809 return ret;
5810
5811 /* Enable ETS TCs with equal BW Share for now across all VSIs */
5812 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5813 if (enabled_tc & BIT(i))
5814 bw_share[i] = 1;
5815 }
5816
5817 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
5818 if (ret) {
5819 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
5820
5821 dev_info(&pf->pdev->dev,
5822 "Failed configuring TC map %d for VSI %d\n",
5823 enabled_tc, vsi->seid);
5824 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid,
5825 &bw_config, NULL);
5826 if (ret) {
5827 dev_info(&pf->pdev->dev,
5828 "Failed querying vsi bw info, err %pe aq_err %s\n",
5829 ERR_PTR(ret),
5830 i40e_aq_str(hw, hw->aq.asq_last_status));
5831 goto out;
5832 }
5833 if ((bw_config.tc_valid_bits & enabled_tc) != enabled_tc) {
5834 u8 valid_tc = bw_config.tc_valid_bits & enabled_tc;
5835
5836 if (!valid_tc)
5837 valid_tc = bw_config.tc_valid_bits;
5838 /* Always enable TC0, no matter what */
5839 valid_tc |= 1;
5840 dev_info(&pf->pdev->dev,
5841 "Requested tc 0x%x, but FW reports 0x%x as valid. Attempting to use 0x%x.\n",
5842 enabled_tc, bw_config.tc_valid_bits, valid_tc);
5843 enabled_tc = valid_tc;
5844 }
5845
5846 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
5847 if (ret) {
5848 dev_err(&pf->pdev->dev,
5849 "Unable to configure TC map %d for VSI %d\n",
5850 enabled_tc, vsi->seid);
5851 goto out;
5852 }
5853 }
5854
5855 /* Update Queue Pairs Mapping for currently enabled UPs */
5856 ctxt.seid = vsi->seid;
5857 ctxt.pf_num = vsi->back->hw.pf_id;
5858 ctxt.vf_num = 0;
5859 ctxt.uplink_seid = vsi->uplink_seid;
5860 ctxt.info = vsi->info;
5861 if (i40e_is_tc_mqprio_enabled(pf)) {
5862 ret = i40e_vsi_setup_queue_map_mqprio(vsi, &ctxt, enabled_tc);
5863 if (ret)
5864 goto out;
5865 } else {
5866 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
5867 }
5868
5869 /* On destroying the qdisc, reset vsi->rss_size, as number of enabled
5870 * queues changed.
5871 */
5872 if (!vsi->mqprio_qopt.qopt.hw && vsi->reconfig_rss) {
5873 vsi->rss_size = min_t(int, vsi->back->alloc_rss_size,
5874 vsi->num_queue_pairs);
5875 ret = i40e_vsi_config_rss(vsi);
5876 if (ret) {
5877 dev_info(&vsi->back->pdev->dev,
5878 "Failed to reconfig rss for num_queues\n");
5879 return ret;
5880 }
5881 vsi->reconfig_rss = false;
5882 }
5883 if (test_bit(I40E_FLAG_IWARP_ENA, vsi->back->flags)) {
5884 ctxt.info.valid_sections |=
5885 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
5886 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA;
5887 }
5888
5889 /* Update the VSI after updating the VSI queue-mapping
5890 * information
5891 */
5892 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
5893 if (ret) {
5894 dev_info(&pf->pdev->dev,
5895 "Update vsi tc config failed, err %pe aq_err %s\n",
5896 ERR_PTR(ret),
5897 i40e_aq_str(hw, hw->aq.asq_last_status));
5898 goto out;
5899 }
5900 /* update the local VSI info with updated queue map */
5901 i40e_vsi_update_queue_map(vsi, &ctxt);
5902 vsi->info.valid_sections = 0;
5903
5904 /* Update current VSI BW information */
5905 ret = i40e_vsi_get_bw_info(vsi);
5906 if (ret) {
5907 dev_info(&pf->pdev->dev,
5908 "Failed updating vsi bw info, err %pe aq_err %s\n",
5909 ERR_PTR(ret),
5910 i40e_aq_str(hw, hw->aq.asq_last_status));
5911 goto out;
5912 }
5913
5914 /* Update the netdev TC setup */
5915 i40e_vsi_config_netdev_tc(vsi, enabled_tc);
5916 out:
5917 return ret;
5918 }
5919
5920 /**
5921 * i40e_vsi_reconfig_tc - Reconfigure VSI Tx Scheduler for stored TC map
5922 * @vsi: VSI to be reconfigured
5923 *
5924 * This reconfigures a particular VSI for TCs that are mapped to the
5925 * TC bitmap stored previously for the VSI.
5926 *
5927 * Context: It is expected that the VSI queues have been quisced before
5928 * calling this function.
5929 *
5930 * Return: 0 on success, negative value on failure
5931 **/
i40e_vsi_reconfig_tc(struct i40e_vsi * vsi)5932 static int i40e_vsi_reconfig_tc(struct i40e_vsi *vsi)
5933 {
5934 u8 enabled_tc;
5935
5936 enabled_tc = vsi->tc_config.enabled_tc;
5937 vsi->tc_config.enabled_tc = 0;
5938
5939 return i40e_vsi_config_tc(vsi, enabled_tc);
5940 }
5941
5942 /**
5943 * i40e_get_link_speed - Returns link speed for the interface
5944 * @vsi: VSI to be configured
5945 *
5946 **/
i40e_get_link_speed(struct i40e_vsi * vsi)5947 static int i40e_get_link_speed(struct i40e_vsi *vsi)
5948 {
5949 struct i40e_pf *pf = vsi->back;
5950
5951 switch (pf->hw.phy.link_info.link_speed) {
5952 case I40E_LINK_SPEED_40GB:
5953 return 40000;
5954 case I40E_LINK_SPEED_25GB:
5955 return 25000;
5956 case I40E_LINK_SPEED_20GB:
5957 return 20000;
5958 case I40E_LINK_SPEED_10GB:
5959 return 10000;
5960 case I40E_LINK_SPEED_1GB:
5961 return 1000;
5962 default:
5963 return -EINVAL;
5964 }
5965 }
5966
5967 /**
5968 * i40e_bw_bytes_to_mbits - Convert max_tx_rate from bytes to mbits
5969 * @vsi: Pointer to vsi structure
5970 * @max_tx_rate: max TX rate in bytes to be converted into Mbits
5971 *
5972 * Helper function to convert units before send to set BW limit
5973 **/
i40e_bw_bytes_to_mbits(struct i40e_vsi * vsi,u64 max_tx_rate)5974 static u64 i40e_bw_bytes_to_mbits(struct i40e_vsi *vsi, u64 max_tx_rate)
5975 {
5976 if (max_tx_rate < I40E_BW_MBPS_DIVISOR) {
5977 dev_warn(&vsi->back->pdev->dev,
5978 "Setting max tx rate to minimum usable value of 50Mbps.\n");
5979 max_tx_rate = I40E_BW_CREDIT_DIVISOR;
5980 } else {
5981 do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR);
5982 }
5983
5984 return max_tx_rate;
5985 }
5986
5987 /**
5988 * i40e_set_bw_limit - setup BW limit for Tx traffic based on max_tx_rate
5989 * @vsi: VSI to be configured
5990 * @seid: seid of the channel/VSI
5991 * @max_tx_rate: max TX rate to be configured as BW limit
5992 *
5993 * Helper function to set BW limit for a given VSI
5994 **/
i40e_set_bw_limit(struct i40e_vsi * vsi,u16 seid,u64 max_tx_rate)5995 int i40e_set_bw_limit(struct i40e_vsi *vsi, u16 seid, u64 max_tx_rate)
5996 {
5997 struct i40e_pf *pf = vsi->back;
5998 u64 credits = 0;
5999 int speed = 0;
6000 int ret = 0;
6001
6002 speed = i40e_get_link_speed(vsi);
6003 if (max_tx_rate > speed) {
6004 dev_err(&pf->pdev->dev,
6005 "Invalid max tx rate %llu specified for VSI seid %d.",
6006 max_tx_rate, seid);
6007 return -EINVAL;
6008 }
6009 if (max_tx_rate && max_tx_rate < I40E_BW_CREDIT_DIVISOR) {
6010 dev_warn(&pf->pdev->dev,
6011 "Setting max tx rate to minimum usable value of 50Mbps.\n");
6012 max_tx_rate = I40E_BW_CREDIT_DIVISOR;
6013 }
6014
6015 /* Tx rate credits are in values of 50Mbps, 0 is disabled */
6016 credits = max_tx_rate;
6017 do_div(credits, I40E_BW_CREDIT_DIVISOR);
6018 ret = i40e_aq_config_vsi_bw_limit(&pf->hw, seid, credits,
6019 I40E_MAX_BW_INACTIVE_ACCUM, NULL);
6020 if (ret)
6021 dev_err(&pf->pdev->dev,
6022 "Failed set tx rate (%llu Mbps) for vsi->seid %u, err %pe aq_err %s\n",
6023 max_tx_rate, seid, ERR_PTR(ret),
6024 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6025 return ret;
6026 }
6027
6028 /**
6029 * i40e_remove_queue_channels - Remove queue channels for the TCs
6030 * @vsi: VSI to be configured
6031 *
6032 * Remove queue channels for the TCs
6033 **/
i40e_remove_queue_channels(struct i40e_vsi * vsi)6034 static void i40e_remove_queue_channels(struct i40e_vsi *vsi)
6035 {
6036 enum i40e_admin_queue_err last_aq_status;
6037 struct i40e_cloud_filter *cfilter;
6038 struct i40e_channel *ch, *ch_tmp;
6039 struct i40e_pf *pf = vsi->back;
6040 struct hlist_node *node;
6041 int ret, i;
6042
6043 /* Reset rss size that was stored when reconfiguring rss for
6044 * channel VSIs with non-power-of-2 queue count.
6045 */
6046 vsi->current_rss_size = 0;
6047
6048 /* perform cleanup for channels if they exist */
6049 if (list_empty(&vsi->ch_list))
6050 return;
6051
6052 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
6053 struct i40e_vsi *p_vsi;
6054
6055 list_del(&ch->list);
6056 p_vsi = ch->parent_vsi;
6057 if (!p_vsi || !ch->initialized) {
6058 kfree(ch);
6059 continue;
6060 }
6061 /* Reset queue contexts */
6062 for (i = 0; i < ch->num_queue_pairs; i++) {
6063 struct i40e_ring *tx_ring, *rx_ring;
6064 u16 pf_q;
6065
6066 pf_q = ch->base_queue + i;
6067 tx_ring = vsi->tx_rings[pf_q];
6068 tx_ring->ch = NULL;
6069
6070 rx_ring = vsi->rx_rings[pf_q];
6071 rx_ring->ch = NULL;
6072 }
6073
6074 /* Reset BW configured for this VSI via mqprio */
6075 ret = i40e_set_bw_limit(vsi, ch->seid, 0);
6076 if (ret)
6077 dev_info(&vsi->back->pdev->dev,
6078 "Failed to reset tx rate for ch->seid %u\n",
6079 ch->seid);
6080
6081 /* delete cloud filters associated with this channel */
6082 hlist_for_each_entry_safe(cfilter, node,
6083 &pf->cloud_filter_list, cloud_node) {
6084 if (cfilter->seid != ch->seid)
6085 continue;
6086
6087 hash_del(&cfilter->cloud_node);
6088 if (cfilter->dst_port)
6089 ret = i40e_add_del_cloud_filter_big_buf(vsi,
6090 cfilter,
6091 false);
6092 else
6093 ret = i40e_add_del_cloud_filter(vsi, cfilter,
6094 false);
6095 last_aq_status = pf->hw.aq.asq_last_status;
6096 if (ret)
6097 dev_info(&pf->pdev->dev,
6098 "Failed to delete cloud filter, err %pe aq_err %s\n",
6099 ERR_PTR(ret),
6100 i40e_aq_str(&pf->hw, last_aq_status));
6101 kfree(cfilter);
6102 }
6103
6104 /* delete VSI from FW */
6105 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid,
6106 NULL);
6107 if (ret)
6108 dev_err(&vsi->back->pdev->dev,
6109 "unable to remove channel (%d) for parent VSI(%d)\n",
6110 ch->seid, p_vsi->seid);
6111 kfree(ch);
6112 }
6113 INIT_LIST_HEAD(&vsi->ch_list);
6114 }
6115
6116 /**
6117 * i40e_get_max_queues_for_channel
6118 * @vsi: ptr to VSI to which channels are associated with
6119 *
6120 * Helper function which returns max value among the queue counts set on the
6121 * channels/TCs created.
6122 **/
i40e_get_max_queues_for_channel(struct i40e_vsi * vsi)6123 static int i40e_get_max_queues_for_channel(struct i40e_vsi *vsi)
6124 {
6125 struct i40e_channel *ch, *ch_tmp;
6126 int max = 0;
6127
6128 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
6129 if (!ch->initialized)
6130 continue;
6131 if (ch->num_queue_pairs > max)
6132 max = ch->num_queue_pairs;
6133 }
6134
6135 return max;
6136 }
6137
6138 /**
6139 * i40e_validate_num_queues - validate num_queues w.r.t channel
6140 * @pf: ptr to PF device
6141 * @num_queues: number of queues
6142 * @vsi: the parent VSI
6143 * @reconfig_rss: indicates should the RSS be reconfigured or not
6144 *
6145 * This function validates number of queues in the context of new channel
6146 * which is being established and determines if RSS should be reconfigured
6147 * or not for parent VSI.
6148 **/
i40e_validate_num_queues(struct i40e_pf * pf,int num_queues,struct i40e_vsi * vsi,bool * reconfig_rss)6149 static int i40e_validate_num_queues(struct i40e_pf *pf, int num_queues,
6150 struct i40e_vsi *vsi, bool *reconfig_rss)
6151 {
6152 int max_ch_queues;
6153
6154 if (!reconfig_rss)
6155 return -EINVAL;
6156
6157 *reconfig_rss = false;
6158 if (vsi->current_rss_size) {
6159 if (num_queues > vsi->current_rss_size) {
6160 dev_dbg(&pf->pdev->dev,
6161 "Error: num_queues (%d) > vsi's current_size(%d)\n",
6162 num_queues, vsi->current_rss_size);
6163 return -EINVAL;
6164 } else if ((num_queues < vsi->current_rss_size) &&
6165 (!is_power_of_2(num_queues))) {
6166 dev_dbg(&pf->pdev->dev,
6167 "Error: num_queues (%d) < vsi's current_size(%d), but not power of 2\n",
6168 num_queues, vsi->current_rss_size);
6169 return -EINVAL;
6170 }
6171 }
6172
6173 if (!is_power_of_2(num_queues)) {
6174 /* Find the max num_queues configured for channel if channel
6175 * exist.
6176 * if channel exist, then enforce 'num_queues' to be more than
6177 * max ever queues configured for channel.
6178 */
6179 max_ch_queues = i40e_get_max_queues_for_channel(vsi);
6180 if (num_queues < max_ch_queues) {
6181 dev_dbg(&pf->pdev->dev,
6182 "Error: num_queues (%d) < max queues configured for channel(%d)\n",
6183 num_queues, max_ch_queues);
6184 return -EINVAL;
6185 }
6186 *reconfig_rss = true;
6187 }
6188
6189 return 0;
6190 }
6191
6192 /**
6193 * i40e_vsi_reconfig_rss - reconfig RSS based on specified rss_size
6194 * @vsi: the VSI being setup
6195 * @rss_size: size of RSS, accordingly LUT gets reprogrammed
6196 *
6197 * This function reconfigures RSS by reprogramming LUTs using 'rss_size'
6198 **/
i40e_vsi_reconfig_rss(struct i40e_vsi * vsi,u16 rss_size)6199 static int i40e_vsi_reconfig_rss(struct i40e_vsi *vsi, u16 rss_size)
6200 {
6201 struct i40e_pf *pf = vsi->back;
6202 u8 seed[I40E_HKEY_ARRAY_SIZE];
6203 struct i40e_hw *hw = &pf->hw;
6204 int local_rss_size;
6205 u8 *lut;
6206 int ret;
6207
6208 if (!vsi->rss_size)
6209 return -EINVAL;
6210
6211 if (rss_size > vsi->rss_size)
6212 return -EINVAL;
6213
6214 local_rss_size = min_t(int, vsi->rss_size, rss_size);
6215 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
6216 if (!lut)
6217 return -ENOMEM;
6218
6219 /* Ignoring user configured lut if there is one */
6220 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, local_rss_size);
6221
6222 /* Use user configured hash key if there is one, otherwise
6223 * use default.
6224 */
6225 if (vsi->rss_hkey_user)
6226 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
6227 else
6228 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
6229
6230 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);
6231 if (ret) {
6232 dev_info(&pf->pdev->dev,
6233 "Cannot set RSS lut, err %pe aq_err %s\n",
6234 ERR_PTR(ret),
6235 i40e_aq_str(hw, hw->aq.asq_last_status));
6236 kfree(lut);
6237 return ret;
6238 }
6239 kfree(lut);
6240
6241 /* Do the update w.r.t. storing rss_size */
6242 if (!vsi->orig_rss_size)
6243 vsi->orig_rss_size = vsi->rss_size;
6244 vsi->current_rss_size = local_rss_size;
6245
6246 return ret;
6247 }
6248
6249 /**
6250 * i40e_channel_setup_queue_map - Setup a channel queue map
6251 * @pf: ptr to PF device
6252 * @ctxt: VSI context structure
6253 * @ch: ptr to channel structure
6254 *
6255 * Setup queue map for a specific channel
6256 **/
i40e_channel_setup_queue_map(struct i40e_pf * pf,struct i40e_vsi_context * ctxt,struct i40e_channel * ch)6257 static void i40e_channel_setup_queue_map(struct i40e_pf *pf,
6258 struct i40e_vsi_context *ctxt,
6259 struct i40e_channel *ch)
6260 {
6261 u16 qcount, qmap, sections = 0;
6262 u8 offset = 0;
6263 int pow;
6264
6265 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
6266 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
6267
6268 qcount = min_t(int, ch->num_queue_pairs, pf->num_lan_msix);
6269 ch->num_queue_pairs = qcount;
6270
6271 /* find the next higher power-of-2 of num queue pairs */
6272 pow = ilog2(qcount);
6273 if (!is_power_of_2(qcount))
6274 pow++;
6275
6276 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
6277 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
6278
6279 /* Setup queue TC[0].qmap for given VSI context */
6280 ctxt->info.tc_mapping[0] = cpu_to_le16(qmap);
6281
6282 ctxt->info.up_enable_bits = 0x1; /* TC0 enabled */
6283 ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
6284 ctxt->info.queue_mapping[0] = cpu_to_le16(ch->base_queue);
6285 ctxt->info.valid_sections |= cpu_to_le16(sections);
6286 }
6287
6288 /**
6289 * i40e_add_channel - add a channel by adding VSI
6290 * @pf: ptr to PF device
6291 * @uplink_seid: underlying HW switching element (VEB) ID
6292 * @ch: ptr to channel structure
6293 *
6294 * Add a channel (VSI) using add_vsi and queue_map
6295 **/
i40e_add_channel(struct i40e_pf * pf,u16 uplink_seid,struct i40e_channel * ch)6296 static int i40e_add_channel(struct i40e_pf *pf, u16 uplink_seid,
6297 struct i40e_channel *ch)
6298 {
6299 struct i40e_hw *hw = &pf->hw;
6300 struct i40e_vsi_context ctxt;
6301 u8 enabled_tc = 0x1; /* TC0 enabled */
6302 int ret;
6303
6304 if (ch->type != I40E_VSI_VMDQ2) {
6305 dev_info(&pf->pdev->dev,
6306 "add new vsi failed, ch->type %d\n", ch->type);
6307 return -EINVAL;
6308 }
6309
6310 memset(&ctxt, 0, sizeof(ctxt));
6311 ctxt.pf_num = hw->pf_id;
6312 ctxt.vf_num = 0;
6313 ctxt.uplink_seid = uplink_seid;
6314 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
6315 if (ch->type == I40E_VSI_VMDQ2)
6316 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
6317
6318 if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) {
6319 ctxt.info.valid_sections |=
6320 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
6321 ctxt.info.switch_id =
6322 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
6323 }
6324
6325 /* Set queue map for a given VSI context */
6326 i40e_channel_setup_queue_map(pf, &ctxt, ch);
6327
6328 /* Now time to create VSI */
6329 ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
6330 if (ret) {
6331 dev_info(&pf->pdev->dev,
6332 "add new vsi failed, err %pe aq_err %s\n",
6333 ERR_PTR(ret),
6334 i40e_aq_str(&pf->hw,
6335 pf->hw.aq.asq_last_status));
6336 return -ENOENT;
6337 }
6338
6339 /* Success, update channel, set enabled_tc only if the channel
6340 * is not a macvlan
6341 */
6342 ch->enabled_tc = !i40e_is_channel_macvlan(ch) && enabled_tc;
6343 ch->seid = ctxt.seid;
6344 ch->vsi_number = ctxt.vsi_number;
6345 ch->stat_counter_idx = le16_to_cpu(ctxt.info.stat_counter_idx);
6346
6347 /* copy just the sections touched not the entire info
6348 * since not all sections are valid as returned by
6349 * update vsi params
6350 */
6351 ch->info.mapping_flags = ctxt.info.mapping_flags;
6352 memcpy(&ch->info.queue_mapping,
6353 &ctxt.info.queue_mapping, sizeof(ctxt.info.queue_mapping));
6354 memcpy(&ch->info.tc_mapping, ctxt.info.tc_mapping,
6355 sizeof(ctxt.info.tc_mapping));
6356
6357 return 0;
6358 }
6359
i40e_channel_config_bw(struct i40e_vsi * vsi,struct i40e_channel * ch,u8 * bw_share)6360 static int i40e_channel_config_bw(struct i40e_vsi *vsi, struct i40e_channel *ch,
6361 u8 *bw_share)
6362 {
6363 struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
6364 int ret;
6365 int i;
6366
6367 memset(&bw_data, 0, sizeof(bw_data));
6368 bw_data.tc_valid_bits = ch->enabled_tc;
6369 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
6370 bw_data.tc_bw_credits[i] = bw_share[i];
6371
6372 ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, ch->seid,
6373 &bw_data, NULL);
6374 if (ret) {
6375 dev_info(&vsi->back->pdev->dev,
6376 "Config VSI BW allocation per TC failed, aq_err: %d for new_vsi->seid %u\n",
6377 vsi->back->hw.aq.asq_last_status, ch->seid);
6378 return -EINVAL;
6379 }
6380
6381 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
6382 ch->info.qs_handle[i] = bw_data.qs_handles[i];
6383
6384 return 0;
6385 }
6386
6387 /**
6388 * i40e_channel_config_tx_ring - config TX ring associated with new channel
6389 * @pf: ptr to PF device
6390 * @vsi: the VSI being setup
6391 * @ch: ptr to channel structure
6392 *
6393 * Configure TX rings associated with channel (VSI) since queues are being
6394 * from parent VSI.
6395 **/
i40e_channel_config_tx_ring(struct i40e_pf * pf,struct i40e_vsi * vsi,struct i40e_channel * ch)6396 static int i40e_channel_config_tx_ring(struct i40e_pf *pf,
6397 struct i40e_vsi *vsi,
6398 struct i40e_channel *ch)
6399 {
6400 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
6401 int ret;
6402 int i;
6403
6404 /* Enable ETS TCs with equal BW Share for now across all VSIs */
6405 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6406 if (ch->enabled_tc & BIT(i))
6407 bw_share[i] = 1;
6408 }
6409
6410 /* configure BW for new VSI */
6411 ret = i40e_channel_config_bw(vsi, ch, bw_share);
6412 if (ret) {
6413 dev_info(&vsi->back->pdev->dev,
6414 "Failed configuring TC map %d for channel (seid %u)\n",
6415 ch->enabled_tc, ch->seid);
6416 return ret;
6417 }
6418
6419 for (i = 0; i < ch->num_queue_pairs; i++) {
6420 struct i40e_ring *tx_ring, *rx_ring;
6421 u16 pf_q;
6422
6423 pf_q = ch->base_queue + i;
6424
6425 /* Get to TX ring ptr of main VSI, for re-setup TX queue
6426 * context
6427 */
6428 tx_ring = vsi->tx_rings[pf_q];
6429 tx_ring->ch = ch;
6430
6431 /* Get the RX ring ptr */
6432 rx_ring = vsi->rx_rings[pf_q];
6433 rx_ring->ch = ch;
6434 }
6435
6436 return 0;
6437 }
6438
6439 /**
6440 * i40e_setup_hw_channel - setup new channel
6441 * @pf: ptr to PF device
6442 * @vsi: the VSI being setup
6443 * @ch: ptr to channel structure
6444 * @uplink_seid: underlying HW switching element (VEB) ID
6445 * @type: type of channel to be created (VMDq2/VF)
6446 *
6447 * Setup new channel (VSI) based on specified type (VMDq2/VF)
6448 * and configures TX rings accordingly
6449 **/
i40e_setup_hw_channel(struct i40e_pf * pf,struct i40e_vsi * vsi,struct i40e_channel * ch,u16 uplink_seid,u8 type)6450 static inline int i40e_setup_hw_channel(struct i40e_pf *pf,
6451 struct i40e_vsi *vsi,
6452 struct i40e_channel *ch,
6453 u16 uplink_seid, u8 type)
6454 {
6455 int ret;
6456
6457 ch->initialized = false;
6458 ch->base_queue = vsi->next_base_queue;
6459 ch->type = type;
6460
6461 /* Proceed with creation of channel (VMDq2) VSI */
6462 ret = i40e_add_channel(pf, uplink_seid, ch);
6463 if (ret) {
6464 dev_info(&pf->pdev->dev,
6465 "failed to add_channel using uplink_seid %u\n",
6466 uplink_seid);
6467 return ret;
6468 }
6469
6470 /* Mark the successful creation of channel */
6471 ch->initialized = true;
6472
6473 /* Reconfigure TX queues using QTX_CTL register */
6474 ret = i40e_channel_config_tx_ring(pf, vsi, ch);
6475 if (ret) {
6476 dev_info(&pf->pdev->dev,
6477 "failed to configure TX rings for channel %u\n",
6478 ch->seid);
6479 return ret;
6480 }
6481
6482 /* update 'next_base_queue' */
6483 vsi->next_base_queue = vsi->next_base_queue + ch->num_queue_pairs;
6484 dev_dbg(&pf->pdev->dev,
6485 "Added channel: vsi_seid %u, vsi_number %u, stat_counter_idx %u, num_queue_pairs %u, pf->next_base_queue %d\n",
6486 ch->seid, ch->vsi_number, ch->stat_counter_idx,
6487 ch->num_queue_pairs,
6488 vsi->next_base_queue);
6489 return ret;
6490 }
6491
6492 /**
6493 * i40e_setup_channel - setup new channel using uplink element
6494 * @pf: ptr to PF device
6495 * @vsi: pointer to the VSI to set up the channel within
6496 * @ch: ptr to channel structure
6497 *
6498 * Setup new channel (VSI) based on specified type (VMDq2/VF)
6499 * and uplink switching element (uplink_seid)
6500 **/
i40e_setup_channel(struct i40e_pf * pf,struct i40e_vsi * vsi,struct i40e_channel * ch)6501 static bool i40e_setup_channel(struct i40e_pf *pf, struct i40e_vsi *vsi,
6502 struct i40e_channel *ch)
6503 {
6504 struct i40e_vsi *main_vsi;
6505 u8 vsi_type;
6506 u16 seid;
6507 int ret;
6508
6509 if (vsi->type == I40E_VSI_MAIN) {
6510 vsi_type = I40E_VSI_VMDQ2;
6511 } else {
6512 dev_err(&pf->pdev->dev, "unsupported parent vsi type(%d)\n",
6513 vsi->type);
6514 return false;
6515 }
6516
6517 /* underlying switching element */
6518 main_vsi = i40e_pf_get_main_vsi(pf);
6519 seid = main_vsi->uplink_seid;
6520
6521 /* create channel (VSI), configure TX rings */
6522 ret = i40e_setup_hw_channel(pf, vsi, ch, seid, vsi_type);
6523 if (ret) {
6524 dev_err(&pf->pdev->dev, "failed to setup hw_channel\n");
6525 return false;
6526 }
6527
6528 return ch->initialized ? true : false;
6529 }
6530
6531 /**
6532 * i40e_validate_and_set_switch_mode - sets up switch mode correctly
6533 * @vsi: ptr to VSI which has PF backing
6534 *
6535 * Sets up switch mode correctly if it needs to be changed and perform
6536 * what are allowed modes.
6537 **/
i40e_validate_and_set_switch_mode(struct i40e_vsi * vsi)6538 static int i40e_validate_and_set_switch_mode(struct i40e_vsi *vsi)
6539 {
6540 u8 mode;
6541 struct i40e_pf *pf = vsi->back;
6542 struct i40e_hw *hw = &pf->hw;
6543 int ret;
6544
6545 ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_dev_capabilities);
6546 if (ret)
6547 return -EINVAL;
6548
6549 if (hw->dev_caps.switch_mode) {
6550 /* if switch mode is set, support mode2 (non-tunneled for
6551 * cloud filter) for now
6552 */
6553 u32 switch_mode = hw->dev_caps.switch_mode &
6554 I40E_SWITCH_MODE_MASK;
6555 if (switch_mode >= I40E_CLOUD_FILTER_MODE1) {
6556 if (switch_mode == I40E_CLOUD_FILTER_MODE2)
6557 return 0;
6558 dev_err(&pf->pdev->dev,
6559 "Invalid switch_mode (%d), only non-tunneled mode for cloud filter is supported\n",
6560 hw->dev_caps.switch_mode);
6561 return -EINVAL;
6562 }
6563 }
6564
6565 /* Set Bit 7 to be valid */
6566 mode = I40E_AQ_SET_SWITCH_BIT7_VALID;
6567
6568 /* Set L4type for TCP support */
6569 mode |= I40E_AQ_SET_SWITCH_L4_TYPE_TCP;
6570
6571 /* Set cloud filter mode */
6572 mode |= I40E_AQ_SET_SWITCH_MODE_NON_TUNNEL;
6573
6574 /* Prep mode field for set_switch_config */
6575 ret = i40e_aq_set_switch_config(hw, pf->last_sw_conf_flags,
6576 pf->last_sw_conf_valid_flags,
6577 mode, NULL);
6578 if (ret && hw->aq.asq_last_status != I40E_AQ_RC_ESRCH)
6579 dev_err(&pf->pdev->dev,
6580 "couldn't set switch config bits, err %pe aq_err %s\n",
6581 ERR_PTR(ret),
6582 i40e_aq_str(hw,
6583 hw->aq.asq_last_status));
6584
6585 return ret;
6586 }
6587
6588 /**
6589 * i40e_create_queue_channel - function to create channel
6590 * @vsi: VSI to be configured
6591 * @ch: ptr to channel (it contains channel specific params)
6592 *
6593 * This function creates channel (VSI) using num_queues specified by user,
6594 * reconfigs RSS if needed.
6595 **/
i40e_create_queue_channel(struct i40e_vsi * vsi,struct i40e_channel * ch)6596 int i40e_create_queue_channel(struct i40e_vsi *vsi,
6597 struct i40e_channel *ch)
6598 {
6599 struct i40e_pf *pf = vsi->back;
6600 bool reconfig_rss;
6601 int err;
6602
6603 if (!ch)
6604 return -EINVAL;
6605
6606 if (!ch->num_queue_pairs) {
6607 dev_err(&pf->pdev->dev, "Invalid num_queues requested: %d\n",
6608 ch->num_queue_pairs);
6609 return -EINVAL;
6610 }
6611
6612 /* validate user requested num_queues for channel */
6613 err = i40e_validate_num_queues(pf, ch->num_queue_pairs, vsi,
6614 &reconfig_rss);
6615 if (err) {
6616 dev_info(&pf->pdev->dev, "Failed to validate num_queues (%d)\n",
6617 ch->num_queue_pairs);
6618 return -EINVAL;
6619 }
6620
6621 /* By default we are in VEPA mode, if this is the first VF/VMDq
6622 * VSI to be added switch to VEB mode.
6623 */
6624
6625 if (!test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) {
6626 set_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);
6627
6628 if (vsi->type == I40E_VSI_MAIN) {
6629 if (i40e_is_tc_mqprio_enabled(pf))
6630 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
6631 else
6632 i40e_do_reset_safe(pf, I40E_PF_RESET_FLAG);
6633 }
6634 /* now onwards for main VSI, number of queues will be value
6635 * of TC0's queue count
6636 */
6637 }
6638
6639 /* By this time, vsi->cnt_q_avail shall be set to non-zero and
6640 * it should be more than num_queues
6641 */
6642 if (!vsi->cnt_q_avail || vsi->cnt_q_avail < ch->num_queue_pairs) {
6643 dev_dbg(&pf->pdev->dev,
6644 "Error: cnt_q_avail (%u) less than num_queues %d\n",
6645 vsi->cnt_q_avail, ch->num_queue_pairs);
6646 return -EINVAL;
6647 }
6648
6649 /* reconfig_rss only if vsi type is MAIN_VSI */
6650 if (reconfig_rss && (vsi->type == I40E_VSI_MAIN)) {
6651 err = i40e_vsi_reconfig_rss(vsi, ch->num_queue_pairs);
6652 if (err) {
6653 dev_info(&pf->pdev->dev,
6654 "Error: unable to reconfig rss for num_queues (%u)\n",
6655 ch->num_queue_pairs);
6656 return -EINVAL;
6657 }
6658 }
6659
6660 if (!i40e_setup_channel(pf, vsi, ch)) {
6661 dev_info(&pf->pdev->dev, "Failed to setup channel\n");
6662 return -EINVAL;
6663 }
6664
6665 dev_info(&pf->pdev->dev,
6666 "Setup channel (id:%u) utilizing num_queues %d\n",
6667 ch->seid, ch->num_queue_pairs);
6668
6669 /* configure VSI for BW limit */
6670 if (ch->max_tx_rate) {
6671 u64 credits = ch->max_tx_rate;
6672
6673 if (i40e_set_bw_limit(vsi, ch->seid, ch->max_tx_rate))
6674 return -EINVAL;
6675
6676 do_div(credits, I40E_BW_CREDIT_DIVISOR);
6677 dev_dbg(&pf->pdev->dev,
6678 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
6679 ch->max_tx_rate,
6680 credits,
6681 ch->seid);
6682 }
6683
6684 /* in case of VF, this will be main SRIOV VSI */
6685 ch->parent_vsi = vsi;
6686
6687 /* and update main_vsi's count for queue_available to use */
6688 vsi->cnt_q_avail -= ch->num_queue_pairs;
6689
6690 return 0;
6691 }
6692
6693 /**
6694 * i40e_configure_queue_channels - Add queue channel for the given TCs
6695 * @vsi: VSI to be configured
6696 *
6697 * Configures queue channel mapping to the given TCs
6698 **/
i40e_configure_queue_channels(struct i40e_vsi * vsi)6699 static int i40e_configure_queue_channels(struct i40e_vsi *vsi)
6700 {
6701 struct i40e_channel *ch;
6702 u64 max_rate = 0;
6703 int ret = 0, i;
6704
6705 /* Create app vsi with the TCs. Main VSI with TC0 is already set up */
6706 vsi->tc_seid_map[0] = vsi->seid;
6707 for (i = 1; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6708 if (vsi->tc_config.enabled_tc & BIT(i)) {
6709 ch = kzalloc(sizeof(*ch), GFP_KERNEL);
6710 if (!ch) {
6711 ret = -ENOMEM;
6712 goto err_free;
6713 }
6714
6715 INIT_LIST_HEAD(&ch->list);
6716 ch->num_queue_pairs =
6717 vsi->tc_config.tc_info[i].qcount;
6718 ch->base_queue =
6719 vsi->tc_config.tc_info[i].qoffset;
6720
6721 /* Bandwidth limit through tc interface is in bytes/s,
6722 * change to Mbit/s
6723 */
6724 max_rate = vsi->mqprio_qopt.max_rate[i];
6725 do_div(max_rate, I40E_BW_MBPS_DIVISOR);
6726 ch->max_tx_rate = max_rate;
6727
6728 list_add_tail(&ch->list, &vsi->ch_list);
6729
6730 ret = i40e_create_queue_channel(vsi, ch);
6731 if (ret) {
6732 dev_err(&vsi->back->pdev->dev,
6733 "Failed creating queue channel with TC%d: queues %d\n",
6734 i, ch->num_queue_pairs);
6735 goto err_free;
6736 }
6737 vsi->tc_seid_map[i] = ch->seid;
6738 }
6739 }
6740
6741 /* reset to reconfigure TX queue contexts */
6742 i40e_do_reset(vsi->back, I40E_PF_RESET_FLAG, true);
6743 return ret;
6744
6745 err_free:
6746 i40e_remove_queue_channels(vsi);
6747 return ret;
6748 }
6749
6750 /**
6751 * i40e_veb_config_tc - Configure TCs for given VEB
6752 * @veb: given VEB
6753 * @enabled_tc: TC bitmap
6754 *
6755 * Configures given TC bitmap for VEB (switching) element
6756 **/
i40e_veb_config_tc(struct i40e_veb * veb,u8 enabled_tc)6757 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc)
6758 {
6759 struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0};
6760 struct i40e_pf *pf = veb->pf;
6761 int ret = 0;
6762 int i;
6763
6764 /* No TCs or already enabled TCs just return */
6765 if (!enabled_tc || veb->enabled_tc == enabled_tc)
6766 return ret;
6767
6768 bw_data.tc_valid_bits = enabled_tc;
6769 /* bw_data.absolute_credits is not set (relative) */
6770
6771 /* Enable ETS TCs with equal BW Share for now */
6772 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6773 if (enabled_tc & BIT(i))
6774 bw_data.tc_bw_share_credits[i] = 1;
6775 }
6776
6777 ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid,
6778 &bw_data, NULL);
6779 if (ret) {
6780 dev_info(&pf->pdev->dev,
6781 "VEB bw config failed, err %pe aq_err %s\n",
6782 ERR_PTR(ret),
6783 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6784 goto out;
6785 }
6786
6787 /* Update the BW information */
6788 ret = i40e_veb_get_bw_info(veb);
6789 if (ret) {
6790 dev_info(&pf->pdev->dev,
6791 "Failed getting veb bw config, err %pe aq_err %s\n",
6792 ERR_PTR(ret),
6793 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6794 }
6795
6796 out:
6797 return ret;
6798 }
6799
6800 #ifdef CONFIG_I40E_DCB
6801 /**
6802 * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs
6803 * @pf: PF struct
6804 *
6805 * Reconfigure VEB/VSIs on a given PF; it is assumed that
6806 * the caller would've quiesce all the VSIs before calling
6807 * this function
6808 **/
i40e_dcb_reconfigure(struct i40e_pf * pf)6809 static void i40e_dcb_reconfigure(struct i40e_pf *pf)
6810 {
6811 struct i40e_vsi *vsi;
6812 struct i40e_veb *veb;
6813 u8 tc_map = 0;
6814 int ret;
6815 int v;
6816
6817 /* Enable the TCs available on PF to all VEBs */
6818 tc_map = i40e_pf_get_tc_map(pf);
6819 if (tc_map == I40E_DEFAULT_TRAFFIC_CLASS)
6820 return;
6821
6822 i40e_pf_for_each_veb(pf, v, veb) {
6823 ret = i40e_veb_config_tc(veb, tc_map);
6824 if (ret) {
6825 dev_info(&pf->pdev->dev,
6826 "Failed configuring TC for VEB seid=%d\n",
6827 veb->seid);
6828 /* Will try to configure as many components */
6829 }
6830 }
6831
6832 /* Update each VSI */
6833 i40e_pf_for_each_vsi(pf, v, vsi) {
6834 /* - Enable all TCs for the LAN VSI
6835 * - For all others keep them at TC0 for now
6836 */
6837 if (vsi->type == I40E_VSI_MAIN)
6838 tc_map = i40e_pf_get_tc_map(pf);
6839 else
6840 tc_map = I40E_DEFAULT_TRAFFIC_CLASS;
6841
6842 ret = i40e_vsi_config_tc(vsi, tc_map);
6843 if (ret) {
6844 dev_info(&pf->pdev->dev,
6845 "Failed configuring TC for VSI seid=%d\n",
6846 vsi->seid);
6847 /* Will try to configure as many components */
6848 } else {
6849 /* Re-configure VSI vectors based on updated TC map */
6850 i40e_vsi_map_rings_to_vectors(vsi);
6851 if (vsi->netdev)
6852 i40e_dcbnl_set_all(vsi);
6853 }
6854 }
6855 }
6856
6857 /**
6858 * i40e_resume_port_tx - Resume port Tx
6859 * @pf: PF struct
6860 *
6861 * Resume a port's Tx and issue a PF reset in case of failure to
6862 * resume.
6863 **/
i40e_resume_port_tx(struct i40e_pf * pf)6864 static int i40e_resume_port_tx(struct i40e_pf *pf)
6865 {
6866 struct i40e_hw *hw = &pf->hw;
6867 int ret;
6868
6869 ret = i40e_aq_resume_port_tx(hw, NULL);
6870 if (ret) {
6871 dev_info(&pf->pdev->dev,
6872 "Resume Port Tx failed, err %pe aq_err %s\n",
6873 ERR_PTR(ret),
6874 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6875 /* Schedule PF reset to recover */
6876 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
6877 i40e_service_event_schedule(pf);
6878 }
6879
6880 return ret;
6881 }
6882
6883 /**
6884 * i40e_suspend_port_tx - Suspend port Tx
6885 * @pf: PF struct
6886 *
6887 * Suspend a port's Tx and issue a PF reset in case of failure.
6888 **/
i40e_suspend_port_tx(struct i40e_pf * pf)6889 static int i40e_suspend_port_tx(struct i40e_pf *pf)
6890 {
6891 struct i40e_hw *hw = &pf->hw;
6892 int ret;
6893
6894 ret = i40e_aq_suspend_port_tx(hw, pf->mac_seid, NULL);
6895 if (ret) {
6896 dev_info(&pf->pdev->dev,
6897 "Suspend Port Tx failed, err %pe aq_err %s\n",
6898 ERR_PTR(ret),
6899 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6900 /* Schedule PF reset to recover */
6901 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
6902 i40e_service_event_schedule(pf);
6903 }
6904
6905 return ret;
6906 }
6907
6908 /**
6909 * i40e_hw_set_dcb_config - Program new DCBX settings into HW
6910 * @pf: PF being configured
6911 * @new_cfg: New DCBX configuration
6912 *
6913 * Program DCB settings into HW and reconfigure VEB/VSIs on
6914 * given PF. Uses "Set LLDP MIB" AQC to program the hardware.
6915 **/
i40e_hw_set_dcb_config(struct i40e_pf * pf,struct i40e_dcbx_config * new_cfg)6916 static int i40e_hw_set_dcb_config(struct i40e_pf *pf,
6917 struct i40e_dcbx_config *new_cfg)
6918 {
6919 struct i40e_dcbx_config *old_cfg = &pf->hw.local_dcbx_config;
6920 int ret;
6921
6922 /* Check if need reconfiguration */
6923 if (!memcmp(&new_cfg, &old_cfg, sizeof(new_cfg))) {
6924 dev_dbg(&pf->pdev->dev, "No Change in DCB Config required.\n");
6925 return 0;
6926 }
6927
6928 /* Config change disable all VSIs */
6929 i40e_pf_quiesce_all_vsi(pf);
6930
6931 /* Copy the new config to the current config */
6932 *old_cfg = *new_cfg;
6933 old_cfg->etsrec = old_cfg->etscfg;
6934 ret = i40e_set_dcb_config(&pf->hw);
6935 if (ret) {
6936 dev_info(&pf->pdev->dev,
6937 "Set DCB Config failed, err %pe aq_err %s\n",
6938 ERR_PTR(ret),
6939 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6940 goto out;
6941 }
6942
6943 /* Changes in configuration update VEB/VSI */
6944 i40e_dcb_reconfigure(pf);
6945 out:
6946 /* In case of reset do not try to resume anything */
6947 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) {
6948 /* Re-start the VSIs if disabled */
6949 ret = i40e_resume_port_tx(pf);
6950 /* In case of error no point in resuming VSIs */
6951 if (ret)
6952 goto err;
6953 i40e_pf_unquiesce_all_vsi(pf);
6954 }
6955 err:
6956 return ret;
6957 }
6958
6959 /**
6960 * i40e_hw_dcb_config - Program new DCBX settings into HW
6961 * @pf: PF being configured
6962 * @new_cfg: New DCBX configuration
6963 *
6964 * Program DCB settings into HW and reconfigure VEB/VSIs on
6965 * given PF
6966 **/
i40e_hw_dcb_config(struct i40e_pf * pf,struct i40e_dcbx_config * new_cfg)6967 int i40e_hw_dcb_config(struct i40e_pf *pf, struct i40e_dcbx_config *new_cfg)
6968 {
6969 struct i40e_aqc_configure_switching_comp_ets_data ets_data;
6970 u8 prio_type[I40E_MAX_TRAFFIC_CLASS] = {0};
6971 u32 mfs_tc[I40E_MAX_TRAFFIC_CLASS];
6972 struct i40e_dcbx_config *old_cfg;
6973 u8 mode[I40E_MAX_TRAFFIC_CLASS];
6974 struct i40e_rx_pb_config pb_cfg;
6975 struct i40e_hw *hw = &pf->hw;
6976 u8 num_ports = hw->num_ports;
6977 bool need_reconfig;
6978 int ret = -EINVAL;
6979 u8 lltc_map = 0;
6980 u8 tc_map = 0;
6981 u8 new_numtc;
6982 u8 i;
6983
6984 dev_dbg(&pf->pdev->dev, "Configuring DCB registers directly\n");
6985 /* Un-pack information to Program ETS HW via shared API
6986 * numtc, tcmap
6987 * LLTC map
6988 * ETS/NON-ETS arbiter mode
6989 * max exponent (credit refills)
6990 * Total number of ports
6991 * PFC priority bit-map
6992 * Priority Table
6993 * BW % per TC
6994 * Arbiter mode between UPs sharing same TC
6995 * TSA table (ETS or non-ETS)
6996 * EEE enabled or not
6997 * MFS TC table
6998 */
6999
7000 new_numtc = i40e_dcb_get_num_tc(new_cfg);
7001
7002 memset(&ets_data, 0, sizeof(ets_data));
7003 for (i = 0; i < new_numtc; i++) {
7004 tc_map |= BIT(i);
7005 switch (new_cfg->etscfg.tsatable[i]) {
7006 case I40E_IEEE_TSA_ETS:
7007 prio_type[i] = I40E_DCB_PRIO_TYPE_ETS;
7008 ets_data.tc_bw_share_credits[i] =
7009 new_cfg->etscfg.tcbwtable[i];
7010 break;
7011 case I40E_IEEE_TSA_STRICT:
7012 prio_type[i] = I40E_DCB_PRIO_TYPE_STRICT;
7013 lltc_map |= BIT(i);
7014 ets_data.tc_bw_share_credits[i] =
7015 I40E_DCB_STRICT_PRIO_CREDITS;
7016 break;
7017 default:
7018 /* Invalid TSA type */
7019 need_reconfig = false;
7020 goto out;
7021 }
7022 }
7023
7024 old_cfg = &hw->local_dcbx_config;
7025 /* Check if need reconfiguration */
7026 need_reconfig = i40e_dcb_need_reconfig(pf, old_cfg, new_cfg);
7027
7028 /* If needed, enable/disable frame tagging, disable all VSIs
7029 * and suspend port tx
7030 */
7031 if (need_reconfig) {
7032 /* Enable DCB tagging only when more than one TC */
7033 if (new_numtc > 1)
7034 set_bit(I40E_FLAG_DCB_ENA, pf->flags);
7035 else
7036 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
7037
7038 set_bit(__I40E_PORT_SUSPENDED, pf->state);
7039 /* Reconfiguration needed quiesce all VSIs */
7040 i40e_pf_quiesce_all_vsi(pf);
7041 ret = i40e_suspend_port_tx(pf);
7042 if (ret)
7043 goto err;
7044 }
7045
7046 /* Configure Port ETS Tx Scheduler */
7047 ets_data.tc_valid_bits = tc_map;
7048 ets_data.tc_strict_priority_flags = lltc_map;
7049 ret = i40e_aq_config_switch_comp_ets
7050 (hw, pf->mac_seid, &ets_data,
7051 i40e_aqc_opc_modify_switching_comp_ets, NULL);
7052 if (ret) {
7053 dev_info(&pf->pdev->dev,
7054 "Modify Port ETS failed, err %pe aq_err %s\n",
7055 ERR_PTR(ret),
7056 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7057 goto out;
7058 }
7059
7060 /* Configure Rx ETS HW */
7061 memset(&mode, I40E_DCB_ARB_MODE_ROUND_ROBIN, sizeof(mode));
7062 i40e_dcb_hw_set_num_tc(hw, new_numtc);
7063 i40e_dcb_hw_rx_fifo_config(hw, I40E_DCB_ARB_MODE_ROUND_ROBIN,
7064 I40E_DCB_ARB_MODE_STRICT_PRIORITY,
7065 I40E_DCB_DEFAULT_MAX_EXPONENT,
7066 lltc_map);
7067 i40e_dcb_hw_rx_cmd_monitor_config(hw, new_numtc, num_ports);
7068 i40e_dcb_hw_rx_ets_bw_config(hw, new_cfg->etscfg.tcbwtable, mode,
7069 prio_type);
7070 i40e_dcb_hw_pfc_config(hw, new_cfg->pfc.pfcenable,
7071 new_cfg->etscfg.prioritytable);
7072 i40e_dcb_hw_rx_up2tc_config(hw, new_cfg->etscfg.prioritytable);
7073
7074 /* Configure Rx Packet Buffers in HW */
7075 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
7076 struct i40e_vsi *main_vsi = i40e_pf_get_main_vsi(pf);
7077
7078 mfs_tc[i] = main_vsi->netdev->mtu;
7079 mfs_tc[i] += I40E_PACKET_HDR_PAD;
7080 }
7081
7082 i40e_dcb_hw_calculate_pool_sizes(hw, num_ports,
7083 false, new_cfg->pfc.pfcenable,
7084 mfs_tc, &pb_cfg);
7085 i40e_dcb_hw_rx_pb_config(hw, &pf->pb_cfg, &pb_cfg);
7086
7087 /* Update the local Rx Packet buffer config */
7088 pf->pb_cfg = pb_cfg;
7089
7090 /* Inform the FW about changes to DCB configuration */
7091 ret = i40e_aq_dcb_updated(&pf->hw, NULL);
7092 if (ret) {
7093 dev_info(&pf->pdev->dev,
7094 "DCB Updated failed, err %pe aq_err %s\n",
7095 ERR_PTR(ret),
7096 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7097 goto out;
7098 }
7099
7100 /* Update the port DCBx configuration */
7101 *old_cfg = *new_cfg;
7102
7103 /* Changes in configuration update VEB/VSI */
7104 i40e_dcb_reconfigure(pf);
7105 out:
7106 /* Re-start the VSIs if disabled */
7107 if (need_reconfig) {
7108 ret = i40e_resume_port_tx(pf);
7109
7110 clear_bit(__I40E_PORT_SUSPENDED, pf->state);
7111 /* In case of error no point in resuming VSIs */
7112 if (ret)
7113 goto err;
7114
7115 /* Wait for the PF's queues to be disabled */
7116 ret = i40e_pf_wait_queues_disabled(pf);
7117 if (ret) {
7118 /* Schedule PF reset to recover */
7119 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
7120 i40e_service_event_schedule(pf);
7121 goto err;
7122 } else {
7123 i40e_pf_unquiesce_all_vsi(pf);
7124 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
7125 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);
7126 }
7127 /* registers are set, lets apply */
7128 if (test_bit(I40E_HW_CAP_USE_SET_LLDP_MIB, pf->hw.caps))
7129 ret = i40e_hw_set_dcb_config(pf, new_cfg);
7130 }
7131
7132 err:
7133 return ret;
7134 }
7135
7136 /**
7137 * i40e_dcb_sw_default_config - Set default DCB configuration when DCB in SW
7138 * @pf: PF being queried
7139 *
7140 * Set default DCB configuration in case DCB is to be done in SW.
7141 **/
i40e_dcb_sw_default_config(struct i40e_pf * pf)7142 int i40e_dcb_sw_default_config(struct i40e_pf *pf)
7143 {
7144 struct i40e_dcbx_config *dcb_cfg = &pf->hw.local_dcbx_config;
7145 struct i40e_aqc_configure_switching_comp_ets_data ets_data;
7146 struct i40e_hw *hw = &pf->hw;
7147 int err;
7148
7149 if (test_bit(I40E_HW_CAP_USE_SET_LLDP_MIB, pf->hw.caps)) {
7150 /* Update the local cached instance with TC0 ETS */
7151 memset(&pf->tmp_cfg, 0, sizeof(struct i40e_dcbx_config));
7152 pf->tmp_cfg.etscfg.willing = I40E_IEEE_DEFAULT_ETS_WILLING;
7153 pf->tmp_cfg.etscfg.maxtcs = 0;
7154 pf->tmp_cfg.etscfg.tcbwtable[0] = I40E_IEEE_DEFAULT_ETS_TCBW;
7155 pf->tmp_cfg.etscfg.tsatable[0] = I40E_IEEE_TSA_ETS;
7156 pf->tmp_cfg.pfc.willing = I40E_IEEE_DEFAULT_PFC_WILLING;
7157 pf->tmp_cfg.pfc.pfccap = I40E_MAX_TRAFFIC_CLASS;
7158 /* FW needs one App to configure HW */
7159 pf->tmp_cfg.numapps = I40E_IEEE_DEFAULT_NUM_APPS;
7160 pf->tmp_cfg.app[0].selector = I40E_APP_SEL_ETHTYPE;
7161 pf->tmp_cfg.app[0].priority = I40E_IEEE_DEFAULT_APP_PRIO;
7162 pf->tmp_cfg.app[0].protocolid = I40E_APP_PROTOID_FCOE;
7163
7164 return i40e_hw_set_dcb_config(pf, &pf->tmp_cfg);
7165 }
7166
7167 memset(&ets_data, 0, sizeof(ets_data));
7168 ets_data.tc_valid_bits = I40E_DEFAULT_TRAFFIC_CLASS; /* TC0 only */
7169 ets_data.tc_strict_priority_flags = 0; /* ETS */
7170 ets_data.tc_bw_share_credits[0] = I40E_IEEE_DEFAULT_ETS_TCBW; /* 100% to TC0 */
7171
7172 /* Enable ETS on the Physical port */
7173 err = i40e_aq_config_switch_comp_ets
7174 (hw, pf->mac_seid, &ets_data,
7175 i40e_aqc_opc_enable_switching_comp_ets, NULL);
7176 if (err) {
7177 dev_info(&pf->pdev->dev,
7178 "Enable Port ETS failed, err %pe aq_err %s\n",
7179 ERR_PTR(err),
7180 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7181 err = -ENOENT;
7182 goto out;
7183 }
7184
7185 /* Update the local cached instance with TC0 ETS */
7186 dcb_cfg->etscfg.willing = I40E_IEEE_DEFAULT_ETS_WILLING;
7187 dcb_cfg->etscfg.cbs = 0;
7188 dcb_cfg->etscfg.maxtcs = I40E_MAX_TRAFFIC_CLASS;
7189 dcb_cfg->etscfg.tcbwtable[0] = I40E_IEEE_DEFAULT_ETS_TCBW;
7190
7191 out:
7192 return err;
7193 }
7194
7195 /**
7196 * i40e_init_pf_dcb - Initialize DCB configuration
7197 * @pf: PF being configured
7198 *
7199 * Query the current DCB configuration and cache it
7200 * in the hardware structure
7201 **/
i40e_init_pf_dcb(struct i40e_pf * pf)7202 static int i40e_init_pf_dcb(struct i40e_pf *pf)
7203 {
7204 struct i40e_hw *hw = &pf->hw;
7205 int err;
7206
7207 /* Do not enable DCB for SW1 and SW2 images even if the FW is capable
7208 * Also do not enable DCBx if FW LLDP agent is disabled
7209 */
7210 if (test_bit(I40E_HW_CAP_NO_DCB_SUPPORT, pf->hw.caps)) {
7211 dev_info(&pf->pdev->dev, "DCB is not supported.\n");
7212 err = -EOPNOTSUPP;
7213 goto out;
7214 }
7215 if (test_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags)) {
7216 dev_info(&pf->pdev->dev, "FW LLDP is disabled, attempting SW DCB\n");
7217 err = i40e_dcb_sw_default_config(pf);
7218 if (err) {
7219 dev_info(&pf->pdev->dev, "Could not initialize SW DCB\n");
7220 goto out;
7221 }
7222 dev_info(&pf->pdev->dev, "SW DCB initialization succeeded.\n");
7223 pf->dcbx_cap = DCB_CAP_DCBX_HOST |
7224 DCB_CAP_DCBX_VER_IEEE;
7225 /* at init capable but disabled */
7226 set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
7227 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
7228 goto out;
7229 }
7230 err = i40e_init_dcb(hw, true);
7231 if (!err) {
7232 /* Device/Function is not DCBX capable */
7233 if ((!hw->func_caps.dcb) ||
7234 (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) {
7235 dev_info(&pf->pdev->dev,
7236 "DCBX offload is not supported or is disabled for this PF.\n");
7237 } else {
7238 /* When status is not DISABLED then DCBX in FW */
7239 pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED |
7240 DCB_CAP_DCBX_VER_IEEE;
7241
7242 set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
7243 /* Enable DCB tagging only when more than one TC
7244 * or explicitly disable if only one TC
7245 */
7246 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
7247 set_bit(I40E_FLAG_DCB_ENA, pf->flags);
7248 else
7249 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
7250 dev_dbg(&pf->pdev->dev,
7251 "DCBX offload is supported for this PF.\n");
7252 }
7253 } else if (pf->hw.aq.asq_last_status == I40E_AQ_RC_EPERM) {
7254 dev_info(&pf->pdev->dev, "FW LLDP disabled for this PF.\n");
7255 set_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags);
7256 } else {
7257 dev_info(&pf->pdev->dev,
7258 "Query for DCB configuration failed, err %pe aq_err %s\n",
7259 ERR_PTR(err),
7260 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7261 }
7262
7263 out:
7264 return err;
7265 }
7266 #endif /* CONFIG_I40E_DCB */
7267
7268 /**
7269 * i40e_print_link_message - print link up or down
7270 * @vsi: the VSI for which link needs a message
7271 * @isup: true of link is up, false otherwise
7272 */
i40e_print_link_message(struct i40e_vsi * vsi,bool isup)7273 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup)
7274 {
7275 enum i40e_aq_link_speed new_speed;
7276 struct i40e_pf *pf = vsi->back;
7277 char *speed = "Unknown";
7278 char *fc = "Unknown";
7279 char *fec = "";
7280 char *req_fec = "";
7281 char *an = "";
7282
7283 if (isup)
7284 new_speed = pf->hw.phy.link_info.link_speed;
7285 else
7286 new_speed = I40E_LINK_SPEED_UNKNOWN;
7287
7288 if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed))
7289 return;
7290 vsi->current_isup = isup;
7291 vsi->current_speed = new_speed;
7292 if (!isup) {
7293 netdev_info(vsi->netdev, "NIC Link is Down\n");
7294 return;
7295 }
7296
7297 /* Warn user if link speed on NPAR enabled partition is not at
7298 * least 10GB
7299 */
7300 if (pf->hw.func_caps.npar_enable &&
7301 (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB ||
7302 pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB))
7303 netdev_warn(vsi->netdev,
7304 "The partition detected link speed that is less than 10Gbps\n");
7305
7306 switch (pf->hw.phy.link_info.link_speed) {
7307 case I40E_LINK_SPEED_40GB:
7308 speed = "40 G";
7309 break;
7310 case I40E_LINK_SPEED_20GB:
7311 speed = "20 G";
7312 break;
7313 case I40E_LINK_SPEED_25GB:
7314 speed = "25 G";
7315 break;
7316 case I40E_LINK_SPEED_10GB:
7317 speed = "10 G";
7318 break;
7319 case I40E_LINK_SPEED_5GB:
7320 speed = "5 G";
7321 break;
7322 case I40E_LINK_SPEED_2_5GB:
7323 speed = "2.5 G";
7324 break;
7325 case I40E_LINK_SPEED_1GB:
7326 speed = "1000 M";
7327 break;
7328 case I40E_LINK_SPEED_100MB:
7329 speed = "100 M";
7330 break;
7331 default:
7332 break;
7333 }
7334
7335 switch (pf->hw.fc.current_mode) {
7336 case I40E_FC_FULL:
7337 fc = "RX/TX";
7338 break;
7339 case I40E_FC_TX_PAUSE:
7340 fc = "TX";
7341 break;
7342 case I40E_FC_RX_PAUSE:
7343 fc = "RX";
7344 break;
7345 default:
7346 fc = "None";
7347 break;
7348 }
7349
7350 if (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) {
7351 req_fec = "None";
7352 fec = "None";
7353 an = "False";
7354
7355 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED)
7356 an = "True";
7357
7358 if (pf->hw.phy.link_info.fec_info &
7359 I40E_AQ_CONFIG_FEC_KR_ENA)
7360 fec = "CL74 FC-FEC/BASE-R";
7361 else if (pf->hw.phy.link_info.fec_info &
7362 I40E_AQ_CONFIG_FEC_RS_ENA)
7363 fec = "CL108 RS-FEC";
7364
7365 /* 'CL108 RS-FEC' should be displayed when RS is requested, or
7366 * both RS and FC are requested
7367 */
7368 if (vsi->back->hw.phy.link_info.req_fec_info &
7369 (I40E_AQ_REQUEST_FEC_KR | I40E_AQ_REQUEST_FEC_RS)) {
7370 if (vsi->back->hw.phy.link_info.req_fec_info &
7371 I40E_AQ_REQUEST_FEC_RS)
7372 req_fec = "CL108 RS-FEC";
7373 else
7374 req_fec = "CL74 FC-FEC/BASE-R";
7375 }
7376 netdev_info(vsi->netdev,
7377 "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n",
7378 speed, req_fec, fec, an, fc);
7379 } else if (pf->hw.device_id == I40E_DEV_ID_KX_X722) {
7380 req_fec = "None";
7381 fec = "None";
7382 an = "False";
7383
7384 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED)
7385 an = "True";
7386
7387 if (pf->hw.phy.link_info.fec_info &
7388 I40E_AQ_CONFIG_FEC_KR_ENA)
7389 fec = "CL74 FC-FEC/BASE-R";
7390
7391 if (pf->hw.phy.link_info.req_fec_info &
7392 I40E_AQ_REQUEST_FEC_KR)
7393 req_fec = "CL74 FC-FEC/BASE-R";
7394
7395 netdev_info(vsi->netdev,
7396 "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n",
7397 speed, req_fec, fec, an, fc);
7398 } else {
7399 netdev_info(vsi->netdev,
7400 "NIC Link is Up, %sbps Full Duplex, Flow Control: %s\n",
7401 speed, fc);
7402 }
7403
7404 }
7405
7406 /**
7407 * i40e_up_complete - Finish the last steps of bringing up a connection
7408 * @vsi: the VSI being configured
7409 **/
i40e_up_complete(struct i40e_vsi * vsi)7410 static int i40e_up_complete(struct i40e_vsi *vsi)
7411 {
7412 struct i40e_pf *pf = vsi->back;
7413 int err;
7414
7415 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
7416 i40e_vsi_configure_msix(vsi);
7417 else
7418 i40e_configure_msi_and_legacy(vsi);
7419
7420 /* start rings */
7421 err = i40e_vsi_start_rings(vsi);
7422 if (err)
7423 return err;
7424
7425 clear_bit(__I40E_VSI_DOWN, vsi->state);
7426 i40e_napi_enable_all(vsi);
7427 i40e_vsi_enable_irq(vsi);
7428
7429 if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) &&
7430 (vsi->netdev)) {
7431 i40e_print_link_message(vsi, true);
7432 netif_tx_start_all_queues(vsi->netdev);
7433 netif_carrier_on(vsi->netdev);
7434 }
7435
7436 /* replay FDIR SB filters */
7437 if (vsi->type == I40E_VSI_FDIR) {
7438 /* reset fd counters */
7439 pf->fd_add_err = 0;
7440 pf->fd_atr_cnt = 0;
7441 i40e_fdir_filter_restore(vsi);
7442 }
7443
7444 /* On the next run of the service_task, notify any clients of the new
7445 * opened netdev
7446 */
7447 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
7448 i40e_service_event_schedule(pf);
7449
7450 return 0;
7451 }
7452
7453 /**
7454 * i40e_vsi_reinit_locked - Reset the VSI
7455 * @vsi: the VSI being configured
7456 *
7457 * Rebuild the ring structs after some configuration
7458 * has changed, e.g. MTU size.
7459 **/
i40e_vsi_reinit_locked(struct i40e_vsi * vsi)7460 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi)
7461 {
7462 struct i40e_pf *pf = vsi->back;
7463
7464 while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state))
7465 usleep_range(1000, 2000);
7466 i40e_down(vsi);
7467
7468 i40e_up(vsi);
7469 clear_bit(__I40E_CONFIG_BUSY, pf->state);
7470 }
7471
7472 /**
7473 * i40e_force_link_state - Force the link status
7474 * @pf: board private structure
7475 * @is_up: whether the link state should be forced up or down
7476 **/
i40e_force_link_state(struct i40e_pf * pf,bool is_up)7477 static int i40e_force_link_state(struct i40e_pf *pf, bool is_up)
7478 {
7479 struct i40e_aq_get_phy_abilities_resp abilities;
7480 struct i40e_aq_set_phy_config config = {0};
7481 bool non_zero_phy_type = is_up;
7482 struct i40e_hw *hw = &pf->hw;
7483 u64 mask;
7484 u8 speed;
7485 int err;
7486
7487 /* Card might've been put in an unstable state by other drivers
7488 * and applications, which causes incorrect speed values being
7489 * set on startup. In order to clear speed registers, we call
7490 * get_phy_capabilities twice, once to get initial state of
7491 * available speeds, and once to get current PHY config.
7492 */
7493 err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities,
7494 NULL);
7495 if (err) {
7496 dev_err(&pf->pdev->dev,
7497 "failed to get phy cap., ret = %pe last_status = %s\n",
7498 ERR_PTR(err),
7499 i40e_aq_str(hw, hw->aq.asq_last_status));
7500 return err;
7501 }
7502 speed = abilities.link_speed;
7503
7504 /* Get the current phy config */
7505 err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities,
7506 NULL);
7507 if (err) {
7508 dev_err(&pf->pdev->dev,
7509 "failed to get phy cap., ret = %pe last_status = %s\n",
7510 ERR_PTR(err),
7511 i40e_aq_str(hw, hw->aq.asq_last_status));
7512 return err;
7513 }
7514
7515 /* If link needs to go up, but was not forced to go down,
7516 * and its speed values are OK, no need for a flap
7517 * if non_zero_phy_type was set, still need to force up
7518 */
7519 if (test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags))
7520 non_zero_phy_type = true;
7521 else if (is_up && abilities.phy_type != 0 && abilities.link_speed != 0)
7522 return 0;
7523
7524 /* To force link we need to set bits for all supported PHY types,
7525 * but there are now more than 32, so we need to split the bitmap
7526 * across two fields.
7527 */
7528 mask = I40E_PHY_TYPES_BITMASK;
7529 config.phy_type =
7530 non_zero_phy_type ? cpu_to_le32((u32)(mask & 0xffffffff)) : 0;
7531 config.phy_type_ext =
7532 non_zero_phy_type ? (u8)((mask >> 32) & 0xff) : 0;
7533 /* Copy the old settings, except of phy_type */
7534 config.abilities = abilities.abilities;
7535 if (test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags)) {
7536 if (is_up)
7537 config.abilities |= I40E_AQ_PHY_ENABLE_LINK;
7538 else
7539 config.abilities &= ~(I40E_AQ_PHY_ENABLE_LINK);
7540 }
7541 if (abilities.link_speed != 0)
7542 config.link_speed = abilities.link_speed;
7543 else
7544 config.link_speed = speed;
7545 config.eee_capability = abilities.eee_capability;
7546 config.eeer = abilities.eeer_val;
7547 config.low_power_ctrl = abilities.d3_lpan;
7548 config.fec_config = abilities.fec_cfg_curr_mod_ext_info &
7549 I40E_AQ_PHY_FEC_CONFIG_MASK;
7550 err = i40e_aq_set_phy_config(hw, &config, NULL);
7551
7552 if (err) {
7553 dev_err(&pf->pdev->dev,
7554 "set phy config ret = %pe last_status = %s\n",
7555 ERR_PTR(err),
7556 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7557 return err;
7558 }
7559
7560 /* Update the link info */
7561 err = i40e_update_link_info(hw);
7562 if (err) {
7563 /* Wait a little bit (on 40G cards it sometimes takes a really
7564 * long time for link to come back from the atomic reset)
7565 * and try once more
7566 */
7567 msleep(1000);
7568 i40e_update_link_info(hw);
7569 }
7570
7571 i40e_aq_set_link_restart_an(hw, is_up, NULL);
7572
7573 return 0;
7574 }
7575
7576 /**
7577 * i40e_up - Bring the connection back up after being down
7578 * @vsi: the VSI being configured
7579 **/
i40e_up(struct i40e_vsi * vsi)7580 int i40e_up(struct i40e_vsi *vsi)
7581 {
7582 int err;
7583
7584 if (vsi->type == I40E_VSI_MAIN &&
7585 (test_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags) ||
7586 test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, vsi->back->flags)))
7587 i40e_force_link_state(vsi->back, true);
7588
7589 err = i40e_vsi_configure(vsi);
7590 if (!err)
7591 err = i40e_up_complete(vsi);
7592
7593 return err;
7594 }
7595
7596 /**
7597 * i40e_down - Shutdown the connection processing
7598 * @vsi: the VSI being stopped
7599 **/
i40e_down(struct i40e_vsi * vsi)7600 void i40e_down(struct i40e_vsi *vsi)
7601 {
7602 int i;
7603
7604 /* It is assumed that the caller of this function
7605 * sets the vsi->state __I40E_VSI_DOWN bit.
7606 */
7607 if (vsi->netdev) {
7608 netif_carrier_off(vsi->netdev);
7609 netif_tx_disable(vsi->netdev);
7610 }
7611 i40e_vsi_disable_irq(vsi);
7612 i40e_vsi_stop_rings(vsi);
7613 if (vsi->type == I40E_VSI_MAIN &&
7614 (test_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags) ||
7615 test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, vsi->back->flags)))
7616 i40e_force_link_state(vsi->back, false);
7617 i40e_napi_disable_all(vsi);
7618
7619 for (i = 0; i < vsi->num_queue_pairs; i++) {
7620 i40e_clean_tx_ring(vsi->tx_rings[i]);
7621 if (i40e_enabled_xdp_vsi(vsi)) {
7622 /* Make sure that in-progress ndo_xdp_xmit and
7623 * ndo_xsk_wakeup calls are completed.
7624 */
7625 synchronize_rcu();
7626 i40e_clean_tx_ring(vsi->xdp_rings[i]);
7627 }
7628 i40e_clean_rx_ring(vsi->rx_rings[i]);
7629 }
7630
7631 }
7632
7633 /**
7634 * i40e_validate_mqprio_qopt- validate queue mapping info
7635 * @vsi: the VSI being configured
7636 * @mqprio_qopt: queue parametrs
7637 **/
i40e_validate_mqprio_qopt(struct i40e_vsi * vsi,struct tc_mqprio_qopt_offload * mqprio_qopt)7638 static int i40e_validate_mqprio_qopt(struct i40e_vsi *vsi,
7639 struct tc_mqprio_qopt_offload *mqprio_qopt)
7640 {
7641 u64 sum_max_rate = 0;
7642 u64 max_rate = 0;
7643 int i;
7644
7645 if (mqprio_qopt->qopt.offset[0] != 0 ||
7646 mqprio_qopt->qopt.num_tc < 1 ||
7647 mqprio_qopt->qopt.num_tc > I40E_MAX_TRAFFIC_CLASS)
7648 return -EINVAL;
7649 for (i = 0; ; i++) {
7650 if (!mqprio_qopt->qopt.count[i])
7651 return -EINVAL;
7652 if (mqprio_qopt->min_rate[i]) {
7653 dev_err(&vsi->back->pdev->dev,
7654 "Invalid min tx rate (greater than 0) specified\n");
7655 return -EINVAL;
7656 }
7657 max_rate = mqprio_qopt->max_rate[i];
7658 do_div(max_rate, I40E_BW_MBPS_DIVISOR);
7659 sum_max_rate += max_rate;
7660
7661 if (i >= mqprio_qopt->qopt.num_tc - 1)
7662 break;
7663 if (mqprio_qopt->qopt.offset[i + 1] !=
7664 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i]))
7665 return -EINVAL;
7666 }
7667 if (vsi->num_queue_pairs <
7668 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) {
7669 dev_err(&vsi->back->pdev->dev,
7670 "Failed to create traffic channel, insufficient number of queues.\n");
7671 return -EINVAL;
7672 }
7673 if (sum_max_rate > i40e_get_link_speed(vsi)) {
7674 dev_err(&vsi->back->pdev->dev,
7675 "Invalid max tx rate specified\n");
7676 return -EINVAL;
7677 }
7678 return 0;
7679 }
7680
7681 /**
7682 * i40e_vsi_set_default_tc_config - set default values for tc configuration
7683 * @vsi: the VSI being configured
7684 **/
i40e_vsi_set_default_tc_config(struct i40e_vsi * vsi)7685 static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi)
7686 {
7687 u16 qcount;
7688 int i;
7689
7690 /* Only TC0 is enabled */
7691 vsi->tc_config.numtc = 1;
7692 vsi->tc_config.enabled_tc = 1;
7693 qcount = min_t(int, vsi->alloc_queue_pairs,
7694 i40e_pf_get_max_q_per_tc(vsi->back));
7695 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
7696 /* For the TC that is not enabled set the offset to default
7697 * queue and allocate one queue for the given TC.
7698 */
7699 vsi->tc_config.tc_info[i].qoffset = 0;
7700 if (i == 0)
7701 vsi->tc_config.tc_info[i].qcount = qcount;
7702 else
7703 vsi->tc_config.tc_info[i].qcount = 1;
7704 vsi->tc_config.tc_info[i].netdev_tc = 0;
7705 }
7706 }
7707
7708 /**
7709 * i40e_del_macvlan_filter
7710 * @hw: pointer to the HW structure
7711 * @seid: seid of the channel VSI
7712 * @macaddr: the mac address to apply as a filter
7713 * @aq_err: store the admin Q error
7714 *
7715 * This function deletes a mac filter on the channel VSI which serves as the
7716 * macvlan. Returns 0 on success.
7717 **/
i40e_del_macvlan_filter(struct i40e_hw * hw,u16 seid,const u8 * macaddr,int * aq_err)7718 static int i40e_del_macvlan_filter(struct i40e_hw *hw, u16 seid,
7719 const u8 *macaddr, int *aq_err)
7720 {
7721 struct i40e_aqc_remove_macvlan_element_data element;
7722 int status;
7723
7724 memset(&element, 0, sizeof(element));
7725 ether_addr_copy(element.mac_addr, macaddr);
7726 element.vlan_tag = 0;
7727 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
7728 status = i40e_aq_remove_macvlan(hw, seid, &element, 1, NULL);
7729 *aq_err = hw->aq.asq_last_status;
7730
7731 return status;
7732 }
7733
7734 /**
7735 * i40e_add_macvlan_filter
7736 * @hw: pointer to the HW structure
7737 * @seid: seid of the channel VSI
7738 * @macaddr: the mac address to apply as a filter
7739 * @aq_err: store the admin Q error
7740 *
7741 * This function adds a mac filter on the channel VSI which serves as the
7742 * macvlan. Returns 0 on success.
7743 **/
i40e_add_macvlan_filter(struct i40e_hw * hw,u16 seid,const u8 * macaddr,int * aq_err)7744 static int i40e_add_macvlan_filter(struct i40e_hw *hw, u16 seid,
7745 const u8 *macaddr, int *aq_err)
7746 {
7747 struct i40e_aqc_add_macvlan_element_data element;
7748 u16 cmd_flags = 0;
7749 int status;
7750
7751 ether_addr_copy(element.mac_addr, macaddr);
7752 element.vlan_tag = 0;
7753 element.queue_number = 0;
7754 element.match_method = I40E_AQC_MM_ERR_NO_RES;
7755 cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
7756 element.flags = cpu_to_le16(cmd_flags);
7757 status = i40e_aq_add_macvlan(hw, seid, &element, 1, NULL);
7758 *aq_err = hw->aq.asq_last_status;
7759
7760 return status;
7761 }
7762
7763 /**
7764 * i40e_reset_ch_rings - Reset the queue contexts in a channel
7765 * @vsi: the VSI we want to access
7766 * @ch: the channel we want to access
7767 */
i40e_reset_ch_rings(struct i40e_vsi * vsi,struct i40e_channel * ch)7768 static void i40e_reset_ch_rings(struct i40e_vsi *vsi, struct i40e_channel *ch)
7769 {
7770 struct i40e_ring *tx_ring, *rx_ring;
7771 u16 pf_q;
7772 int i;
7773
7774 for (i = 0; i < ch->num_queue_pairs; i++) {
7775 pf_q = ch->base_queue + i;
7776 tx_ring = vsi->tx_rings[pf_q];
7777 tx_ring->ch = NULL;
7778 rx_ring = vsi->rx_rings[pf_q];
7779 rx_ring->ch = NULL;
7780 }
7781 }
7782
7783 /**
7784 * i40e_free_macvlan_channels
7785 * @vsi: the VSI we want to access
7786 *
7787 * This function frees the Qs of the channel VSI from
7788 * the stack and also deletes the channel VSIs which
7789 * serve as macvlans.
7790 */
i40e_free_macvlan_channels(struct i40e_vsi * vsi)7791 static void i40e_free_macvlan_channels(struct i40e_vsi *vsi)
7792 {
7793 struct i40e_channel *ch, *ch_tmp;
7794 int ret;
7795
7796 if (list_empty(&vsi->macvlan_list))
7797 return;
7798
7799 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
7800 struct i40e_vsi *parent_vsi;
7801
7802 if (i40e_is_channel_macvlan(ch)) {
7803 i40e_reset_ch_rings(vsi, ch);
7804 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
7805 netdev_unbind_sb_channel(vsi->netdev, ch->fwd->netdev);
7806 netdev_set_sb_channel(ch->fwd->netdev, 0);
7807 kfree(ch->fwd);
7808 ch->fwd = NULL;
7809 }
7810
7811 list_del(&ch->list);
7812 parent_vsi = ch->parent_vsi;
7813 if (!parent_vsi || !ch->initialized) {
7814 kfree(ch);
7815 continue;
7816 }
7817
7818 /* remove the VSI */
7819 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid,
7820 NULL);
7821 if (ret)
7822 dev_err(&vsi->back->pdev->dev,
7823 "unable to remove channel (%d) for parent VSI(%d)\n",
7824 ch->seid, parent_vsi->seid);
7825 kfree(ch);
7826 }
7827 vsi->macvlan_cnt = 0;
7828 }
7829
7830 /**
7831 * i40e_fwd_ring_up - bring the macvlan device up
7832 * @vsi: the VSI we want to access
7833 * @vdev: macvlan netdevice
7834 * @fwd: the private fwd structure
7835 */
i40e_fwd_ring_up(struct i40e_vsi * vsi,struct net_device * vdev,struct i40e_fwd_adapter * fwd)7836 static int i40e_fwd_ring_up(struct i40e_vsi *vsi, struct net_device *vdev,
7837 struct i40e_fwd_adapter *fwd)
7838 {
7839 struct i40e_channel *ch = NULL, *ch_tmp, *iter;
7840 int ret = 0, num_tc = 1, i, aq_err;
7841 struct i40e_pf *pf = vsi->back;
7842 struct i40e_hw *hw = &pf->hw;
7843
7844 /* Go through the list and find an available channel */
7845 list_for_each_entry_safe(iter, ch_tmp, &vsi->macvlan_list, list) {
7846 if (!i40e_is_channel_macvlan(iter)) {
7847 iter->fwd = fwd;
7848 /* record configuration for macvlan interface in vdev */
7849 for (i = 0; i < num_tc; i++)
7850 netdev_bind_sb_channel_queue(vsi->netdev, vdev,
7851 i,
7852 iter->num_queue_pairs,
7853 iter->base_queue);
7854 for (i = 0; i < iter->num_queue_pairs; i++) {
7855 struct i40e_ring *tx_ring, *rx_ring;
7856 u16 pf_q;
7857
7858 pf_q = iter->base_queue + i;
7859
7860 /* Get to TX ring ptr */
7861 tx_ring = vsi->tx_rings[pf_q];
7862 tx_ring->ch = iter;
7863
7864 /* Get the RX ring ptr */
7865 rx_ring = vsi->rx_rings[pf_q];
7866 rx_ring->ch = iter;
7867 }
7868 ch = iter;
7869 break;
7870 }
7871 }
7872
7873 if (!ch)
7874 return -EINVAL;
7875
7876 /* Guarantee all rings are updated before we update the
7877 * MAC address filter.
7878 */
7879 wmb();
7880
7881 /* Add a mac filter */
7882 ret = i40e_add_macvlan_filter(hw, ch->seid, vdev->dev_addr, &aq_err);
7883 if (ret) {
7884 /* if we cannot add the MAC rule then disable the offload */
7885 macvlan_release_l2fw_offload(vdev);
7886 for (i = 0; i < ch->num_queue_pairs; i++) {
7887 struct i40e_ring *rx_ring;
7888 u16 pf_q;
7889
7890 pf_q = ch->base_queue + i;
7891 rx_ring = vsi->rx_rings[pf_q];
7892 rx_ring->netdev = NULL;
7893 }
7894 dev_info(&pf->pdev->dev,
7895 "Error adding mac filter on macvlan err %pe, aq_err %s\n",
7896 ERR_PTR(ret),
7897 i40e_aq_str(hw, aq_err));
7898 netdev_err(vdev, "L2fwd offload disabled to L2 filter error\n");
7899 }
7900
7901 return ret;
7902 }
7903
7904 /**
7905 * i40e_setup_macvlans - create the channels which will be macvlans
7906 * @vsi: the VSI we want to access
7907 * @macvlan_cnt: no. of macvlans to be setup
7908 * @qcnt: no. of Qs per macvlan
7909 * @vdev: macvlan netdevice
7910 */
i40e_setup_macvlans(struct i40e_vsi * vsi,u16 macvlan_cnt,u16 qcnt,struct net_device * vdev)7911 static int i40e_setup_macvlans(struct i40e_vsi *vsi, u16 macvlan_cnt, u16 qcnt,
7912 struct net_device *vdev)
7913 {
7914 struct i40e_pf *pf = vsi->back;
7915 struct i40e_hw *hw = &pf->hw;
7916 struct i40e_vsi_context ctxt;
7917 u16 sections, qmap, num_qps;
7918 struct i40e_channel *ch;
7919 int i, pow, ret = 0;
7920 u8 offset = 0;
7921
7922 if (vsi->type != I40E_VSI_MAIN || !macvlan_cnt)
7923 return -EINVAL;
7924
7925 num_qps = vsi->num_queue_pairs - (macvlan_cnt * qcnt);
7926
7927 /* find the next higher power-of-2 of num queue pairs */
7928 pow = fls(roundup_pow_of_two(num_qps) - 1);
7929
7930 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
7931 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
7932
7933 /* Setup context bits for the main VSI */
7934 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
7935 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
7936 memset(&ctxt, 0, sizeof(ctxt));
7937 ctxt.seid = vsi->seid;
7938 ctxt.pf_num = vsi->back->hw.pf_id;
7939 ctxt.vf_num = 0;
7940 ctxt.uplink_seid = vsi->uplink_seid;
7941 ctxt.info = vsi->info;
7942 ctxt.info.tc_mapping[0] = cpu_to_le16(qmap);
7943 ctxt.info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
7944 ctxt.info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
7945 ctxt.info.valid_sections |= cpu_to_le16(sections);
7946
7947 /* Reconfigure RSS for main VSI with new max queue count */
7948 vsi->rss_size = max_t(u16, num_qps, qcnt);
7949 ret = i40e_vsi_config_rss(vsi);
7950 if (ret) {
7951 dev_info(&pf->pdev->dev,
7952 "Failed to reconfig RSS for num_queues (%u)\n",
7953 vsi->rss_size);
7954 return ret;
7955 }
7956 vsi->reconfig_rss = true;
7957 dev_dbg(&vsi->back->pdev->dev,
7958 "Reconfigured RSS with num_queues (%u)\n", vsi->rss_size);
7959 vsi->next_base_queue = num_qps;
7960 vsi->cnt_q_avail = vsi->num_queue_pairs - num_qps;
7961
7962 /* Update the VSI after updating the VSI queue-mapping
7963 * information
7964 */
7965 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
7966 if (ret) {
7967 dev_info(&pf->pdev->dev,
7968 "Update vsi tc config failed, err %pe aq_err %s\n",
7969 ERR_PTR(ret),
7970 i40e_aq_str(hw, hw->aq.asq_last_status));
7971 return ret;
7972 }
7973 /* update the local VSI info with updated queue map */
7974 i40e_vsi_update_queue_map(vsi, &ctxt);
7975 vsi->info.valid_sections = 0;
7976
7977 /* Create channels for macvlans */
7978 INIT_LIST_HEAD(&vsi->macvlan_list);
7979 for (i = 0; i < macvlan_cnt; i++) {
7980 ch = kzalloc(sizeof(*ch), GFP_KERNEL);
7981 if (!ch) {
7982 ret = -ENOMEM;
7983 goto err_free;
7984 }
7985 INIT_LIST_HEAD(&ch->list);
7986 ch->num_queue_pairs = qcnt;
7987 if (!i40e_setup_channel(pf, vsi, ch)) {
7988 ret = -EINVAL;
7989 kfree(ch);
7990 goto err_free;
7991 }
7992 ch->parent_vsi = vsi;
7993 vsi->cnt_q_avail -= ch->num_queue_pairs;
7994 vsi->macvlan_cnt++;
7995 list_add_tail(&ch->list, &vsi->macvlan_list);
7996 }
7997
7998 return ret;
7999
8000 err_free:
8001 dev_info(&pf->pdev->dev, "Failed to setup macvlans\n");
8002 i40e_free_macvlan_channels(vsi);
8003
8004 return ret;
8005 }
8006
8007 /**
8008 * i40e_fwd_add - configure macvlans
8009 * @netdev: net device to configure
8010 * @vdev: macvlan netdevice
8011 **/
i40e_fwd_add(struct net_device * netdev,struct net_device * vdev)8012 static void *i40e_fwd_add(struct net_device *netdev, struct net_device *vdev)
8013 {
8014 struct i40e_netdev_priv *np = netdev_priv(netdev);
8015 u16 q_per_macvlan = 0, macvlan_cnt = 0, vectors;
8016 struct i40e_vsi *vsi = np->vsi;
8017 struct i40e_pf *pf = vsi->back;
8018 struct i40e_fwd_adapter *fwd;
8019 int avail_macvlan, ret;
8020
8021 if (test_bit(I40E_FLAG_DCB_ENA, pf->flags)) {
8022 netdev_info(netdev, "Macvlans are not supported when DCB is enabled\n");
8023 return ERR_PTR(-EINVAL);
8024 }
8025 if (i40e_is_tc_mqprio_enabled(pf)) {
8026 netdev_info(netdev, "Macvlans are not supported when HW TC offload is on\n");
8027 return ERR_PTR(-EINVAL);
8028 }
8029 if (pf->num_lan_msix < I40E_MIN_MACVLAN_VECTORS) {
8030 netdev_info(netdev, "Not enough vectors available to support macvlans\n");
8031 return ERR_PTR(-EINVAL);
8032 }
8033
8034 /* The macvlan device has to be a single Q device so that the
8035 * tc_to_txq field can be reused to pick the tx queue.
8036 */
8037 if (netif_is_multiqueue(vdev))
8038 return ERR_PTR(-ERANGE);
8039
8040 if (!vsi->macvlan_cnt) {
8041 /* reserve bit 0 for the pf device */
8042 set_bit(0, vsi->fwd_bitmask);
8043
8044 /* Try to reserve as many queues as possible for macvlans. First
8045 * reserve 3/4th of max vectors, then half, then quarter and
8046 * calculate Qs per macvlan as you go
8047 */
8048 vectors = pf->num_lan_msix;
8049 if (vectors <= I40E_MAX_MACVLANS && vectors > 64) {
8050 /* allocate 4 Qs per macvlan and 32 Qs to the PF*/
8051 q_per_macvlan = 4;
8052 macvlan_cnt = (vectors - 32) / 4;
8053 } else if (vectors <= 64 && vectors > 32) {
8054 /* allocate 2 Qs per macvlan and 16 Qs to the PF*/
8055 q_per_macvlan = 2;
8056 macvlan_cnt = (vectors - 16) / 2;
8057 } else if (vectors <= 32 && vectors > 16) {
8058 /* allocate 1 Q per macvlan and 16 Qs to the PF*/
8059 q_per_macvlan = 1;
8060 macvlan_cnt = vectors - 16;
8061 } else if (vectors <= 16 && vectors > 8) {
8062 /* allocate 1 Q per macvlan and 8 Qs to the PF */
8063 q_per_macvlan = 1;
8064 macvlan_cnt = vectors - 8;
8065 } else {
8066 /* allocate 1 Q per macvlan and 1 Q to the PF */
8067 q_per_macvlan = 1;
8068 macvlan_cnt = vectors - 1;
8069 }
8070
8071 if (macvlan_cnt == 0)
8072 return ERR_PTR(-EBUSY);
8073
8074 /* Quiesce VSI queues */
8075 i40e_quiesce_vsi(vsi);
8076
8077 /* sets up the macvlans but does not "enable" them */
8078 ret = i40e_setup_macvlans(vsi, macvlan_cnt, q_per_macvlan,
8079 vdev);
8080 if (ret)
8081 return ERR_PTR(ret);
8082
8083 /* Unquiesce VSI */
8084 i40e_unquiesce_vsi(vsi);
8085 }
8086 avail_macvlan = find_first_zero_bit(vsi->fwd_bitmask,
8087 vsi->macvlan_cnt);
8088 if (avail_macvlan >= I40E_MAX_MACVLANS)
8089 return ERR_PTR(-EBUSY);
8090
8091 /* create the fwd struct */
8092 fwd = kzalloc(sizeof(*fwd), GFP_KERNEL);
8093 if (!fwd)
8094 return ERR_PTR(-ENOMEM);
8095
8096 set_bit(avail_macvlan, vsi->fwd_bitmask);
8097 fwd->bit_no = avail_macvlan;
8098 netdev_set_sb_channel(vdev, avail_macvlan);
8099 fwd->netdev = vdev;
8100
8101 if (!netif_running(netdev))
8102 return fwd;
8103
8104 /* Set fwd ring up */
8105 ret = i40e_fwd_ring_up(vsi, vdev, fwd);
8106 if (ret) {
8107 /* unbind the queues and drop the subordinate channel config */
8108 netdev_unbind_sb_channel(netdev, vdev);
8109 netdev_set_sb_channel(vdev, 0);
8110
8111 kfree(fwd);
8112 return ERR_PTR(-EINVAL);
8113 }
8114
8115 return fwd;
8116 }
8117
8118 /**
8119 * i40e_del_all_macvlans - Delete all the mac filters on the channels
8120 * @vsi: the VSI we want to access
8121 */
i40e_del_all_macvlans(struct i40e_vsi * vsi)8122 static void i40e_del_all_macvlans(struct i40e_vsi *vsi)
8123 {
8124 struct i40e_channel *ch, *ch_tmp;
8125 struct i40e_pf *pf = vsi->back;
8126 struct i40e_hw *hw = &pf->hw;
8127 int aq_err, ret = 0;
8128
8129 if (list_empty(&vsi->macvlan_list))
8130 return;
8131
8132 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
8133 if (i40e_is_channel_macvlan(ch)) {
8134 ret = i40e_del_macvlan_filter(hw, ch->seid,
8135 i40e_channel_mac(ch),
8136 &aq_err);
8137 if (!ret) {
8138 /* Reset queue contexts */
8139 i40e_reset_ch_rings(vsi, ch);
8140 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
8141 netdev_unbind_sb_channel(vsi->netdev,
8142 ch->fwd->netdev);
8143 netdev_set_sb_channel(ch->fwd->netdev, 0);
8144 kfree(ch->fwd);
8145 ch->fwd = NULL;
8146 }
8147 }
8148 }
8149 }
8150
8151 /**
8152 * i40e_fwd_del - delete macvlan interfaces
8153 * @netdev: net device to configure
8154 * @vdev: macvlan netdevice
8155 */
i40e_fwd_del(struct net_device * netdev,void * vdev)8156 static void i40e_fwd_del(struct net_device *netdev, void *vdev)
8157 {
8158 struct i40e_netdev_priv *np = netdev_priv(netdev);
8159 struct i40e_fwd_adapter *fwd = vdev;
8160 struct i40e_channel *ch, *ch_tmp;
8161 struct i40e_vsi *vsi = np->vsi;
8162 struct i40e_pf *pf = vsi->back;
8163 struct i40e_hw *hw = &pf->hw;
8164 int aq_err, ret = 0;
8165
8166 /* Find the channel associated with the macvlan and del mac filter */
8167 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
8168 if (i40e_is_channel_macvlan(ch) &&
8169 ether_addr_equal(i40e_channel_mac(ch),
8170 fwd->netdev->dev_addr)) {
8171 ret = i40e_del_macvlan_filter(hw, ch->seid,
8172 i40e_channel_mac(ch),
8173 &aq_err);
8174 if (!ret) {
8175 /* Reset queue contexts */
8176 i40e_reset_ch_rings(vsi, ch);
8177 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
8178 netdev_unbind_sb_channel(netdev, fwd->netdev);
8179 netdev_set_sb_channel(fwd->netdev, 0);
8180 kfree(ch->fwd);
8181 ch->fwd = NULL;
8182 } else {
8183 dev_info(&pf->pdev->dev,
8184 "Error deleting mac filter on macvlan err %pe, aq_err %s\n",
8185 ERR_PTR(ret),
8186 i40e_aq_str(hw, aq_err));
8187 }
8188 break;
8189 }
8190 }
8191 }
8192
8193 /**
8194 * i40e_setup_tc - configure multiple traffic classes
8195 * @netdev: net device to configure
8196 * @type_data: tc offload data
8197 **/
i40e_setup_tc(struct net_device * netdev,void * type_data)8198 static int i40e_setup_tc(struct net_device *netdev, void *type_data)
8199 {
8200 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
8201 struct i40e_netdev_priv *np = netdev_priv(netdev);
8202 struct i40e_vsi *vsi = np->vsi;
8203 struct i40e_pf *pf = vsi->back;
8204 u8 enabled_tc = 0, num_tc, hw;
8205 bool need_reset = false;
8206 int old_queue_pairs;
8207 int ret = -EINVAL;
8208 u16 mode;
8209 int i;
8210
8211 old_queue_pairs = vsi->num_queue_pairs;
8212 num_tc = mqprio_qopt->qopt.num_tc;
8213 hw = mqprio_qopt->qopt.hw;
8214 mode = mqprio_qopt->mode;
8215 if (!hw) {
8216 clear_bit(I40E_FLAG_TC_MQPRIO_ENA, pf->flags);
8217 memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt));
8218 goto config_tc;
8219 }
8220
8221 /* Check if MFP enabled */
8222 if (test_bit(I40E_FLAG_MFP_ENA, pf->flags)) {
8223 netdev_info(netdev,
8224 "Configuring TC not supported in MFP mode\n");
8225 return ret;
8226 }
8227 switch (mode) {
8228 case TC_MQPRIO_MODE_DCB:
8229 clear_bit(I40E_FLAG_TC_MQPRIO_ENA, pf->flags);
8230
8231 /* Check if DCB enabled to continue */
8232 if (!test_bit(I40E_FLAG_DCB_ENA, pf->flags)) {
8233 netdev_info(netdev,
8234 "DCB is not enabled for adapter\n");
8235 return ret;
8236 }
8237
8238 /* Check whether tc count is within enabled limit */
8239 if (num_tc > i40e_pf_get_num_tc(pf)) {
8240 netdev_info(netdev,
8241 "TC count greater than enabled on link for adapter\n");
8242 return ret;
8243 }
8244 break;
8245 case TC_MQPRIO_MODE_CHANNEL:
8246 if (test_bit(I40E_FLAG_DCB_ENA, pf->flags)) {
8247 netdev_info(netdev,
8248 "Full offload of TC Mqprio options is not supported when DCB is enabled\n");
8249 return ret;
8250 }
8251 if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
8252 return ret;
8253 ret = i40e_validate_mqprio_qopt(vsi, mqprio_qopt);
8254 if (ret)
8255 return ret;
8256 memcpy(&vsi->mqprio_qopt, mqprio_qopt,
8257 sizeof(*mqprio_qopt));
8258 set_bit(I40E_FLAG_TC_MQPRIO_ENA, pf->flags);
8259 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
8260 break;
8261 default:
8262 return -EINVAL;
8263 }
8264
8265 config_tc:
8266 /* Generate TC map for number of tc requested */
8267 for (i = 0; i < num_tc; i++)
8268 enabled_tc |= BIT(i);
8269
8270 /* Requesting same TC configuration as already enabled */
8271 if (enabled_tc == vsi->tc_config.enabled_tc &&
8272 mode != TC_MQPRIO_MODE_CHANNEL)
8273 return 0;
8274
8275 /* Quiesce VSI queues */
8276 i40e_quiesce_vsi(vsi);
8277
8278 if (!hw && !i40e_is_tc_mqprio_enabled(pf))
8279 i40e_remove_queue_channels(vsi);
8280
8281 /* Configure VSI for enabled TCs */
8282 ret = i40e_vsi_config_tc(vsi, enabled_tc);
8283 if (ret) {
8284 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n",
8285 vsi->seid);
8286 need_reset = true;
8287 goto exit;
8288 } else if (enabled_tc &&
8289 (!is_power_of_2(vsi->tc_config.tc_info[0].qcount))) {
8290 netdev_info(netdev,
8291 "Failed to create channel. Override queues (%u) not power of 2\n",
8292 vsi->tc_config.tc_info[0].qcount);
8293 ret = -EINVAL;
8294 need_reset = true;
8295 goto exit;
8296 }
8297
8298 dev_info(&vsi->back->pdev->dev,
8299 "Setup channel (id:%u) utilizing num_queues %d\n",
8300 vsi->seid, vsi->tc_config.tc_info[0].qcount);
8301
8302 if (i40e_is_tc_mqprio_enabled(pf)) {
8303 if (vsi->mqprio_qopt.max_rate[0]) {
8304 u64 max_tx_rate = i40e_bw_bytes_to_mbits(vsi,
8305 vsi->mqprio_qopt.max_rate[0]);
8306
8307 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate);
8308 if (!ret) {
8309 u64 credits = max_tx_rate;
8310
8311 do_div(credits, I40E_BW_CREDIT_DIVISOR);
8312 dev_dbg(&vsi->back->pdev->dev,
8313 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
8314 max_tx_rate,
8315 credits,
8316 vsi->seid);
8317 } else {
8318 need_reset = true;
8319 goto exit;
8320 }
8321 }
8322 ret = i40e_configure_queue_channels(vsi);
8323 if (ret) {
8324 vsi->num_queue_pairs = old_queue_pairs;
8325 netdev_info(netdev,
8326 "Failed configuring queue channels\n");
8327 need_reset = true;
8328 goto exit;
8329 }
8330 }
8331
8332 exit:
8333 /* Reset the configuration data to defaults, only TC0 is enabled */
8334 if (need_reset) {
8335 i40e_vsi_set_default_tc_config(vsi);
8336 need_reset = false;
8337 }
8338
8339 /* Unquiesce VSI */
8340 i40e_unquiesce_vsi(vsi);
8341 return ret;
8342 }
8343
8344 /**
8345 * i40e_set_cld_element - sets cloud filter element data
8346 * @filter: cloud filter rule
8347 * @cld: ptr to cloud filter element data
8348 *
8349 * This is helper function to copy data into cloud filter element
8350 **/
8351 static inline void
i40e_set_cld_element(struct i40e_cloud_filter * filter,struct i40e_aqc_cloud_filters_element_data * cld)8352 i40e_set_cld_element(struct i40e_cloud_filter *filter,
8353 struct i40e_aqc_cloud_filters_element_data *cld)
8354 {
8355 u32 ipa;
8356 int i;
8357
8358 memset(cld, 0, sizeof(*cld));
8359 ether_addr_copy(cld->outer_mac, filter->dst_mac);
8360 ether_addr_copy(cld->inner_mac, filter->src_mac);
8361
8362 if (filter->n_proto != ETH_P_IP && filter->n_proto != ETH_P_IPV6)
8363 return;
8364
8365 if (filter->n_proto == ETH_P_IPV6) {
8366 #define IPV6_MAX_INDEX (ARRAY_SIZE(filter->dst_ipv6) - 1)
8367 for (i = 0; i < ARRAY_SIZE(filter->dst_ipv6); i++) {
8368 ipa = be32_to_cpu(filter->dst_ipv6[IPV6_MAX_INDEX - i]);
8369
8370 *(__le32 *)&cld->ipaddr.raw_v6.data[i * 2] = cpu_to_le32(ipa);
8371 }
8372 } else {
8373 ipa = be32_to_cpu(filter->dst_ipv4);
8374
8375 memcpy(&cld->ipaddr.v4.data, &ipa, sizeof(ipa));
8376 }
8377
8378 cld->inner_vlan = cpu_to_le16(ntohs(filter->vlan_id));
8379
8380 /* tenant_id is not supported by FW now, once the support is enabled
8381 * fill the cld->tenant_id with cpu_to_le32(filter->tenant_id)
8382 */
8383 if (filter->tenant_id)
8384 return;
8385 }
8386
8387 /**
8388 * i40e_add_del_cloud_filter - Add/del cloud filter
8389 * @vsi: pointer to VSI
8390 * @filter: cloud filter rule
8391 * @add: if true, add, if false, delete
8392 *
8393 * Add or delete a cloud filter for a specific flow spec.
8394 * Returns 0 if the filter were successfully added.
8395 **/
i40e_add_del_cloud_filter(struct i40e_vsi * vsi,struct i40e_cloud_filter * filter,bool add)8396 int i40e_add_del_cloud_filter(struct i40e_vsi *vsi,
8397 struct i40e_cloud_filter *filter, bool add)
8398 {
8399 struct i40e_aqc_cloud_filters_element_data cld_filter;
8400 struct i40e_pf *pf = vsi->back;
8401 int ret;
8402 static const u16 flag_table[128] = {
8403 [I40E_CLOUD_FILTER_FLAGS_OMAC] =
8404 I40E_AQC_ADD_CLOUD_FILTER_OMAC,
8405 [I40E_CLOUD_FILTER_FLAGS_IMAC] =
8406 I40E_AQC_ADD_CLOUD_FILTER_IMAC,
8407 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN] =
8408 I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN,
8409 [I40E_CLOUD_FILTER_FLAGS_IMAC_TEN_ID] =
8410 I40E_AQC_ADD_CLOUD_FILTER_IMAC_TEN_ID,
8411 [I40E_CLOUD_FILTER_FLAGS_OMAC_TEN_ID_IMAC] =
8412 I40E_AQC_ADD_CLOUD_FILTER_OMAC_TEN_ID_IMAC,
8413 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN_TEN_ID] =
8414 I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN_TEN_ID,
8415 [I40E_CLOUD_FILTER_FLAGS_IIP] =
8416 I40E_AQC_ADD_CLOUD_FILTER_IIP,
8417 };
8418
8419 if (filter->flags >= ARRAY_SIZE(flag_table))
8420 return -EIO;
8421
8422 memset(&cld_filter, 0, sizeof(cld_filter));
8423
8424 /* copy element needed to add cloud filter from filter */
8425 i40e_set_cld_element(filter, &cld_filter);
8426
8427 if (filter->tunnel_type != I40E_CLOUD_TNL_TYPE_NONE)
8428 cld_filter.flags = cpu_to_le16(filter->tunnel_type <<
8429 I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT);
8430
8431 if (filter->n_proto == ETH_P_IPV6)
8432 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] |
8433 I40E_AQC_ADD_CLOUD_FLAGS_IPV6);
8434 else
8435 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] |
8436 I40E_AQC_ADD_CLOUD_FLAGS_IPV4);
8437
8438 if (add)
8439 ret = i40e_aq_add_cloud_filters(&pf->hw, filter->seid,
8440 &cld_filter, 1);
8441 else
8442 ret = i40e_aq_rem_cloud_filters(&pf->hw, filter->seid,
8443 &cld_filter, 1);
8444 if (ret)
8445 dev_dbg(&pf->pdev->dev,
8446 "Failed to %s cloud filter using l4 port %u, err %d aq_err %d\n",
8447 add ? "add" : "delete", filter->dst_port, ret,
8448 pf->hw.aq.asq_last_status);
8449 else
8450 dev_info(&pf->pdev->dev,
8451 "%s cloud filter for VSI: %d\n",
8452 add ? "Added" : "Deleted", filter->seid);
8453 return ret;
8454 }
8455
8456 /**
8457 * i40e_add_del_cloud_filter_big_buf - Add/del cloud filter using big_buf
8458 * @vsi: pointer to VSI
8459 * @filter: cloud filter rule
8460 * @add: if true, add, if false, delete
8461 *
8462 * Add or delete a cloud filter for a specific flow spec using big buffer.
8463 * Returns 0 if the filter were successfully added.
8464 **/
i40e_add_del_cloud_filter_big_buf(struct i40e_vsi * vsi,struct i40e_cloud_filter * filter,bool add)8465 int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi,
8466 struct i40e_cloud_filter *filter,
8467 bool add)
8468 {
8469 struct i40e_aqc_cloud_filters_element_bb cld_filter;
8470 struct i40e_pf *pf = vsi->back;
8471 int ret;
8472
8473 /* Both (src/dst) valid mac_addr are not supported */
8474 if ((is_valid_ether_addr(filter->dst_mac) &&
8475 is_valid_ether_addr(filter->src_mac)) ||
8476 (is_multicast_ether_addr(filter->dst_mac) &&
8477 is_multicast_ether_addr(filter->src_mac)))
8478 return -EOPNOTSUPP;
8479
8480 /* Big buffer cloud filter needs 'L4 port' to be non-zero. Also, UDP
8481 * ports are not supported via big buffer now.
8482 */
8483 if (!filter->dst_port || filter->ip_proto == IPPROTO_UDP)
8484 return -EOPNOTSUPP;
8485
8486 /* adding filter using src_port/src_ip is not supported at this stage */
8487 if (filter->src_port ||
8488 (filter->src_ipv4 && filter->n_proto != ETH_P_IPV6) ||
8489 !ipv6_addr_any(&filter->ip.v6.src_ip6))
8490 return -EOPNOTSUPP;
8491
8492 memset(&cld_filter, 0, sizeof(cld_filter));
8493
8494 /* copy element needed to add cloud filter from filter */
8495 i40e_set_cld_element(filter, &cld_filter.element);
8496
8497 if (is_valid_ether_addr(filter->dst_mac) ||
8498 is_valid_ether_addr(filter->src_mac) ||
8499 is_multicast_ether_addr(filter->dst_mac) ||
8500 is_multicast_ether_addr(filter->src_mac)) {
8501 /* MAC + IP : unsupported mode */
8502 if (filter->dst_ipv4)
8503 return -EOPNOTSUPP;
8504
8505 /* since we validated that L4 port must be valid before
8506 * we get here, start with respective "flags" value
8507 * and update if vlan is present or not
8508 */
8509 cld_filter.element.flags =
8510 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_PORT);
8511
8512 if (filter->vlan_id) {
8513 cld_filter.element.flags =
8514 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_VLAN_PORT);
8515 }
8516
8517 } else if ((filter->dst_ipv4 && filter->n_proto != ETH_P_IPV6) ||
8518 !ipv6_addr_any(&filter->ip.v6.dst_ip6)) {
8519 cld_filter.element.flags =
8520 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_IP_PORT);
8521 if (filter->n_proto == ETH_P_IPV6)
8522 cld_filter.element.flags |=
8523 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV6);
8524 else
8525 cld_filter.element.flags |=
8526 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV4);
8527 } else {
8528 dev_err(&pf->pdev->dev,
8529 "either mac or ip has to be valid for cloud filter\n");
8530 return -EINVAL;
8531 }
8532
8533 /* Now copy L4 port in Byte 6..7 in general fields */
8534 cld_filter.general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD0] =
8535 be16_to_cpu(filter->dst_port);
8536
8537 if (add) {
8538 /* Validate current device switch mode, change if necessary */
8539 ret = i40e_validate_and_set_switch_mode(vsi);
8540 if (ret) {
8541 dev_err(&pf->pdev->dev,
8542 "failed to set switch mode, ret %d\n",
8543 ret);
8544 return ret;
8545 }
8546
8547 ret = i40e_aq_add_cloud_filters_bb(&pf->hw, filter->seid,
8548 &cld_filter, 1);
8549 } else {
8550 ret = i40e_aq_rem_cloud_filters_bb(&pf->hw, filter->seid,
8551 &cld_filter, 1);
8552 }
8553
8554 if (ret)
8555 dev_dbg(&pf->pdev->dev,
8556 "Failed to %s cloud filter(big buffer) err %d aq_err %d\n",
8557 add ? "add" : "delete", ret, pf->hw.aq.asq_last_status);
8558 else
8559 dev_info(&pf->pdev->dev,
8560 "%s cloud filter for VSI: %d, L4 port: %d\n",
8561 add ? "add" : "delete", filter->seid,
8562 ntohs(filter->dst_port));
8563 return ret;
8564 }
8565
8566 /**
8567 * i40e_parse_cls_flower - Parse tc flower filters provided by kernel
8568 * @vsi: Pointer to VSI
8569 * @f: Pointer to struct flow_cls_offload
8570 * @filter: Pointer to cloud filter structure
8571 *
8572 **/
i40e_parse_cls_flower(struct i40e_vsi * vsi,struct flow_cls_offload * f,struct i40e_cloud_filter * filter)8573 static int i40e_parse_cls_flower(struct i40e_vsi *vsi,
8574 struct flow_cls_offload *f,
8575 struct i40e_cloud_filter *filter)
8576 {
8577 struct flow_rule *rule = flow_cls_offload_flow_rule(f);
8578 struct flow_dissector *dissector = rule->match.dissector;
8579 u16 n_proto_mask = 0, n_proto_key = 0, addr_type = 0;
8580 struct i40e_pf *pf = vsi->back;
8581 u8 field_flags = 0;
8582
8583 if (dissector->used_keys &
8584 ~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) |
8585 BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) |
8586 BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
8587 BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) |
8588 BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
8589 BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
8590 BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) |
8591 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID))) {
8592 dev_err(&pf->pdev->dev, "Unsupported key used: 0x%llx\n",
8593 dissector->used_keys);
8594 return -EOPNOTSUPP;
8595 }
8596
8597 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
8598 struct flow_match_enc_keyid match;
8599
8600 flow_rule_match_enc_keyid(rule, &match);
8601 if (match.mask->keyid != 0)
8602 field_flags |= I40E_CLOUD_FIELD_TEN_ID;
8603
8604 filter->tenant_id = be32_to_cpu(match.key->keyid);
8605 }
8606
8607 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
8608 struct flow_match_basic match;
8609
8610 flow_rule_match_basic(rule, &match);
8611 n_proto_key = ntohs(match.key->n_proto);
8612 n_proto_mask = ntohs(match.mask->n_proto);
8613
8614 if (n_proto_key == ETH_P_ALL) {
8615 n_proto_key = 0;
8616 n_proto_mask = 0;
8617 }
8618 filter->n_proto = n_proto_key & n_proto_mask;
8619 filter->ip_proto = match.key->ip_proto;
8620 }
8621
8622 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
8623 struct flow_match_eth_addrs match;
8624
8625 flow_rule_match_eth_addrs(rule, &match);
8626
8627 /* use is_broadcast and is_zero to check for all 0xf or 0 */
8628 if (!is_zero_ether_addr(match.mask->dst)) {
8629 if (is_broadcast_ether_addr(match.mask->dst)) {
8630 field_flags |= I40E_CLOUD_FIELD_OMAC;
8631 } else {
8632 dev_err(&pf->pdev->dev, "Bad ether dest mask %pM\n",
8633 match.mask->dst);
8634 return -EIO;
8635 }
8636 }
8637
8638 if (!is_zero_ether_addr(match.mask->src)) {
8639 if (is_broadcast_ether_addr(match.mask->src)) {
8640 field_flags |= I40E_CLOUD_FIELD_IMAC;
8641 } else {
8642 dev_err(&pf->pdev->dev, "Bad ether src mask %pM\n",
8643 match.mask->src);
8644 return -EIO;
8645 }
8646 }
8647 ether_addr_copy(filter->dst_mac, match.key->dst);
8648 ether_addr_copy(filter->src_mac, match.key->src);
8649 }
8650
8651 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
8652 struct flow_match_vlan match;
8653
8654 flow_rule_match_vlan(rule, &match);
8655 if (match.mask->vlan_id) {
8656 if (match.mask->vlan_id == VLAN_VID_MASK) {
8657 field_flags |= I40E_CLOUD_FIELD_IVLAN;
8658
8659 } else {
8660 dev_err(&pf->pdev->dev, "Bad vlan mask 0x%04x\n",
8661 match.mask->vlan_id);
8662 return -EIO;
8663 }
8664 }
8665
8666 filter->vlan_id = cpu_to_be16(match.key->vlan_id);
8667 }
8668
8669 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
8670 struct flow_match_control match;
8671
8672 flow_rule_match_control(rule, &match);
8673 addr_type = match.key->addr_type;
8674
8675 if (flow_rule_has_control_flags(match.mask->flags,
8676 f->common.extack))
8677 return -EOPNOTSUPP;
8678 }
8679
8680 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
8681 struct flow_match_ipv4_addrs match;
8682
8683 flow_rule_match_ipv4_addrs(rule, &match);
8684 if (match.mask->dst) {
8685 if (match.mask->dst == cpu_to_be32(0xffffffff)) {
8686 field_flags |= I40E_CLOUD_FIELD_IIP;
8687 } else {
8688 dev_err(&pf->pdev->dev, "Bad ip dst mask %pI4b\n",
8689 &match.mask->dst);
8690 return -EIO;
8691 }
8692 }
8693
8694 if (match.mask->src) {
8695 if (match.mask->src == cpu_to_be32(0xffffffff)) {
8696 field_flags |= I40E_CLOUD_FIELD_IIP;
8697 } else {
8698 dev_err(&pf->pdev->dev, "Bad ip src mask %pI4b\n",
8699 &match.mask->src);
8700 return -EIO;
8701 }
8702 }
8703
8704 if (field_flags & I40E_CLOUD_FIELD_TEN_ID) {
8705 dev_err(&pf->pdev->dev, "Tenant id not allowed for ip filter\n");
8706 return -EIO;
8707 }
8708 filter->dst_ipv4 = match.key->dst;
8709 filter->src_ipv4 = match.key->src;
8710 }
8711
8712 if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
8713 struct flow_match_ipv6_addrs match;
8714
8715 flow_rule_match_ipv6_addrs(rule, &match);
8716
8717 /* src and dest IPV6 address should not be LOOPBACK
8718 * (0:0:0:0:0:0:0:1), which can be represented as ::1
8719 */
8720 if (ipv6_addr_loopback(&match.key->dst) ||
8721 ipv6_addr_loopback(&match.key->src)) {
8722 dev_err(&pf->pdev->dev,
8723 "Bad ipv6, addr is LOOPBACK\n");
8724 return -EIO;
8725 }
8726 if (!ipv6_addr_any(&match.mask->dst) ||
8727 !ipv6_addr_any(&match.mask->src))
8728 field_flags |= I40E_CLOUD_FIELD_IIP;
8729
8730 memcpy(&filter->src_ipv6, &match.key->src.s6_addr32,
8731 sizeof(filter->src_ipv6));
8732 memcpy(&filter->dst_ipv6, &match.key->dst.s6_addr32,
8733 sizeof(filter->dst_ipv6));
8734 }
8735
8736 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
8737 struct flow_match_ports match;
8738
8739 flow_rule_match_ports(rule, &match);
8740 if (match.mask->src) {
8741 if (match.mask->src == cpu_to_be16(0xffff)) {
8742 field_flags |= I40E_CLOUD_FIELD_IIP;
8743 } else {
8744 dev_err(&pf->pdev->dev, "Bad src port mask 0x%04x\n",
8745 be16_to_cpu(match.mask->src));
8746 return -EIO;
8747 }
8748 }
8749
8750 if (match.mask->dst) {
8751 if (match.mask->dst == cpu_to_be16(0xffff)) {
8752 field_flags |= I40E_CLOUD_FIELD_IIP;
8753 } else {
8754 dev_err(&pf->pdev->dev, "Bad dst port mask 0x%04x\n",
8755 be16_to_cpu(match.mask->dst));
8756 return -EIO;
8757 }
8758 }
8759
8760 filter->dst_port = match.key->dst;
8761 filter->src_port = match.key->src;
8762
8763 switch (filter->ip_proto) {
8764 case IPPROTO_TCP:
8765 case IPPROTO_UDP:
8766 break;
8767 default:
8768 dev_err(&pf->pdev->dev,
8769 "Only UDP and TCP transport are supported\n");
8770 return -EINVAL;
8771 }
8772 }
8773 filter->flags = field_flags;
8774 return 0;
8775 }
8776
8777 /**
8778 * i40e_handle_tclass: Forward to a traffic class on the device
8779 * @vsi: Pointer to VSI
8780 * @tc: traffic class index on the device
8781 * @filter: Pointer to cloud filter structure
8782 *
8783 **/
i40e_handle_tclass(struct i40e_vsi * vsi,u32 tc,struct i40e_cloud_filter * filter)8784 static int i40e_handle_tclass(struct i40e_vsi *vsi, u32 tc,
8785 struct i40e_cloud_filter *filter)
8786 {
8787 struct i40e_channel *ch, *ch_tmp;
8788
8789 /* direct to a traffic class on the same device */
8790 if (tc == 0) {
8791 filter->seid = vsi->seid;
8792 return 0;
8793 } else if (vsi->tc_config.enabled_tc & BIT(tc)) {
8794 if (!filter->dst_port) {
8795 dev_err(&vsi->back->pdev->dev,
8796 "Specify destination port to direct to traffic class that is not default\n");
8797 return -EINVAL;
8798 }
8799 if (list_empty(&vsi->ch_list))
8800 return -EINVAL;
8801 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list,
8802 list) {
8803 if (ch->seid == vsi->tc_seid_map[tc])
8804 filter->seid = ch->seid;
8805 }
8806 return 0;
8807 }
8808 dev_err(&vsi->back->pdev->dev, "TC is not enabled\n");
8809 return -EINVAL;
8810 }
8811
8812 /**
8813 * i40e_configure_clsflower - Configure tc flower filters
8814 * @vsi: Pointer to VSI
8815 * @cls_flower: Pointer to struct flow_cls_offload
8816 *
8817 **/
i40e_configure_clsflower(struct i40e_vsi * vsi,struct flow_cls_offload * cls_flower)8818 static int i40e_configure_clsflower(struct i40e_vsi *vsi,
8819 struct flow_cls_offload *cls_flower)
8820 {
8821 int tc = tc_classid_to_hwtc(vsi->netdev, cls_flower->classid);
8822 struct i40e_cloud_filter *filter = NULL;
8823 struct i40e_pf *pf = vsi->back;
8824 int err = 0;
8825
8826 if (tc < 0) {
8827 dev_err(&vsi->back->pdev->dev, "Invalid traffic class\n");
8828 return -EOPNOTSUPP;
8829 }
8830
8831 if (!tc) {
8832 dev_err(&pf->pdev->dev, "Unable to add filter because of invalid destination");
8833 return -EINVAL;
8834 }
8835
8836 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) ||
8837 test_bit(__I40E_RESET_INTR_RECEIVED, pf->state))
8838 return -EBUSY;
8839
8840 if (pf->fdir_pf_active_filters ||
8841 (!hlist_empty(&pf->fdir_filter_list))) {
8842 dev_err(&vsi->back->pdev->dev,
8843 "Flow Director Sideband filters exists, turn ntuple off to configure cloud filters\n");
8844 return -EINVAL;
8845 }
8846
8847 if (test_bit(I40E_FLAG_FD_SB_ENA, vsi->back->flags)) {
8848 dev_err(&vsi->back->pdev->dev,
8849 "Disable Flow Director Sideband, configuring Cloud filters via tc-flower\n");
8850 clear_bit(I40E_FLAG_FD_SB_ENA, vsi->back->flags);
8851 clear_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, vsi->back->flags);
8852 }
8853
8854 filter = kzalloc(sizeof(*filter), GFP_KERNEL);
8855 if (!filter)
8856 return -ENOMEM;
8857
8858 filter->cookie = cls_flower->cookie;
8859
8860 err = i40e_parse_cls_flower(vsi, cls_flower, filter);
8861 if (err < 0)
8862 goto err;
8863
8864 err = i40e_handle_tclass(vsi, tc, filter);
8865 if (err < 0)
8866 goto err;
8867
8868 /* Add cloud filter */
8869 if (filter->dst_port)
8870 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, true);
8871 else
8872 err = i40e_add_del_cloud_filter(vsi, filter, true);
8873
8874 if (err) {
8875 dev_err(&pf->pdev->dev, "Failed to add cloud filter, err %d\n",
8876 err);
8877 goto err;
8878 }
8879
8880 /* add filter to the ordered list */
8881 INIT_HLIST_NODE(&filter->cloud_node);
8882
8883 hlist_add_head(&filter->cloud_node, &pf->cloud_filter_list);
8884
8885 pf->num_cloud_filters++;
8886
8887 return err;
8888 err:
8889 kfree(filter);
8890 return err;
8891 }
8892
8893 /**
8894 * i40e_find_cloud_filter - Find the could filter in the list
8895 * @vsi: Pointer to VSI
8896 * @cookie: filter specific cookie
8897 *
8898 **/
i40e_find_cloud_filter(struct i40e_vsi * vsi,unsigned long * cookie)8899 static struct i40e_cloud_filter *i40e_find_cloud_filter(struct i40e_vsi *vsi,
8900 unsigned long *cookie)
8901 {
8902 struct i40e_cloud_filter *filter = NULL;
8903 struct hlist_node *node2;
8904
8905 hlist_for_each_entry_safe(filter, node2,
8906 &vsi->back->cloud_filter_list, cloud_node)
8907 if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie)))
8908 return filter;
8909 return NULL;
8910 }
8911
8912 /**
8913 * i40e_delete_clsflower - Remove tc flower filters
8914 * @vsi: Pointer to VSI
8915 * @cls_flower: Pointer to struct flow_cls_offload
8916 *
8917 **/
i40e_delete_clsflower(struct i40e_vsi * vsi,struct flow_cls_offload * cls_flower)8918 static int i40e_delete_clsflower(struct i40e_vsi *vsi,
8919 struct flow_cls_offload *cls_flower)
8920 {
8921 struct i40e_cloud_filter *filter = NULL;
8922 struct i40e_pf *pf = vsi->back;
8923 int err = 0;
8924
8925 filter = i40e_find_cloud_filter(vsi, &cls_flower->cookie);
8926
8927 if (!filter)
8928 return -EINVAL;
8929
8930 hash_del(&filter->cloud_node);
8931
8932 if (filter->dst_port)
8933 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, false);
8934 else
8935 err = i40e_add_del_cloud_filter(vsi, filter, false);
8936
8937 kfree(filter);
8938 if (err) {
8939 dev_err(&pf->pdev->dev,
8940 "Failed to delete cloud filter, err %pe\n",
8941 ERR_PTR(err));
8942 return i40e_aq_rc_to_posix(err, pf->hw.aq.asq_last_status);
8943 }
8944
8945 pf->num_cloud_filters--;
8946 if (!pf->num_cloud_filters)
8947 if (test_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags) &&
8948 !test_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags)) {
8949 set_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
8950 clear_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags);
8951 clear_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
8952 }
8953 return 0;
8954 }
8955
8956 /**
8957 * i40e_setup_tc_cls_flower - flower classifier offloads
8958 * @np: net device to configure
8959 * @cls_flower: offload data
8960 **/
i40e_setup_tc_cls_flower(struct i40e_netdev_priv * np,struct flow_cls_offload * cls_flower)8961 static int i40e_setup_tc_cls_flower(struct i40e_netdev_priv *np,
8962 struct flow_cls_offload *cls_flower)
8963 {
8964 struct i40e_vsi *vsi = np->vsi;
8965
8966 switch (cls_flower->command) {
8967 case FLOW_CLS_REPLACE:
8968 return i40e_configure_clsflower(vsi, cls_flower);
8969 case FLOW_CLS_DESTROY:
8970 return i40e_delete_clsflower(vsi, cls_flower);
8971 case FLOW_CLS_STATS:
8972 return -EOPNOTSUPP;
8973 default:
8974 return -EOPNOTSUPP;
8975 }
8976 }
8977
i40e_setup_tc_block_cb(enum tc_setup_type type,void * type_data,void * cb_priv)8978 static int i40e_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
8979 void *cb_priv)
8980 {
8981 struct i40e_netdev_priv *np = cb_priv;
8982
8983 if (!tc_cls_can_offload_and_chain0(np->vsi->netdev, type_data))
8984 return -EOPNOTSUPP;
8985
8986 switch (type) {
8987 case TC_SETUP_CLSFLOWER:
8988 return i40e_setup_tc_cls_flower(np, type_data);
8989
8990 default:
8991 return -EOPNOTSUPP;
8992 }
8993 }
8994
8995 static LIST_HEAD(i40e_block_cb_list);
8996
__i40e_setup_tc(struct net_device * netdev,enum tc_setup_type type,void * type_data)8997 static int __i40e_setup_tc(struct net_device *netdev, enum tc_setup_type type,
8998 void *type_data)
8999 {
9000 struct i40e_netdev_priv *np = netdev_priv(netdev);
9001
9002 switch (type) {
9003 case TC_SETUP_QDISC_MQPRIO:
9004 return i40e_setup_tc(netdev, type_data);
9005 case TC_SETUP_BLOCK:
9006 return flow_block_cb_setup_simple(type_data,
9007 &i40e_block_cb_list,
9008 i40e_setup_tc_block_cb,
9009 np, np, true);
9010 default:
9011 return -EOPNOTSUPP;
9012 }
9013 }
9014
9015 /**
9016 * i40e_open - Called when a network interface is made active
9017 * @netdev: network interface device structure
9018 *
9019 * The open entry point is called when a network interface is made
9020 * active by the system (IFF_UP). At this point all resources needed
9021 * for transmit and receive operations are allocated, the interrupt
9022 * handler is registered with the OS, the netdev watchdog subtask is
9023 * enabled, and the stack is notified that the interface is ready.
9024 *
9025 * Returns 0 on success, negative value on failure
9026 **/
i40e_open(struct net_device * netdev)9027 int i40e_open(struct net_device *netdev)
9028 {
9029 struct i40e_netdev_priv *np = netdev_priv(netdev);
9030 struct i40e_vsi *vsi = np->vsi;
9031 struct i40e_pf *pf = vsi->back;
9032 int err;
9033
9034 /* disallow open during test or if eeprom is broken */
9035 if (test_bit(__I40E_TESTING, pf->state) ||
9036 test_bit(__I40E_BAD_EEPROM, pf->state))
9037 return -EBUSY;
9038
9039 netif_carrier_off(netdev);
9040
9041 if (i40e_force_link_state(pf, true))
9042 return -EAGAIN;
9043
9044 err = i40e_vsi_open(vsi);
9045 if (err)
9046 return err;
9047
9048 /* configure global TSO hardware offload settings */
9049 wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH |
9050 TCP_FLAG_FIN) >> 16);
9051 wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH |
9052 TCP_FLAG_FIN |
9053 TCP_FLAG_CWR) >> 16);
9054 wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16);
9055 udp_tunnel_get_rx_info(netdev);
9056
9057 return 0;
9058 }
9059
9060 /**
9061 * i40e_netif_set_realnum_tx_rx_queues - Update number of tx/rx queues
9062 * @vsi: vsi structure
9063 *
9064 * This updates netdev's number of tx/rx queues
9065 *
9066 * Returns status of setting tx/rx queues
9067 **/
i40e_netif_set_realnum_tx_rx_queues(struct i40e_vsi * vsi)9068 static int i40e_netif_set_realnum_tx_rx_queues(struct i40e_vsi *vsi)
9069 {
9070 int ret;
9071
9072 ret = netif_set_real_num_rx_queues(vsi->netdev,
9073 vsi->num_queue_pairs);
9074 if (ret)
9075 return ret;
9076
9077 return netif_set_real_num_tx_queues(vsi->netdev,
9078 vsi->num_queue_pairs);
9079 }
9080
9081 /**
9082 * i40e_vsi_open -
9083 * @vsi: the VSI to open
9084 *
9085 * Finish initialization of the VSI.
9086 *
9087 * Returns 0 on success, negative value on failure
9088 *
9089 * Note: expects to be called while under rtnl_lock()
9090 **/
i40e_vsi_open(struct i40e_vsi * vsi)9091 int i40e_vsi_open(struct i40e_vsi *vsi)
9092 {
9093 struct i40e_pf *pf = vsi->back;
9094 char int_name[I40E_INT_NAME_STR_LEN];
9095 int err;
9096
9097 /* allocate descriptors */
9098 err = i40e_vsi_setup_tx_resources(vsi);
9099 if (err)
9100 goto err_setup_tx;
9101 err = i40e_vsi_setup_rx_resources(vsi);
9102 if (err)
9103 goto err_setup_rx;
9104
9105 err = i40e_vsi_configure(vsi);
9106 if (err)
9107 goto err_setup_rx;
9108
9109 if (vsi->netdev) {
9110 snprintf(int_name, sizeof(int_name) - 1, "%s-%s",
9111 dev_driver_string(&pf->pdev->dev), vsi->netdev->name);
9112 err = i40e_vsi_request_irq(vsi, int_name);
9113 if (err)
9114 goto err_setup_rx;
9115
9116 /* Notify the stack of the actual queue counts. */
9117 err = i40e_netif_set_realnum_tx_rx_queues(vsi);
9118 if (err)
9119 goto err_set_queues;
9120
9121 } else if (vsi->type == I40E_VSI_FDIR) {
9122 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir",
9123 dev_driver_string(&pf->pdev->dev),
9124 dev_name(&pf->pdev->dev));
9125 err = i40e_vsi_request_irq(vsi, int_name);
9126 if (err)
9127 goto err_setup_rx;
9128
9129 } else {
9130 err = -EINVAL;
9131 goto err_setup_rx;
9132 }
9133
9134 err = i40e_up_complete(vsi);
9135 if (err)
9136 goto err_up_complete;
9137
9138 return 0;
9139
9140 err_up_complete:
9141 i40e_down(vsi);
9142 err_set_queues:
9143 i40e_vsi_free_irq(vsi);
9144 err_setup_rx:
9145 i40e_vsi_free_rx_resources(vsi);
9146 err_setup_tx:
9147 i40e_vsi_free_tx_resources(vsi);
9148 if (vsi->type == I40E_VSI_MAIN)
9149 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
9150
9151 return err;
9152 }
9153
9154 /**
9155 * i40e_fdir_filter_exit - Cleans up the Flow Director accounting
9156 * @pf: Pointer to PF
9157 *
9158 * This function destroys the hlist where all the Flow Director
9159 * filters were saved.
9160 **/
i40e_fdir_filter_exit(struct i40e_pf * pf)9161 static void i40e_fdir_filter_exit(struct i40e_pf *pf)
9162 {
9163 struct i40e_fdir_filter *filter;
9164 struct i40e_flex_pit *pit_entry, *tmp;
9165 struct hlist_node *node2;
9166
9167 hlist_for_each_entry_safe(filter, node2,
9168 &pf->fdir_filter_list, fdir_node) {
9169 hlist_del(&filter->fdir_node);
9170 kfree(filter);
9171 }
9172
9173 list_for_each_entry_safe(pit_entry, tmp, &pf->l3_flex_pit_list, list) {
9174 list_del(&pit_entry->list);
9175 kfree(pit_entry);
9176 }
9177 INIT_LIST_HEAD(&pf->l3_flex_pit_list);
9178
9179 list_for_each_entry_safe(pit_entry, tmp, &pf->l4_flex_pit_list, list) {
9180 list_del(&pit_entry->list);
9181 kfree(pit_entry);
9182 }
9183 INIT_LIST_HEAD(&pf->l4_flex_pit_list);
9184
9185 pf->fdir_pf_active_filters = 0;
9186 i40e_reset_fdir_filter_cnt(pf);
9187
9188 /* Reprogram the default input set for TCP/IPv4 */
9189 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
9190 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
9191 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9192
9193 /* Reprogram the default input set for TCP/IPv6 */
9194 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_TCP,
9195 I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK |
9196 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9197
9198 /* Reprogram the default input set for UDP/IPv4 */
9199 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_UDP,
9200 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
9201 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9202
9203 /* Reprogram the default input set for UDP/IPv6 */
9204 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_UDP,
9205 I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK |
9206 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9207
9208 /* Reprogram the default input set for SCTP/IPv4 */
9209 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_SCTP,
9210 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
9211 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9212
9213 /* Reprogram the default input set for SCTP/IPv6 */
9214 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_SCTP,
9215 I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK |
9216 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9217
9218 /* Reprogram the default input set for Other/IPv4 */
9219 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_OTHER,
9220 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
9221
9222 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_FRAG_IPV4,
9223 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
9224
9225 /* Reprogram the default input set for Other/IPv6 */
9226 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_OTHER,
9227 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
9228
9229 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_FRAG_IPV6,
9230 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
9231 }
9232
9233 /**
9234 * i40e_cloud_filter_exit - Cleans up the cloud filters
9235 * @pf: Pointer to PF
9236 *
9237 * This function destroys the hlist where all the cloud filters
9238 * were saved.
9239 **/
i40e_cloud_filter_exit(struct i40e_pf * pf)9240 static void i40e_cloud_filter_exit(struct i40e_pf *pf)
9241 {
9242 struct i40e_cloud_filter *cfilter;
9243 struct hlist_node *node;
9244
9245 hlist_for_each_entry_safe(cfilter, node,
9246 &pf->cloud_filter_list, cloud_node) {
9247 hlist_del(&cfilter->cloud_node);
9248 kfree(cfilter);
9249 }
9250 pf->num_cloud_filters = 0;
9251
9252 if (test_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags) &&
9253 !test_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags)) {
9254 set_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
9255 clear_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags);
9256 clear_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
9257 }
9258 }
9259
9260 /**
9261 * i40e_close - Disables a network interface
9262 * @netdev: network interface device structure
9263 *
9264 * The close entry point is called when an interface is de-activated
9265 * by the OS. The hardware is still under the driver's control, but
9266 * this netdev interface is disabled.
9267 *
9268 * Returns 0, this is not allowed to fail
9269 **/
i40e_close(struct net_device * netdev)9270 int i40e_close(struct net_device *netdev)
9271 {
9272 struct i40e_netdev_priv *np = netdev_priv(netdev);
9273 struct i40e_vsi *vsi = np->vsi;
9274
9275 i40e_vsi_close(vsi);
9276
9277 return 0;
9278 }
9279
9280 /**
9281 * i40e_do_reset - Start a PF or Core Reset sequence
9282 * @pf: board private structure
9283 * @reset_flags: which reset is requested
9284 * @lock_acquired: indicates whether or not the lock has been acquired
9285 * before this function was called.
9286 *
9287 * The essential difference in resets is that the PF Reset
9288 * doesn't clear the packet buffers, doesn't reset the PE
9289 * firmware, and doesn't bother the other PFs on the chip.
9290 **/
i40e_do_reset(struct i40e_pf * pf,u32 reset_flags,bool lock_acquired)9291 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired)
9292 {
9293 struct i40e_vsi *vsi;
9294 u32 val;
9295 int i;
9296
9297 /* do the biggest reset indicated */
9298 if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) {
9299
9300 /* Request a Global Reset
9301 *
9302 * This will start the chip's countdown to the actual full
9303 * chip reset event, and a warning interrupt to be sent
9304 * to all PFs, including the requestor. Our handler
9305 * for the warning interrupt will deal with the shutdown
9306 * and recovery of the switch setup.
9307 */
9308 dev_dbg(&pf->pdev->dev, "GlobalR requested\n");
9309 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
9310 val |= I40E_GLGEN_RTRIG_GLOBR_MASK;
9311 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
9312
9313 } else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) {
9314
9315 /* Request a Core Reset
9316 *
9317 * Same as Global Reset, except does *not* include the MAC/PHY
9318 */
9319 dev_dbg(&pf->pdev->dev, "CoreR requested\n");
9320 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
9321 val |= I40E_GLGEN_RTRIG_CORER_MASK;
9322 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
9323 i40e_flush(&pf->hw);
9324
9325 } else if (reset_flags & I40E_PF_RESET_FLAG) {
9326
9327 /* Request a PF Reset
9328 *
9329 * Resets only the PF-specific registers
9330 *
9331 * This goes directly to the tear-down and rebuild of
9332 * the switch, since we need to do all the recovery as
9333 * for the Core Reset.
9334 */
9335 dev_dbg(&pf->pdev->dev, "PFR requested\n");
9336 i40e_handle_reset_warning(pf, lock_acquired);
9337
9338 } else if (reset_flags & I40E_PF_RESET_AND_REBUILD_FLAG) {
9339 /* Request a PF Reset
9340 *
9341 * Resets PF and reinitializes PFs VSI.
9342 */
9343 i40e_prep_for_reset(pf);
9344 i40e_reset_and_rebuild(pf, true, lock_acquired);
9345 dev_info(&pf->pdev->dev,
9346 test_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags) ?
9347 "FW LLDP is disabled\n" :
9348 "FW LLDP is enabled\n");
9349
9350 } else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) {
9351 /* Find the VSI(s) that requested a re-init */
9352 dev_info(&pf->pdev->dev, "VSI reinit requested\n");
9353
9354 i40e_pf_for_each_vsi(pf, i, vsi) {
9355 if (test_and_clear_bit(__I40E_VSI_REINIT_REQUESTED,
9356 vsi->state))
9357 i40e_vsi_reinit_locked(vsi);
9358 }
9359 } else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) {
9360 /* Find the VSI(s) that needs to be brought down */
9361 dev_info(&pf->pdev->dev, "VSI down requested\n");
9362
9363 i40e_pf_for_each_vsi(pf, i, vsi) {
9364 if (test_and_clear_bit(__I40E_VSI_DOWN_REQUESTED,
9365 vsi->state)) {
9366 set_bit(__I40E_VSI_DOWN, vsi->state);
9367 i40e_down(vsi);
9368 }
9369 }
9370 } else {
9371 dev_info(&pf->pdev->dev,
9372 "bad reset request 0x%08x\n", reset_flags);
9373 }
9374 }
9375
9376 #ifdef CONFIG_I40E_DCB
9377 /**
9378 * i40e_dcb_need_reconfig - Check if DCB needs reconfig
9379 * @pf: board private structure
9380 * @old_cfg: current DCB config
9381 * @new_cfg: new DCB config
9382 **/
i40e_dcb_need_reconfig(struct i40e_pf * pf,struct i40e_dcbx_config * old_cfg,struct i40e_dcbx_config * new_cfg)9383 bool i40e_dcb_need_reconfig(struct i40e_pf *pf,
9384 struct i40e_dcbx_config *old_cfg,
9385 struct i40e_dcbx_config *new_cfg)
9386 {
9387 bool need_reconfig = false;
9388
9389 /* Check if ETS configuration has changed */
9390 if (memcmp(&new_cfg->etscfg,
9391 &old_cfg->etscfg,
9392 sizeof(new_cfg->etscfg))) {
9393 /* If Priority Table has changed reconfig is needed */
9394 if (memcmp(&new_cfg->etscfg.prioritytable,
9395 &old_cfg->etscfg.prioritytable,
9396 sizeof(new_cfg->etscfg.prioritytable))) {
9397 need_reconfig = true;
9398 dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n");
9399 }
9400
9401 if (memcmp(&new_cfg->etscfg.tcbwtable,
9402 &old_cfg->etscfg.tcbwtable,
9403 sizeof(new_cfg->etscfg.tcbwtable)))
9404 dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n");
9405
9406 if (memcmp(&new_cfg->etscfg.tsatable,
9407 &old_cfg->etscfg.tsatable,
9408 sizeof(new_cfg->etscfg.tsatable)))
9409 dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n");
9410 }
9411
9412 /* Check if PFC configuration has changed */
9413 if (memcmp(&new_cfg->pfc,
9414 &old_cfg->pfc,
9415 sizeof(new_cfg->pfc))) {
9416 need_reconfig = true;
9417 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n");
9418 }
9419
9420 /* Check if APP Table has changed */
9421 if (memcmp(&new_cfg->app,
9422 &old_cfg->app,
9423 sizeof(new_cfg->app))) {
9424 need_reconfig = true;
9425 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n");
9426 }
9427
9428 dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig);
9429 return need_reconfig;
9430 }
9431
9432 /**
9433 * i40e_handle_lldp_event - Handle LLDP Change MIB event
9434 * @pf: board private structure
9435 * @e: event info posted on ARQ
9436 **/
i40e_handle_lldp_event(struct i40e_pf * pf,struct i40e_arq_event_info * e)9437 static int i40e_handle_lldp_event(struct i40e_pf *pf,
9438 struct i40e_arq_event_info *e)
9439 {
9440 struct i40e_aqc_lldp_get_mib *mib =
9441 (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw;
9442 struct i40e_hw *hw = &pf->hw;
9443 struct i40e_dcbx_config tmp_dcbx_cfg;
9444 bool need_reconfig = false;
9445 int ret = 0;
9446 u8 type;
9447
9448 /* X710-T*L 2.5G and 5G speeds don't support DCB */
9449 if (I40E_IS_X710TL_DEVICE(hw->device_id) &&
9450 (hw->phy.link_info.link_speed &
9451 ~(I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB)) &&
9452 !test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags))
9453 /* let firmware decide if the DCB should be disabled */
9454 set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
9455
9456 /* Not DCB capable or capability disabled */
9457 if (!test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags))
9458 return ret;
9459
9460 /* Ignore if event is not for Nearest Bridge */
9461 type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT)
9462 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK);
9463 dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type);
9464 if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE)
9465 return ret;
9466
9467 /* Check MIB Type and return if event for Remote MIB update */
9468 type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK;
9469 dev_dbg(&pf->pdev->dev,
9470 "LLDP event mib type %s\n", type ? "remote" : "local");
9471 if (type == I40E_AQ_LLDP_MIB_REMOTE) {
9472 /* Update the remote cached instance and return */
9473 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE,
9474 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE,
9475 &hw->remote_dcbx_config);
9476 goto exit;
9477 }
9478
9479 /* Store the old configuration */
9480 tmp_dcbx_cfg = hw->local_dcbx_config;
9481
9482 /* Reset the old DCBx configuration data */
9483 memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config));
9484 /* Get updated DCBX data from firmware */
9485 ret = i40e_get_dcb_config(&pf->hw);
9486 if (ret) {
9487 /* X710-T*L 2.5G and 5G speeds don't support DCB */
9488 if (I40E_IS_X710TL_DEVICE(hw->device_id) &&
9489 (hw->phy.link_info.link_speed &
9490 (I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB))) {
9491 dev_warn(&pf->pdev->dev,
9492 "DCB is not supported for X710-T*L 2.5/5G speeds\n");
9493 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
9494 } else {
9495 dev_info(&pf->pdev->dev,
9496 "Failed querying DCB configuration data from firmware, err %pe aq_err %s\n",
9497 ERR_PTR(ret),
9498 i40e_aq_str(&pf->hw,
9499 pf->hw.aq.asq_last_status));
9500 }
9501 goto exit;
9502 }
9503
9504 /* No change detected in DCBX configs */
9505 if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config,
9506 sizeof(tmp_dcbx_cfg))) {
9507 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n");
9508 goto exit;
9509 }
9510
9511 need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg,
9512 &hw->local_dcbx_config);
9513
9514 i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config);
9515
9516 if (!need_reconfig)
9517 goto exit;
9518
9519 /* Enable DCB tagging only when more than one TC */
9520 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
9521 set_bit(I40E_FLAG_DCB_ENA, pf->flags);
9522 else
9523 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
9524
9525 set_bit(__I40E_PORT_SUSPENDED, pf->state);
9526 /* Reconfiguration needed quiesce all VSIs */
9527 i40e_pf_quiesce_all_vsi(pf);
9528
9529 /* Changes in configuration update VEB/VSI */
9530 i40e_dcb_reconfigure(pf);
9531
9532 ret = i40e_resume_port_tx(pf);
9533
9534 clear_bit(__I40E_PORT_SUSPENDED, pf->state);
9535 /* In case of error no point in resuming VSIs */
9536 if (ret)
9537 goto exit;
9538
9539 /* Wait for the PF's queues to be disabled */
9540 ret = i40e_pf_wait_queues_disabled(pf);
9541 if (ret) {
9542 /* Schedule PF reset to recover */
9543 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
9544 i40e_service_event_schedule(pf);
9545 } else {
9546 i40e_pf_unquiesce_all_vsi(pf);
9547 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
9548 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);
9549 }
9550
9551 exit:
9552 return ret;
9553 }
9554 #endif /* CONFIG_I40E_DCB */
9555
9556 /**
9557 * i40e_do_reset_safe - Protected reset path for userland calls.
9558 * @pf: board private structure
9559 * @reset_flags: which reset is requested
9560 *
9561 **/
i40e_do_reset_safe(struct i40e_pf * pf,u32 reset_flags)9562 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags)
9563 {
9564 rtnl_lock();
9565 i40e_do_reset(pf, reset_flags, true);
9566 rtnl_unlock();
9567 }
9568
9569 /**
9570 * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event
9571 * @pf: board private structure
9572 * @e: event info posted on ARQ
9573 *
9574 * Handler for LAN Queue Overflow Event generated by the firmware for PF
9575 * and VF queues
9576 **/
i40e_handle_lan_overflow_event(struct i40e_pf * pf,struct i40e_arq_event_info * e)9577 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf,
9578 struct i40e_arq_event_info *e)
9579 {
9580 struct i40e_aqc_lan_overflow *data =
9581 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw;
9582 u32 queue = le32_to_cpu(data->prtdcb_rupto);
9583 u32 qtx_ctl = le32_to_cpu(data->otx_ctl);
9584 struct i40e_hw *hw = &pf->hw;
9585 struct i40e_vf *vf;
9586 u16 vf_id;
9587
9588 dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n",
9589 queue, qtx_ctl);
9590
9591 if (FIELD_GET(I40E_QTX_CTL_PFVF_Q_MASK, qtx_ctl) !=
9592 I40E_QTX_CTL_VF_QUEUE)
9593 return;
9594
9595 /* Queue belongs to VF, find the VF and issue VF reset */
9596 vf_id = FIELD_GET(I40E_QTX_CTL_VFVM_INDX_MASK, qtx_ctl);
9597 vf_id -= hw->func_caps.vf_base_id;
9598 vf = &pf->vf[vf_id];
9599 i40e_vc_notify_vf_reset(vf);
9600 /* Allow VF to process pending reset notification */
9601 msleep(20);
9602 i40e_reset_vf(vf, false);
9603 }
9604
9605 /**
9606 * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters
9607 * @pf: board private structure
9608 **/
i40e_get_cur_guaranteed_fd_count(struct i40e_pf * pf)9609 u32 i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf)
9610 {
9611 u32 val, fcnt_prog;
9612
9613 val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
9614 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK);
9615 return fcnt_prog;
9616 }
9617
9618 /**
9619 * i40e_get_current_fd_count - Get total FD filters programmed for this PF
9620 * @pf: board private structure
9621 **/
i40e_get_current_fd_count(struct i40e_pf * pf)9622 u32 i40e_get_current_fd_count(struct i40e_pf *pf)
9623 {
9624 u32 val, fcnt_prog;
9625
9626 val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
9627 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) +
9628 FIELD_GET(I40E_PFQF_FDSTAT_BEST_CNT_MASK, val);
9629 return fcnt_prog;
9630 }
9631
9632 /**
9633 * i40e_get_global_fd_count - Get total FD filters programmed on device
9634 * @pf: board private structure
9635 **/
i40e_get_global_fd_count(struct i40e_pf * pf)9636 u32 i40e_get_global_fd_count(struct i40e_pf *pf)
9637 {
9638 u32 val, fcnt_prog;
9639
9640 val = rd32(&pf->hw, I40E_GLQF_FDCNT_0);
9641 fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) +
9642 FIELD_GET(I40E_GLQF_FDCNT_0_BESTCNT_MASK, val);
9643 return fcnt_prog;
9644 }
9645
9646 /**
9647 * i40e_reenable_fdir_sb - Restore FDir SB capability
9648 * @pf: board private structure
9649 **/
i40e_reenable_fdir_sb(struct i40e_pf * pf)9650 static void i40e_reenable_fdir_sb(struct i40e_pf *pf)
9651 {
9652 if (test_and_clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state))
9653 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) &&
9654 (I40E_DEBUG_FD & pf->hw.debug_mask))
9655 dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n");
9656 }
9657
9658 /**
9659 * i40e_reenable_fdir_atr - Restore FDir ATR capability
9660 * @pf: board private structure
9661 **/
i40e_reenable_fdir_atr(struct i40e_pf * pf)9662 static void i40e_reenable_fdir_atr(struct i40e_pf *pf)
9663 {
9664 if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) {
9665 /* ATR uses the same filtering logic as SB rules. It only
9666 * functions properly if the input set mask is at the default
9667 * settings. It is safe to restore the default input set
9668 * because there are no active TCPv4 filter rules.
9669 */
9670 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
9671 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
9672 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9673
9674 if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) &&
9675 (I40E_DEBUG_FD & pf->hw.debug_mask))
9676 dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n");
9677 }
9678 }
9679
9680 /**
9681 * i40e_delete_invalid_filter - Delete an invalid FDIR filter
9682 * @pf: board private structure
9683 * @filter: FDir filter to remove
9684 */
i40e_delete_invalid_filter(struct i40e_pf * pf,struct i40e_fdir_filter * filter)9685 static void i40e_delete_invalid_filter(struct i40e_pf *pf,
9686 struct i40e_fdir_filter *filter)
9687 {
9688 /* Update counters */
9689 pf->fdir_pf_active_filters--;
9690 pf->fd_inv = 0;
9691
9692 switch (filter->flow_type) {
9693 case TCP_V4_FLOW:
9694 pf->fd_tcp4_filter_cnt--;
9695 break;
9696 case UDP_V4_FLOW:
9697 pf->fd_udp4_filter_cnt--;
9698 break;
9699 case SCTP_V4_FLOW:
9700 pf->fd_sctp4_filter_cnt--;
9701 break;
9702 case TCP_V6_FLOW:
9703 pf->fd_tcp6_filter_cnt--;
9704 break;
9705 case UDP_V6_FLOW:
9706 pf->fd_udp6_filter_cnt--;
9707 break;
9708 case SCTP_V6_FLOW:
9709 pf->fd_udp6_filter_cnt--;
9710 break;
9711 case IP_USER_FLOW:
9712 switch (filter->ipl4_proto) {
9713 case IPPROTO_TCP:
9714 pf->fd_tcp4_filter_cnt--;
9715 break;
9716 case IPPROTO_UDP:
9717 pf->fd_udp4_filter_cnt--;
9718 break;
9719 case IPPROTO_SCTP:
9720 pf->fd_sctp4_filter_cnt--;
9721 break;
9722 case IPPROTO_IP:
9723 pf->fd_ip4_filter_cnt--;
9724 break;
9725 }
9726 break;
9727 case IPV6_USER_FLOW:
9728 switch (filter->ipl4_proto) {
9729 case IPPROTO_TCP:
9730 pf->fd_tcp6_filter_cnt--;
9731 break;
9732 case IPPROTO_UDP:
9733 pf->fd_udp6_filter_cnt--;
9734 break;
9735 case IPPROTO_SCTP:
9736 pf->fd_sctp6_filter_cnt--;
9737 break;
9738 case IPPROTO_IP:
9739 pf->fd_ip6_filter_cnt--;
9740 break;
9741 }
9742 break;
9743 }
9744
9745 /* Remove the filter from the list and free memory */
9746 hlist_del(&filter->fdir_node);
9747 kfree(filter);
9748 }
9749
9750 /**
9751 * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled
9752 * @pf: board private structure
9753 **/
i40e_fdir_check_and_reenable(struct i40e_pf * pf)9754 void i40e_fdir_check_and_reenable(struct i40e_pf *pf)
9755 {
9756 struct i40e_fdir_filter *filter;
9757 u32 fcnt_prog, fcnt_avail;
9758 struct hlist_node *node;
9759
9760 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
9761 return;
9762
9763 /* Check if we have enough room to re-enable FDir SB capability. */
9764 fcnt_prog = i40e_get_global_fd_count(pf);
9765 fcnt_avail = pf->fdir_pf_filter_count;
9766 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) ||
9767 (pf->fd_add_err == 0) ||
9768 (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt))
9769 i40e_reenable_fdir_sb(pf);
9770
9771 /* We should wait for even more space before re-enabling ATR.
9772 * Additionally, we cannot enable ATR as long as we still have TCP SB
9773 * rules active.
9774 */
9775 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) &&
9776 pf->fd_tcp4_filter_cnt == 0 && pf->fd_tcp6_filter_cnt == 0)
9777 i40e_reenable_fdir_atr(pf);
9778
9779 /* if hw had a problem adding a filter, delete it */
9780 if (pf->fd_inv > 0) {
9781 hlist_for_each_entry_safe(filter, node,
9782 &pf->fdir_filter_list, fdir_node)
9783 if (filter->fd_id == pf->fd_inv)
9784 i40e_delete_invalid_filter(pf, filter);
9785 }
9786 }
9787
9788 #define I40E_MIN_FD_FLUSH_INTERVAL 10
9789 #define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30
9790 /**
9791 * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB
9792 * @pf: board private structure
9793 **/
i40e_fdir_flush_and_replay(struct i40e_pf * pf)9794 static void i40e_fdir_flush_and_replay(struct i40e_pf *pf)
9795 {
9796 unsigned long min_flush_time;
9797 int flush_wait_retry = 50;
9798 bool disable_atr = false;
9799 int fd_room;
9800 int reg;
9801
9802 if (!time_after(jiffies, pf->fd_flush_timestamp +
9803 (I40E_MIN_FD_FLUSH_INTERVAL * HZ)))
9804 return;
9805
9806 /* If the flush is happening too quick and we have mostly SB rules we
9807 * should not re-enable ATR for some time.
9808 */
9809 min_flush_time = pf->fd_flush_timestamp +
9810 (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ);
9811 fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters;
9812
9813 if (!(time_after(jiffies, min_flush_time)) &&
9814 (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) {
9815 if (I40E_DEBUG_FD & pf->hw.debug_mask)
9816 dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n");
9817 disable_atr = true;
9818 }
9819
9820 pf->fd_flush_timestamp = jiffies;
9821 set_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state);
9822 /* flush all filters */
9823 wr32(&pf->hw, I40E_PFQF_CTL_1,
9824 I40E_PFQF_CTL_1_CLEARFDTABLE_MASK);
9825 i40e_flush(&pf->hw);
9826 pf->fd_flush_cnt++;
9827 pf->fd_add_err = 0;
9828 do {
9829 /* Check FD flush status every 5-6msec */
9830 usleep_range(5000, 6000);
9831 reg = rd32(&pf->hw, I40E_PFQF_CTL_1);
9832 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK))
9833 break;
9834 } while (flush_wait_retry--);
9835 if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) {
9836 dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n");
9837 } else {
9838 /* replay sideband filters */
9839 i40e_fdir_filter_restore(i40e_pf_get_main_vsi(pf));
9840 if (!disable_atr && !pf->fd_tcp4_filter_cnt)
9841 clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state);
9842 clear_bit(__I40E_FD_FLUSH_REQUESTED, pf->state);
9843 if (I40E_DEBUG_FD & pf->hw.debug_mask)
9844 dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n");
9845 }
9846 }
9847
9848 /**
9849 * i40e_get_current_atr_cnt - Get the count of total FD ATR filters programmed
9850 * @pf: board private structure
9851 **/
i40e_get_current_atr_cnt(struct i40e_pf * pf)9852 u32 i40e_get_current_atr_cnt(struct i40e_pf *pf)
9853 {
9854 return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters;
9855 }
9856
9857 /**
9858 * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table
9859 * @pf: board private structure
9860 **/
i40e_fdir_reinit_subtask(struct i40e_pf * pf)9861 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf)
9862 {
9863
9864 /* if interface is down do nothing */
9865 if (test_bit(__I40E_DOWN, pf->state))
9866 return;
9867
9868 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
9869 i40e_fdir_flush_and_replay(pf);
9870
9871 i40e_fdir_check_and_reenable(pf);
9872
9873 }
9874
9875 /**
9876 * i40e_vsi_link_event - notify VSI of a link event
9877 * @vsi: vsi to be notified
9878 * @link_up: link up or down
9879 **/
i40e_vsi_link_event(struct i40e_vsi * vsi,bool link_up)9880 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up)
9881 {
9882 if (!vsi || test_bit(__I40E_VSI_DOWN, vsi->state))
9883 return;
9884
9885 switch (vsi->type) {
9886 case I40E_VSI_MAIN:
9887 if (!vsi->netdev || !vsi->netdev_registered)
9888 break;
9889
9890 if (link_up) {
9891 netif_carrier_on(vsi->netdev);
9892 netif_tx_wake_all_queues(vsi->netdev);
9893 } else {
9894 netif_carrier_off(vsi->netdev);
9895 netif_tx_stop_all_queues(vsi->netdev);
9896 }
9897 break;
9898
9899 case I40E_VSI_SRIOV:
9900 case I40E_VSI_VMDQ2:
9901 case I40E_VSI_CTRL:
9902 case I40E_VSI_IWARP:
9903 case I40E_VSI_MIRROR:
9904 default:
9905 /* there is no notification for other VSIs */
9906 break;
9907 }
9908 }
9909
9910 /**
9911 * i40e_veb_link_event - notify elements on the veb of a link event
9912 * @veb: veb to be notified
9913 * @link_up: link up or down
9914 **/
i40e_veb_link_event(struct i40e_veb * veb,bool link_up)9915 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up)
9916 {
9917 struct i40e_vsi *vsi;
9918 struct i40e_pf *pf;
9919 int i;
9920
9921 if (!veb || !veb->pf)
9922 return;
9923 pf = veb->pf;
9924
9925 /* Send link event to contained VSIs */
9926 i40e_pf_for_each_vsi(pf, i, vsi)
9927 if (vsi->uplink_seid == veb->seid)
9928 i40e_vsi_link_event(vsi, link_up);
9929 }
9930
9931 /**
9932 * i40e_link_event - Update netif_carrier status
9933 * @pf: board private structure
9934 **/
i40e_link_event(struct i40e_pf * pf)9935 static void i40e_link_event(struct i40e_pf *pf)
9936 {
9937 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);
9938 struct i40e_veb *veb = i40e_pf_get_main_veb(pf);
9939 u8 new_link_speed, old_link_speed;
9940 bool new_link, old_link;
9941 int status;
9942 #ifdef CONFIG_I40E_DCB
9943 int err;
9944 #endif /* CONFIG_I40E_DCB */
9945
9946 /* set this to force the get_link_status call to refresh state */
9947 pf->hw.phy.get_link_info = true;
9948 old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP);
9949 status = i40e_get_link_status(&pf->hw, &new_link);
9950
9951 /* On success, disable temp link polling */
9952 if (status == 0) {
9953 clear_bit(__I40E_TEMP_LINK_POLLING, pf->state);
9954 } else {
9955 /* Enable link polling temporarily until i40e_get_link_status
9956 * returns 0
9957 */
9958 set_bit(__I40E_TEMP_LINK_POLLING, pf->state);
9959 dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n",
9960 status);
9961 return;
9962 }
9963
9964 old_link_speed = pf->hw.phy.link_info_old.link_speed;
9965 new_link_speed = pf->hw.phy.link_info.link_speed;
9966
9967 if (new_link == old_link &&
9968 new_link_speed == old_link_speed &&
9969 (test_bit(__I40E_VSI_DOWN, vsi->state) ||
9970 new_link == netif_carrier_ok(vsi->netdev)))
9971 return;
9972
9973 i40e_print_link_message(vsi, new_link);
9974
9975 /* Notify the base of the switch tree connected to
9976 * the link. Floating VEBs are not notified.
9977 */
9978 if (veb)
9979 i40e_veb_link_event(veb, new_link);
9980 else
9981 i40e_vsi_link_event(vsi, new_link);
9982
9983 if (pf->vf)
9984 i40e_vc_notify_link_state(pf);
9985
9986 if (test_bit(I40E_FLAG_PTP_ENA, pf->flags))
9987 i40e_ptp_set_increment(pf);
9988 #ifdef CONFIG_I40E_DCB
9989 if (new_link == old_link)
9990 return;
9991 /* Not SW DCB so firmware will take care of default settings */
9992 if (pf->dcbx_cap & DCB_CAP_DCBX_LLD_MANAGED)
9993 return;
9994
9995 /* We cover here only link down, as after link up in case of SW DCB
9996 * SW LLDP agent will take care of setting it up
9997 */
9998 if (!new_link) {
9999 dev_dbg(&pf->pdev->dev, "Reconfig DCB to single TC as result of Link Down\n");
10000 memset(&pf->tmp_cfg, 0, sizeof(pf->tmp_cfg));
10001 err = i40e_dcb_sw_default_config(pf);
10002 if (err) {
10003 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
10004 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
10005 } else {
10006 pf->dcbx_cap = DCB_CAP_DCBX_HOST |
10007 DCB_CAP_DCBX_VER_IEEE;
10008 set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
10009 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
10010 }
10011 }
10012 #endif /* CONFIG_I40E_DCB */
10013 }
10014
10015 /**
10016 * i40e_watchdog_subtask - periodic checks not using event driven response
10017 * @pf: board private structure
10018 **/
i40e_watchdog_subtask(struct i40e_pf * pf)10019 static void i40e_watchdog_subtask(struct i40e_pf *pf)
10020 {
10021 struct i40e_vsi *vsi;
10022 struct i40e_veb *veb;
10023 int i;
10024
10025 /* if interface is down do nothing */
10026 if (test_bit(__I40E_DOWN, pf->state) ||
10027 test_bit(__I40E_CONFIG_BUSY, pf->state))
10028 return;
10029
10030 /* make sure we don't do these things too often */
10031 if (time_before(jiffies, (pf->service_timer_previous +
10032 pf->service_timer_period)))
10033 return;
10034 pf->service_timer_previous = jiffies;
10035
10036 if (test_bit(I40E_FLAG_LINK_POLLING_ENA, pf->flags) ||
10037 test_bit(__I40E_TEMP_LINK_POLLING, pf->state))
10038 i40e_link_event(pf);
10039
10040 /* Update the stats for active netdevs so the network stack
10041 * can look at updated numbers whenever it cares to
10042 */
10043 i40e_pf_for_each_vsi(pf, i, vsi)
10044 if (vsi->netdev)
10045 i40e_update_stats(vsi);
10046
10047 if (test_bit(I40E_FLAG_VEB_STATS_ENA, pf->flags)) {
10048 /* Update the stats for the active switching components */
10049 i40e_pf_for_each_veb(pf, i, veb)
10050 i40e_update_veb_stats(veb);
10051 }
10052
10053 i40e_ptp_rx_hang(pf);
10054 i40e_ptp_tx_hang(pf);
10055 }
10056
10057 /**
10058 * i40e_reset_subtask - Set up for resetting the device and driver
10059 * @pf: board private structure
10060 **/
i40e_reset_subtask(struct i40e_pf * pf)10061 static void i40e_reset_subtask(struct i40e_pf *pf)
10062 {
10063 u32 reset_flags = 0;
10064
10065 if (test_bit(__I40E_REINIT_REQUESTED, pf->state)) {
10066 reset_flags |= BIT(__I40E_REINIT_REQUESTED);
10067 clear_bit(__I40E_REINIT_REQUESTED, pf->state);
10068 }
10069 if (test_bit(__I40E_PF_RESET_REQUESTED, pf->state)) {
10070 reset_flags |= BIT(__I40E_PF_RESET_REQUESTED);
10071 clear_bit(__I40E_PF_RESET_REQUESTED, pf->state);
10072 }
10073 if (test_bit(__I40E_CORE_RESET_REQUESTED, pf->state)) {
10074 reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED);
10075 clear_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
10076 }
10077 if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state)) {
10078 reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED);
10079 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);
10080 }
10081 if (test_bit(__I40E_DOWN_REQUESTED, pf->state)) {
10082 reset_flags |= BIT(__I40E_DOWN_REQUESTED);
10083 clear_bit(__I40E_DOWN_REQUESTED, pf->state);
10084 }
10085
10086 /* If there's a recovery already waiting, it takes
10087 * precedence before starting a new reset sequence.
10088 */
10089 if (test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) {
10090 i40e_prep_for_reset(pf);
10091 i40e_reset(pf);
10092 i40e_rebuild(pf, false, false);
10093 }
10094
10095 /* If we're already down or resetting, just bail */
10096 if (reset_flags &&
10097 !test_bit(__I40E_DOWN, pf->state) &&
10098 !test_bit(__I40E_CONFIG_BUSY, pf->state)) {
10099 i40e_do_reset(pf, reset_flags, false);
10100 }
10101 }
10102
10103 /**
10104 * i40e_handle_link_event - Handle link event
10105 * @pf: board private structure
10106 * @e: event info posted on ARQ
10107 **/
i40e_handle_link_event(struct i40e_pf * pf,struct i40e_arq_event_info * e)10108 static void i40e_handle_link_event(struct i40e_pf *pf,
10109 struct i40e_arq_event_info *e)
10110 {
10111 struct i40e_aqc_get_link_status *status =
10112 (struct i40e_aqc_get_link_status *)&e->desc.params.raw;
10113
10114 /* Do a new status request to re-enable LSE reporting
10115 * and load new status information into the hw struct
10116 * This completely ignores any state information
10117 * in the ARQ event info, instead choosing to always
10118 * issue the AQ update link status command.
10119 */
10120 i40e_link_event(pf);
10121
10122 /* Check if module meets thermal requirements */
10123 if (status->phy_type == I40E_PHY_TYPE_NOT_SUPPORTED_HIGH_TEMP) {
10124 dev_err(&pf->pdev->dev,
10125 "Rx/Tx is disabled on this device because the module does not meet thermal requirements.\n");
10126 dev_err(&pf->pdev->dev,
10127 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");
10128 } else {
10129 /* check for unqualified module, if link is down, suppress
10130 * the message if link was forced to be down.
10131 */
10132 if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&
10133 (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&
10134 (!(status->link_info & I40E_AQ_LINK_UP)) &&
10135 (!test_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags))) {
10136 dev_err(&pf->pdev->dev,
10137 "Rx/Tx is disabled on this device because an unsupported SFP module type was detected.\n");
10138 dev_err(&pf->pdev->dev,
10139 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");
10140 }
10141 }
10142 }
10143
10144 /**
10145 * i40e_clean_adminq_subtask - Clean the AdminQ rings
10146 * @pf: board private structure
10147 **/
i40e_clean_adminq_subtask(struct i40e_pf * pf)10148 static void i40e_clean_adminq_subtask(struct i40e_pf *pf)
10149 {
10150 struct i40e_arq_event_info event;
10151 struct i40e_hw *hw = &pf->hw;
10152 u16 pending, i = 0;
10153 u16 opcode;
10154 u32 oldval;
10155 int ret;
10156 u32 val;
10157
10158 /* Do not run clean AQ when PF reset fails */
10159 if (test_bit(__I40E_RESET_FAILED, pf->state))
10160 return;
10161
10162 /* check for error indications */
10163 val = rd32(&pf->hw, I40E_PF_ARQLEN);
10164 oldval = val;
10165 if (val & I40E_PF_ARQLEN_ARQVFE_MASK) {
10166 if (hw->debug_mask & I40E_DEBUG_AQ)
10167 dev_info(&pf->pdev->dev, "ARQ VF Error detected\n");
10168 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK;
10169 }
10170 if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) {
10171 if (hw->debug_mask & I40E_DEBUG_AQ)
10172 dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n");
10173 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK;
10174 pf->arq_overflows++;
10175 }
10176 if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) {
10177 if (hw->debug_mask & I40E_DEBUG_AQ)
10178 dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n");
10179 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK;
10180 }
10181 if (oldval != val)
10182 wr32(&pf->hw, I40E_PF_ARQLEN, val);
10183
10184 val = rd32(&pf->hw, I40E_PF_ATQLEN);
10185 oldval = val;
10186 if (val & I40E_PF_ATQLEN_ATQVFE_MASK) {
10187 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
10188 dev_info(&pf->pdev->dev, "ASQ VF Error detected\n");
10189 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK;
10190 }
10191 if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) {
10192 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
10193 dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n");
10194 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK;
10195 }
10196 if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) {
10197 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
10198 dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n");
10199 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK;
10200 }
10201 if (oldval != val)
10202 wr32(&pf->hw, I40E_PF_ATQLEN, val);
10203
10204 event.buf_len = I40E_MAX_AQ_BUF_SIZE;
10205 event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);
10206 if (!event.msg_buf)
10207 return;
10208
10209 do {
10210 ret = i40e_clean_arq_element(hw, &event, &pending);
10211 if (ret == -EALREADY)
10212 break;
10213 else if (ret) {
10214 dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret);
10215 break;
10216 }
10217
10218 opcode = le16_to_cpu(event.desc.opcode);
10219 switch (opcode) {
10220
10221 case i40e_aqc_opc_get_link_status:
10222 rtnl_lock();
10223 i40e_handle_link_event(pf, &event);
10224 rtnl_unlock();
10225 break;
10226 case i40e_aqc_opc_send_msg_to_pf:
10227 ret = i40e_vc_process_vf_msg(pf,
10228 le16_to_cpu(event.desc.retval),
10229 le32_to_cpu(event.desc.cookie_high),
10230 le32_to_cpu(event.desc.cookie_low),
10231 event.msg_buf,
10232 event.msg_len);
10233 break;
10234 case i40e_aqc_opc_lldp_update_mib:
10235 dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n");
10236 #ifdef CONFIG_I40E_DCB
10237 rtnl_lock();
10238 i40e_handle_lldp_event(pf, &event);
10239 rtnl_unlock();
10240 #endif /* CONFIG_I40E_DCB */
10241 break;
10242 case i40e_aqc_opc_event_lan_overflow:
10243 dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n");
10244 i40e_handle_lan_overflow_event(pf, &event);
10245 break;
10246 case i40e_aqc_opc_send_msg_to_peer:
10247 dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n");
10248 break;
10249 case i40e_aqc_opc_nvm_erase:
10250 case i40e_aqc_opc_nvm_update:
10251 case i40e_aqc_opc_oem_post_update:
10252 i40e_debug(&pf->hw, I40E_DEBUG_NVM,
10253 "ARQ NVM operation 0x%04x completed\n",
10254 opcode);
10255 break;
10256 default:
10257 dev_info(&pf->pdev->dev,
10258 "ARQ: Unknown event 0x%04x ignored\n",
10259 opcode);
10260 break;
10261 }
10262 } while (i++ < I40E_AQ_WORK_LIMIT);
10263
10264 if (i < I40E_AQ_WORK_LIMIT)
10265 clear_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);
10266
10267 /* re-enable Admin queue interrupt cause */
10268 val = rd32(hw, I40E_PFINT_ICR0_ENA);
10269 val |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
10270 wr32(hw, I40E_PFINT_ICR0_ENA, val);
10271 i40e_flush(hw);
10272
10273 kfree(event.msg_buf);
10274 }
10275
10276 /**
10277 * i40e_verify_eeprom - make sure eeprom is good to use
10278 * @pf: board private structure
10279 **/
i40e_verify_eeprom(struct i40e_pf * pf)10280 static void i40e_verify_eeprom(struct i40e_pf *pf)
10281 {
10282 int err;
10283
10284 err = i40e_diag_eeprom_test(&pf->hw);
10285 if (err) {
10286 /* retry in case of garbage read */
10287 err = i40e_diag_eeprom_test(&pf->hw);
10288 if (err) {
10289 dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n",
10290 err);
10291 set_bit(__I40E_BAD_EEPROM, pf->state);
10292 }
10293 }
10294
10295 if (!err && test_bit(__I40E_BAD_EEPROM, pf->state)) {
10296 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n");
10297 clear_bit(__I40E_BAD_EEPROM, pf->state);
10298 }
10299 }
10300
10301 /**
10302 * i40e_enable_pf_switch_lb
10303 * @pf: pointer to the PF structure
10304 *
10305 * enable switch loop back or die - no point in a return value
10306 **/
i40e_enable_pf_switch_lb(struct i40e_pf * pf)10307 static void i40e_enable_pf_switch_lb(struct i40e_pf *pf)
10308 {
10309 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);
10310 struct i40e_vsi_context ctxt;
10311 int ret;
10312
10313 ctxt.seid = pf->main_vsi_seid;
10314 ctxt.pf_num = pf->hw.pf_id;
10315 ctxt.vf_num = 0;
10316 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
10317 if (ret) {
10318 dev_info(&pf->pdev->dev,
10319 "couldn't get PF vsi config, err %pe aq_err %s\n",
10320 ERR_PTR(ret),
10321 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10322 return;
10323 }
10324 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
10325 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
10326 ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
10327
10328 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
10329 if (ret) {
10330 dev_info(&pf->pdev->dev,
10331 "update vsi switch failed, err %pe aq_err %s\n",
10332 ERR_PTR(ret),
10333 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10334 }
10335 }
10336
10337 /**
10338 * i40e_disable_pf_switch_lb
10339 * @pf: pointer to the PF structure
10340 *
10341 * disable switch loop back or die - no point in a return value
10342 **/
i40e_disable_pf_switch_lb(struct i40e_pf * pf)10343 static void i40e_disable_pf_switch_lb(struct i40e_pf *pf)
10344 {
10345 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);
10346 struct i40e_vsi_context ctxt;
10347 int ret;
10348
10349 ctxt.seid = pf->main_vsi_seid;
10350 ctxt.pf_num = pf->hw.pf_id;
10351 ctxt.vf_num = 0;
10352 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
10353 if (ret) {
10354 dev_info(&pf->pdev->dev,
10355 "couldn't get PF vsi config, err %pe aq_err %s\n",
10356 ERR_PTR(ret),
10357 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10358 return;
10359 }
10360 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
10361 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
10362 ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
10363
10364 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
10365 if (ret) {
10366 dev_info(&pf->pdev->dev,
10367 "update vsi switch failed, err %pe aq_err %s\n",
10368 ERR_PTR(ret),
10369 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10370 }
10371 }
10372
10373 /**
10374 * i40e_config_bridge_mode - Configure the HW bridge mode
10375 * @veb: pointer to the bridge instance
10376 *
10377 * Configure the loop back mode for the LAN VSI that is downlink to the
10378 * specified HW bridge instance. It is expected this function is called
10379 * when a new HW bridge is instantiated.
10380 **/
i40e_config_bridge_mode(struct i40e_veb * veb)10381 static void i40e_config_bridge_mode(struct i40e_veb *veb)
10382 {
10383 struct i40e_pf *pf = veb->pf;
10384
10385 if (pf->hw.debug_mask & I40E_DEBUG_LAN)
10386 dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n",
10387 veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
10388 if (veb->bridge_mode & BRIDGE_MODE_VEPA)
10389 i40e_disable_pf_switch_lb(pf);
10390 else
10391 i40e_enable_pf_switch_lb(pf);
10392 }
10393
10394 /**
10395 * i40e_reconstitute_veb - rebuild the VEB and VSIs connected to it
10396 * @veb: pointer to the VEB instance
10397 *
10398 * This is a function that builds the attached VSIs. We track the connections
10399 * through our own index numbers because the seid's from the HW could change
10400 * across the reset.
10401 **/
i40e_reconstitute_veb(struct i40e_veb * veb)10402 static int i40e_reconstitute_veb(struct i40e_veb *veb)
10403 {
10404 struct i40e_vsi *ctl_vsi = NULL;
10405 struct i40e_pf *pf = veb->pf;
10406 struct i40e_vsi *vsi;
10407 int v, ret;
10408
10409 /* As we do not maintain PV (port virtualizer) switch element then
10410 * there can be only one non-floating VEB that have uplink to MAC SEID
10411 * and its control VSI is the main one.
10412 */
10413 if (WARN_ON(veb->uplink_seid && veb->uplink_seid != pf->mac_seid)) {
10414 dev_err(&pf->pdev->dev,
10415 "Invalid uplink SEID for VEB %d\n", veb->idx);
10416 return -ENOENT;
10417 }
10418
10419 if (veb->uplink_seid == pf->mac_seid) {
10420 /* Check that the LAN VSI has VEB owning flag set */
10421 ctl_vsi = i40e_pf_get_main_vsi(pf);
10422
10423 if (WARN_ON(ctl_vsi->veb_idx != veb->idx ||
10424 !(ctl_vsi->flags & I40E_VSI_FLAG_VEB_OWNER))) {
10425 dev_err(&pf->pdev->dev,
10426 "Invalid control VSI for VEB %d\n", veb->idx);
10427 return -ENOENT;
10428 }
10429
10430 /* Add the control VSI to switch */
10431 ret = i40e_add_vsi(ctl_vsi);
10432 if (ret) {
10433 dev_err(&pf->pdev->dev,
10434 "Rebuild of owner VSI for VEB %d failed: %d\n",
10435 veb->idx, ret);
10436 return ret;
10437 }
10438
10439 i40e_vsi_reset_stats(ctl_vsi);
10440 }
10441
10442 /* create the VEB in the switch and move the VSI onto the VEB */
10443 ret = i40e_add_veb(veb, ctl_vsi);
10444 if (ret)
10445 return ret;
10446
10447 if (veb->uplink_seid) {
10448 if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags))
10449 veb->bridge_mode = BRIDGE_MODE_VEB;
10450 else
10451 veb->bridge_mode = BRIDGE_MODE_VEPA;
10452 i40e_config_bridge_mode(veb);
10453 }
10454
10455 /* create the remaining VSIs attached to this VEB */
10456 i40e_pf_for_each_vsi(pf, v, vsi) {
10457 if (vsi == ctl_vsi)
10458 continue;
10459
10460 if (vsi->veb_idx == veb->idx) {
10461 vsi->uplink_seid = veb->seid;
10462 ret = i40e_add_vsi(vsi);
10463 if (ret) {
10464 dev_info(&pf->pdev->dev,
10465 "rebuild of vsi_idx %d failed: %d\n",
10466 v, ret);
10467 return ret;
10468 }
10469 i40e_vsi_reset_stats(vsi);
10470 }
10471 }
10472
10473 return ret;
10474 }
10475
10476 /**
10477 * i40e_get_capabilities - get info about the HW
10478 * @pf: the PF struct
10479 * @list_type: AQ capability to be queried
10480 **/
i40e_get_capabilities(struct i40e_pf * pf,enum i40e_admin_queue_opc list_type)10481 static int i40e_get_capabilities(struct i40e_pf *pf,
10482 enum i40e_admin_queue_opc list_type)
10483 {
10484 struct i40e_aqc_list_capabilities_element_resp *cap_buf;
10485 u16 data_size;
10486 int buf_len;
10487 int err;
10488
10489 buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
10490 do {
10491 cap_buf = kzalloc(buf_len, GFP_KERNEL);
10492 if (!cap_buf)
10493 return -ENOMEM;
10494
10495 /* this loads the data into the hw struct for us */
10496 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len,
10497 &data_size, list_type,
10498 NULL);
10499 /* data loaded, buffer no longer needed */
10500 kfree(cap_buf);
10501
10502 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) {
10503 /* retry with a larger buffer */
10504 buf_len = data_size;
10505 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK || err) {
10506 dev_info(&pf->pdev->dev,
10507 "capability discovery failed, err %pe aq_err %s\n",
10508 ERR_PTR(err),
10509 i40e_aq_str(&pf->hw,
10510 pf->hw.aq.asq_last_status));
10511 return -ENODEV;
10512 }
10513 } while (err);
10514
10515 if (pf->hw.debug_mask & I40E_DEBUG_USER) {
10516 if (list_type == i40e_aqc_opc_list_func_capabilities) {
10517 dev_info(&pf->pdev->dev,
10518 "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n",
10519 pf->hw.pf_id, pf->hw.func_caps.num_vfs,
10520 pf->hw.func_caps.num_msix_vectors,
10521 pf->hw.func_caps.num_msix_vectors_vf,
10522 pf->hw.func_caps.fd_filters_guaranteed,
10523 pf->hw.func_caps.fd_filters_best_effort,
10524 pf->hw.func_caps.num_tx_qp,
10525 pf->hw.func_caps.num_vsis);
10526 } else if (list_type == i40e_aqc_opc_list_dev_capabilities) {
10527 dev_info(&pf->pdev->dev,
10528 "switch_mode=0x%04x, function_valid=0x%08x\n",
10529 pf->hw.dev_caps.switch_mode,
10530 pf->hw.dev_caps.valid_functions);
10531 dev_info(&pf->pdev->dev,
10532 "SR-IOV=%d, num_vfs for all function=%u\n",
10533 pf->hw.dev_caps.sr_iov_1_1,
10534 pf->hw.dev_caps.num_vfs);
10535 dev_info(&pf->pdev->dev,
10536 "num_vsis=%u, num_rx:%u, num_tx=%u\n",
10537 pf->hw.dev_caps.num_vsis,
10538 pf->hw.dev_caps.num_rx_qp,
10539 pf->hw.dev_caps.num_tx_qp);
10540 }
10541 }
10542 if (list_type == i40e_aqc_opc_list_func_capabilities) {
10543 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \
10544 + pf->hw.func_caps.num_vfs)
10545 if (pf->hw.revision_id == 0 &&
10546 pf->hw.func_caps.num_vsis < DEF_NUM_VSI) {
10547 dev_info(&pf->pdev->dev,
10548 "got num_vsis %d, setting num_vsis to %d\n",
10549 pf->hw.func_caps.num_vsis, DEF_NUM_VSI);
10550 pf->hw.func_caps.num_vsis = DEF_NUM_VSI;
10551 }
10552 }
10553 return 0;
10554 }
10555
10556 static int i40e_vsi_clear(struct i40e_vsi *vsi);
10557
10558 /**
10559 * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband
10560 * @pf: board private structure
10561 **/
i40e_fdir_sb_setup(struct i40e_pf * pf)10562 static void i40e_fdir_sb_setup(struct i40e_pf *pf)
10563 {
10564 struct i40e_vsi *main_vsi, *vsi;
10565
10566 /* quick workaround for an NVM issue that leaves a critical register
10567 * uninitialized
10568 */
10569 if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) {
10570 static const u32 hkey[] = {
10571 0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36,
10572 0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb,
10573 0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21,
10574 0x95b3a76d};
10575 int i;
10576
10577 for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++)
10578 wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]);
10579 }
10580
10581 if (!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags))
10582 return;
10583
10584 /* find existing VSI and see if it needs configuring */
10585 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
10586
10587 /* create a new VSI if none exists */
10588 if (!vsi) {
10589 main_vsi = i40e_pf_get_main_vsi(pf);
10590 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR, main_vsi->seid, 0);
10591 if (!vsi) {
10592 dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n");
10593 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
10594 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
10595 return;
10596 }
10597 }
10598
10599 i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring);
10600 }
10601
10602 /**
10603 * i40e_fdir_teardown - release the Flow Director resources
10604 * @pf: board private structure
10605 **/
i40e_fdir_teardown(struct i40e_pf * pf)10606 static void i40e_fdir_teardown(struct i40e_pf *pf)
10607 {
10608 struct i40e_vsi *vsi;
10609
10610 i40e_fdir_filter_exit(pf);
10611 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
10612 if (vsi)
10613 i40e_vsi_release(vsi);
10614 }
10615
10616 /**
10617 * i40e_rebuild_cloud_filters - Rebuilds cloud filters for VSIs
10618 * @vsi: PF main vsi
10619 * @seid: seid of main or channel VSIs
10620 *
10621 * Rebuilds cloud filters associated with main VSI and channel VSIs if they
10622 * existed before reset
10623 **/
i40e_rebuild_cloud_filters(struct i40e_vsi * vsi,u16 seid)10624 static int i40e_rebuild_cloud_filters(struct i40e_vsi *vsi, u16 seid)
10625 {
10626 struct i40e_cloud_filter *cfilter;
10627 struct i40e_pf *pf = vsi->back;
10628 struct hlist_node *node;
10629 int ret;
10630
10631 /* Add cloud filters back if they exist */
10632 hlist_for_each_entry_safe(cfilter, node, &pf->cloud_filter_list,
10633 cloud_node) {
10634 if (cfilter->seid != seid)
10635 continue;
10636
10637 if (cfilter->dst_port)
10638 ret = i40e_add_del_cloud_filter_big_buf(vsi, cfilter,
10639 true);
10640 else
10641 ret = i40e_add_del_cloud_filter(vsi, cfilter, true);
10642
10643 if (ret) {
10644 dev_dbg(&pf->pdev->dev,
10645 "Failed to rebuild cloud filter, err %pe aq_err %s\n",
10646 ERR_PTR(ret),
10647 i40e_aq_str(&pf->hw,
10648 pf->hw.aq.asq_last_status));
10649 return ret;
10650 }
10651 }
10652 return 0;
10653 }
10654
10655 /**
10656 * i40e_rebuild_channels - Rebuilds channel VSIs if they existed before reset
10657 * @vsi: PF main vsi
10658 *
10659 * Rebuilds channel VSIs if they existed before reset
10660 **/
i40e_rebuild_channels(struct i40e_vsi * vsi)10661 static int i40e_rebuild_channels(struct i40e_vsi *vsi)
10662 {
10663 struct i40e_channel *ch, *ch_tmp;
10664 int ret;
10665
10666 if (list_empty(&vsi->ch_list))
10667 return 0;
10668
10669 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
10670 if (!ch->initialized)
10671 break;
10672 /* Proceed with creation of channel (VMDq2) VSI */
10673 ret = i40e_add_channel(vsi->back, vsi->uplink_seid, ch);
10674 if (ret) {
10675 dev_info(&vsi->back->pdev->dev,
10676 "failed to rebuild channels using uplink_seid %u\n",
10677 vsi->uplink_seid);
10678 return ret;
10679 }
10680 /* Reconfigure TX queues using QTX_CTL register */
10681 ret = i40e_channel_config_tx_ring(vsi->back, vsi, ch);
10682 if (ret) {
10683 dev_info(&vsi->back->pdev->dev,
10684 "failed to configure TX rings for channel %u\n",
10685 ch->seid);
10686 return ret;
10687 }
10688 /* update 'next_base_queue' */
10689 vsi->next_base_queue = vsi->next_base_queue +
10690 ch->num_queue_pairs;
10691 if (ch->max_tx_rate) {
10692 u64 credits = ch->max_tx_rate;
10693
10694 if (i40e_set_bw_limit(vsi, ch->seid,
10695 ch->max_tx_rate))
10696 return -EINVAL;
10697
10698 do_div(credits, I40E_BW_CREDIT_DIVISOR);
10699 dev_dbg(&vsi->back->pdev->dev,
10700 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
10701 ch->max_tx_rate,
10702 credits,
10703 ch->seid);
10704 }
10705 ret = i40e_rebuild_cloud_filters(vsi, ch->seid);
10706 if (ret) {
10707 dev_dbg(&vsi->back->pdev->dev,
10708 "Failed to rebuild cloud filters for channel VSI %u\n",
10709 ch->seid);
10710 return ret;
10711 }
10712 }
10713 return 0;
10714 }
10715
10716 /**
10717 * i40e_clean_xps_state - clean xps state for every tx_ring
10718 * @vsi: ptr to the VSI
10719 **/
i40e_clean_xps_state(struct i40e_vsi * vsi)10720 static void i40e_clean_xps_state(struct i40e_vsi *vsi)
10721 {
10722 int i;
10723
10724 if (vsi->tx_rings)
10725 for (i = 0; i < vsi->num_queue_pairs; i++)
10726 if (vsi->tx_rings[i])
10727 clear_bit(__I40E_TX_XPS_INIT_DONE,
10728 vsi->tx_rings[i]->state);
10729 }
10730
10731 /**
10732 * i40e_prep_for_reset - prep for the core to reset
10733 * @pf: board private structure
10734 *
10735 * Close up the VFs and other things in prep for PF Reset.
10736 **/
i40e_prep_for_reset(struct i40e_pf * pf)10737 static void i40e_prep_for_reset(struct i40e_pf *pf)
10738 {
10739 struct i40e_hw *hw = &pf->hw;
10740 struct i40e_vsi *vsi;
10741 int ret = 0;
10742 u32 v;
10743
10744 clear_bit(__I40E_RESET_INTR_RECEIVED, pf->state);
10745 if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
10746 return;
10747 if (i40e_check_asq_alive(&pf->hw))
10748 i40e_vc_notify_reset(pf);
10749
10750 dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n");
10751
10752 /* quiesce the VSIs and their queues that are not already DOWN */
10753 i40e_pf_quiesce_all_vsi(pf);
10754
10755 i40e_pf_for_each_vsi(pf, v, vsi) {
10756 i40e_clean_xps_state(vsi);
10757 vsi->seid = 0;
10758 }
10759
10760 i40e_shutdown_adminq(&pf->hw);
10761
10762 /* call shutdown HMC */
10763 if (hw->hmc.hmc_obj) {
10764 ret = i40e_shutdown_lan_hmc(hw);
10765 if (ret)
10766 dev_warn(&pf->pdev->dev,
10767 "shutdown_lan_hmc failed: %d\n", ret);
10768 }
10769
10770 /* Save the current PTP time so that we can restore the time after the
10771 * reset completes.
10772 */
10773 i40e_ptp_save_hw_time(pf);
10774 }
10775
10776 /**
10777 * i40e_send_version - update firmware with driver version
10778 * @pf: PF struct
10779 */
i40e_send_version(struct i40e_pf * pf)10780 static void i40e_send_version(struct i40e_pf *pf)
10781 {
10782 struct i40e_driver_version dv;
10783
10784 dv.major_version = 0xff;
10785 dv.minor_version = 0xff;
10786 dv.build_version = 0xff;
10787 dv.subbuild_version = 0;
10788 strscpy(dv.driver_string, UTS_RELEASE, sizeof(dv.driver_string));
10789 i40e_aq_send_driver_version(&pf->hw, &dv, NULL);
10790 }
10791
10792 /**
10793 * i40e_get_oem_version - get OEM specific version information
10794 * @hw: pointer to the hardware structure
10795 **/
i40e_get_oem_version(struct i40e_hw * hw)10796 static void i40e_get_oem_version(struct i40e_hw *hw)
10797 {
10798 u16 block_offset = 0xffff;
10799 u16 block_length = 0;
10800 u16 capabilities = 0;
10801 u16 gen_snap = 0;
10802 u16 release = 0;
10803
10804 #define I40E_SR_NVM_OEM_VERSION_PTR 0x1B
10805 #define I40E_NVM_OEM_LENGTH_OFFSET 0x00
10806 #define I40E_NVM_OEM_CAPABILITIES_OFFSET 0x01
10807 #define I40E_NVM_OEM_GEN_OFFSET 0x02
10808 #define I40E_NVM_OEM_RELEASE_OFFSET 0x03
10809 #define I40E_NVM_OEM_CAPABILITIES_MASK 0x000F
10810 #define I40E_NVM_OEM_LENGTH 3
10811
10812 /* Check if pointer to OEM version block is valid. */
10813 i40e_read_nvm_word(hw, I40E_SR_NVM_OEM_VERSION_PTR, &block_offset);
10814 if (block_offset == 0xffff)
10815 return;
10816
10817 /* Check if OEM version block has correct length. */
10818 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_LENGTH_OFFSET,
10819 &block_length);
10820 if (block_length < I40E_NVM_OEM_LENGTH)
10821 return;
10822
10823 /* Check if OEM version format is as expected. */
10824 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_CAPABILITIES_OFFSET,
10825 &capabilities);
10826 if ((capabilities & I40E_NVM_OEM_CAPABILITIES_MASK) != 0)
10827 return;
10828
10829 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_GEN_OFFSET,
10830 &gen_snap);
10831 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_RELEASE_OFFSET,
10832 &release);
10833 hw->nvm.oem_ver =
10834 FIELD_PREP(I40E_OEM_GEN_MASK | I40E_OEM_SNAP_MASK, gen_snap) |
10835 FIELD_PREP(I40E_OEM_RELEASE_MASK, release);
10836 hw->nvm.eetrack = I40E_OEM_EETRACK_ID;
10837 }
10838
10839 /**
10840 * i40e_reset - wait for core reset to finish reset, reset pf if corer not seen
10841 * @pf: board private structure
10842 **/
i40e_reset(struct i40e_pf * pf)10843 static int i40e_reset(struct i40e_pf *pf)
10844 {
10845 struct i40e_hw *hw = &pf->hw;
10846 int ret;
10847
10848 ret = i40e_pf_reset(hw);
10849 if (ret) {
10850 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret);
10851 set_bit(__I40E_RESET_FAILED, pf->state);
10852 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);
10853 } else {
10854 pf->pfr_count++;
10855 }
10856 return ret;
10857 }
10858
10859 /**
10860 * i40e_rebuild - rebuild using a saved config
10861 * @pf: board private structure
10862 * @reinit: if the Main VSI needs to re-initialized.
10863 * @lock_acquired: indicates whether or not the lock has been acquired
10864 * before this function was called.
10865 **/
i40e_rebuild(struct i40e_pf * pf,bool reinit,bool lock_acquired)10866 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired)
10867 {
10868 const bool is_recovery_mode_reported = i40e_check_recovery_mode(pf);
10869 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);
10870 struct i40e_hw *hw = &pf->hw;
10871 struct i40e_veb *veb;
10872 int ret;
10873 u32 val;
10874 int v;
10875
10876 if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) &&
10877 is_recovery_mode_reported)
10878 i40e_set_ethtool_ops(vsi->netdev);
10879
10880 if (test_bit(__I40E_DOWN, pf->state) &&
10881 !test_bit(__I40E_RECOVERY_MODE, pf->state))
10882 goto clear_recovery;
10883 dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n");
10884
10885 /* rebuild the basics for the AdminQ, HMC, and initial HW switch */
10886 ret = i40e_init_adminq(&pf->hw);
10887 if (ret) {
10888 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %pe aq_err %s\n",
10889 ERR_PTR(ret),
10890 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10891 goto clear_recovery;
10892 }
10893 i40e_get_oem_version(&pf->hw);
10894
10895 if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state)) {
10896 /* The following delay is necessary for firmware update. */
10897 mdelay(1000);
10898 }
10899
10900 /* re-verify the eeprom if we just had an EMP reset */
10901 if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state))
10902 i40e_verify_eeprom(pf);
10903
10904 /* if we are going out of or into recovery mode we have to act
10905 * accordingly with regard to resources initialization
10906 * and deinitialization
10907 */
10908 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) {
10909 if (i40e_get_capabilities(pf,
10910 i40e_aqc_opc_list_func_capabilities))
10911 goto end_unlock;
10912
10913 if (is_recovery_mode_reported) {
10914 /* we're staying in recovery mode so we'll reinitialize
10915 * misc vector here
10916 */
10917 if (i40e_setup_misc_vector_for_recovery_mode(pf))
10918 goto end_unlock;
10919 } else {
10920 if (!lock_acquired)
10921 rtnl_lock();
10922 /* we're going out of recovery mode so we'll free
10923 * the IRQ allocated specifically for recovery mode
10924 * and restore the interrupt scheme
10925 */
10926 free_irq(pf->pdev->irq, pf);
10927 i40e_clear_interrupt_scheme(pf);
10928 if (i40e_restore_interrupt_scheme(pf))
10929 goto end_unlock;
10930 }
10931
10932 /* tell the firmware that we're starting */
10933 i40e_send_version(pf);
10934
10935 /* bail out in case recovery mode was detected, as there is
10936 * no need for further configuration.
10937 */
10938 goto end_unlock;
10939 }
10940
10941 i40e_clear_pxe_mode(hw);
10942 ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities);
10943 if (ret)
10944 goto end_core_reset;
10945
10946 ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
10947 hw->func_caps.num_rx_qp, 0, 0);
10948 if (ret) {
10949 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret);
10950 goto end_core_reset;
10951 }
10952 ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
10953 if (ret) {
10954 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret);
10955 goto end_core_reset;
10956 }
10957
10958 #ifdef CONFIG_I40E_DCB
10959 /* Enable FW to write a default DCB config on link-up
10960 * unless I40E_FLAG_TC_MQPRIO was enabled or DCB
10961 * is not supported with new link speed
10962 */
10963 if (i40e_is_tc_mqprio_enabled(pf)) {
10964 i40e_aq_set_dcb_parameters(hw, false, NULL);
10965 } else {
10966 if (I40E_IS_X710TL_DEVICE(hw->device_id) &&
10967 (hw->phy.link_info.link_speed &
10968 (I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB))) {
10969 i40e_aq_set_dcb_parameters(hw, false, NULL);
10970 dev_warn(&pf->pdev->dev,
10971 "DCB is not supported for X710-T*L 2.5/5G speeds\n");
10972 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
10973 } else {
10974 i40e_aq_set_dcb_parameters(hw, true, NULL);
10975 ret = i40e_init_pf_dcb(pf);
10976 if (ret) {
10977 dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n",
10978 ret);
10979 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
10980 /* Continue without DCB enabled */
10981 }
10982 }
10983 }
10984
10985 #endif /* CONFIG_I40E_DCB */
10986 if (!lock_acquired)
10987 rtnl_lock();
10988 ret = i40e_setup_pf_switch(pf, reinit, true);
10989 if (ret)
10990 goto end_unlock;
10991
10992 /* The driver only wants link up/down and module qualification
10993 * reports from firmware. Note the negative logic.
10994 */
10995 ret = i40e_aq_set_phy_int_mask(&pf->hw,
10996 ~(I40E_AQ_EVENT_LINK_UPDOWN |
10997 I40E_AQ_EVENT_MEDIA_NA |
10998 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
10999 if (ret)
11000 dev_info(&pf->pdev->dev, "set phy mask fail, err %pe aq_err %s\n",
11001 ERR_PTR(ret),
11002 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
11003
11004 /* Rebuild the VSIs and VEBs that existed before reset.
11005 * They are still in our local switch element arrays, so only
11006 * need to rebuild the switch model in the HW.
11007 *
11008 * If there were VEBs but the reconstitution failed, we'll try
11009 * to recover minimal use by getting the basic PF VSI working.
11010 */
11011 if (vsi->uplink_seid != pf->mac_seid) {
11012 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n");
11013
11014 /* Rebuild VEBs */
11015 i40e_pf_for_each_veb(pf, v, veb) {
11016 ret = i40e_reconstitute_veb(veb);
11017 if (!ret)
11018 continue;
11019
11020 /* If Main VEB failed, we're in deep doodoo,
11021 * so give up rebuilding the switch and set up
11022 * for minimal rebuild of PF VSI.
11023 * If orphan failed, we'll report the error
11024 * but try to keep going.
11025 */
11026 if (veb->uplink_seid == pf->mac_seid) {
11027 dev_info(&pf->pdev->dev,
11028 "rebuild of switch failed: %d, will try to set up simple PF connection\n",
11029 ret);
11030 vsi->uplink_seid = pf->mac_seid;
11031 break;
11032 } else if (veb->uplink_seid == 0) {
11033 dev_info(&pf->pdev->dev,
11034 "rebuild of orphan VEB failed: %d\n",
11035 ret);
11036 }
11037 }
11038 }
11039
11040 if (vsi->uplink_seid == pf->mac_seid) {
11041 dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n");
11042 /* no VEB, so rebuild only the Main VSI */
11043 ret = i40e_add_vsi(vsi);
11044 if (ret) {
11045 dev_info(&pf->pdev->dev,
11046 "rebuild of Main VSI failed: %d\n", ret);
11047 goto end_unlock;
11048 }
11049 }
11050
11051 if (vsi->mqprio_qopt.max_rate[0]) {
11052 u64 max_tx_rate = i40e_bw_bytes_to_mbits(vsi,
11053 vsi->mqprio_qopt.max_rate[0]);
11054 u64 credits = 0;
11055
11056 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate);
11057 if (ret)
11058 goto end_unlock;
11059
11060 credits = max_tx_rate;
11061 do_div(credits, I40E_BW_CREDIT_DIVISOR);
11062 dev_dbg(&vsi->back->pdev->dev,
11063 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
11064 max_tx_rate,
11065 credits,
11066 vsi->seid);
11067 }
11068
11069 ret = i40e_rebuild_cloud_filters(vsi, vsi->seid);
11070 if (ret)
11071 goto end_unlock;
11072
11073 /* PF Main VSI is rebuild by now, go ahead and rebuild channel VSIs
11074 * for this main VSI if they exist
11075 */
11076 ret = i40e_rebuild_channels(vsi);
11077 if (ret)
11078 goto end_unlock;
11079
11080 /* Reconfigure hardware for allowing smaller MSS in the case
11081 * of TSO, so that we avoid the MDD being fired and causing
11082 * a reset in the case of small MSS+TSO.
11083 */
11084 #define I40E_REG_MSS 0x000E64DC
11085 #define I40E_REG_MSS_MIN_MASK 0x3FF0000
11086 #define I40E_64BYTE_MSS 0x400000
11087 val = rd32(hw, I40E_REG_MSS);
11088 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
11089 val &= ~I40E_REG_MSS_MIN_MASK;
11090 val |= I40E_64BYTE_MSS;
11091 wr32(hw, I40E_REG_MSS, val);
11092 }
11093
11094 if (test_bit(I40E_HW_CAP_RESTART_AUTONEG, pf->hw.caps)) {
11095 msleep(75);
11096 ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
11097 if (ret)
11098 dev_info(&pf->pdev->dev, "link restart failed, err %pe aq_err %s\n",
11099 ERR_PTR(ret),
11100 i40e_aq_str(&pf->hw,
11101 pf->hw.aq.asq_last_status));
11102 }
11103 /* reinit the misc interrupt */
11104 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
11105 ret = i40e_setup_misc_vector(pf);
11106 if (ret)
11107 goto end_unlock;
11108 }
11109
11110 /* Add a filter to drop all Flow control frames from any VSI from being
11111 * transmitted. By doing so we stop a malicious VF from sending out
11112 * PAUSE or PFC frames and potentially controlling traffic for other
11113 * PF/VF VSIs.
11114 * The FW can still send Flow control frames if enabled.
11115 */
11116 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
11117 pf->main_vsi_seid);
11118
11119 /* restart the VSIs that were rebuilt and running before the reset */
11120 i40e_pf_unquiesce_all_vsi(pf);
11121
11122 /* Release the RTNL lock before we start resetting VFs */
11123 if (!lock_acquired)
11124 rtnl_unlock();
11125
11126 /* Restore promiscuous settings */
11127 ret = i40e_set_promiscuous(pf, pf->cur_promisc);
11128 if (ret)
11129 dev_warn(&pf->pdev->dev,
11130 "Failed to restore promiscuous setting: %s, err %pe aq_err %s\n",
11131 pf->cur_promisc ? "on" : "off",
11132 ERR_PTR(ret),
11133 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
11134
11135 i40e_reset_all_vfs(pf, true);
11136
11137 /* tell the firmware that we're starting */
11138 i40e_send_version(pf);
11139
11140 /* We've already released the lock, so don't do it again */
11141 goto end_core_reset;
11142
11143 end_unlock:
11144 if (!lock_acquired)
11145 rtnl_unlock();
11146 end_core_reset:
11147 clear_bit(__I40E_RESET_FAILED, pf->state);
11148 clear_recovery:
11149 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);
11150 clear_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state);
11151 }
11152
11153 /**
11154 * i40e_reset_and_rebuild - reset and rebuild using a saved config
11155 * @pf: board private structure
11156 * @reinit: if the Main VSI needs to re-initialized.
11157 * @lock_acquired: indicates whether or not the lock has been acquired
11158 * before this function was called.
11159 **/
i40e_reset_and_rebuild(struct i40e_pf * pf,bool reinit,bool lock_acquired)11160 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit,
11161 bool lock_acquired)
11162 {
11163 int ret;
11164
11165 if (test_bit(__I40E_IN_REMOVE, pf->state))
11166 return;
11167 /* Now we wait for GRST to settle out.
11168 * We don't have to delete the VEBs or VSIs from the hw switch
11169 * because the reset will make them disappear.
11170 */
11171 ret = i40e_reset(pf);
11172 if (!ret)
11173 i40e_rebuild(pf, reinit, lock_acquired);
11174 else
11175 dev_err(&pf->pdev->dev, "%s: i40e_reset() FAILED", __func__);
11176 }
11177
11178 /**
11179 * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild
11180 * @pf: board private structure
11181 *
11182 * Close up the VFs and other things in prep for a Core Reset,
11183 * then get ready to rebuild the world.
11184 * @lock_acquired: indicates whether or not the lock has been acquired
11185 * before this function was called.
11186 **/
i40e_handle_reset_warning(struct i40e_pf * pf,bool lock_acquired)11187 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired)
11188 {
11189 i40e_prep_for_reset(pf);
11190 i40e_reset_and_rebuild(pf, false, lock_acquired);
11191 }
11192
11193 /**
11194 * i40e_handle_mdd_event
11195 * @pf: pointer to the PF structure
11196 *
11197 * Called from the MDD irq handler to identify possibly malicious vfs
11198 **/
i40e_handle_mdd_event(struct i40e_pf * pf)11199 static void i40e_handle_mdd_event(struct i40e_pf *pf)
11200 {
11201 struct i40e_hw *hw = &pf->hw;
11202 bool mdd_detected = false;
11203 struct i40e_vf *vf;
11204 u32 reg;
11205 int i;
11206
11207 if (!test_bit(__I40E_MDD_EVENT_PENDING, pf->state))
11208 return;
11209
11210 /* find what triggered the MDD event */
11211 reg = rd32(hw, I40E_GL_MDET_TX);
11212 if (reg & I40E_GL_MDET_TX_VALID_MASK) {
11213 u8 pf_num = FIELD_GET(I40E_GL_MDET_TX_PF_NUM_MASK, reg);
11214 u16 vf_num = FIELD_GET(I40E_GL_MDET_TX_VF_NUM_MASK, reg);
11215 u8 event = FIELD_GET(I40E_GL_MDET_TX_EVENT_MASK, reg);
11216 u16 queue = FIELD_GET(I40E_GL_MDET_TX_QUEUE_MASK, reg) -
11217 pf->hw.func_caps.base_queue;
11218 if (netif_msg_tx_err(pf))
11219 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n",
11220 event, queue, pf_num, vf_num);
11221 wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
11222 mdd_detected = true;
11223 }
11224 reg = rd32(hw, I40E_GL_MDET_RX);
11225 if (reg & I40E_GL_MDET_RX_VALID_MASK) {
11226 u8 func = FIELD_GET(I40E_GL_MDET_RX_FUNCTION_MASK, reg);
11227 u8 event = FIELD_GET(I40E_GL_MDET_RX_EVENT_MASK, reg);
11228 u16 queue = FIELD_GET(I40E_GL_MDET_RX_QUEUE_MASK, reg) -
11229 pf->hw.func_caps.base_queue;
11230 if (netif_msg_rx_err(pf))
11231 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n",
11232 event, queue, func);
11233 wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
11234 mdd_detected = true;
11235 }
11236
11237 if (mdd_detected) {
11238 reg = rd32(hw, I40E_PF_MDET_TX);
11239 if (reg & I40E_PF_MDET_TX_VALID_MASK) {
11240 wr32(hw, I40E_PF_MDET_TX, 0xFFFF);
11241 dev_dbg(&pf->pdev->dev, "TX driver issue detected on PF\n");
11242 }
11243 reg = rd32(hw, I40E_PF_MDET_RX);
11244 if (reg & I40E_PF_MDET_RX_VALID_MASK) {
11245 wr32(hw, I40E_PF_MDET_RX, 0xFFFF);
11246 dev_dbg(&pf->pdev->dev, "RX driver issue detected on PF\n");
11247 }
11248 }
11249
11250 /* see if one of the VFs needs its hand slapped */
11251 for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) {
11252 vf = &(pf->vf[i]);
11253 reg = rd32(hw, I40E_VP_MDET_TX(i));
11254 if (reg & I40E_VP_MDET_TX_VALID_MASK) {
11255 wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF);
11256 vf->num_mdd_events++;
11257 dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n",
11258 i);
11259 dev_info(&pf->pdev->dev,
11260 "Use PF Control I/F to re-enable the VF\n");
11261 set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states);
11262 }
11263
11264 reg = rd32(hw, I40E_VP_MDET_RX(i));
11265 if (reg & I40E_VP_MDET_RX_VALID_MASK) {
11266 wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF);
11267 vf->num_mdd_events++;
11268 dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n",
11269 i);
11270 dev_info(&pf->pdev->dev,
11271 "Use PF Control I/F to re-enable the VF\n");
11272 set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states);
11273 }
11274 }
11275
11276 /* re-enable mdd interrupt cause */
11277 clear_bit(__I40E_MDD_EVENT_PENDING, pf->state);
11278 reg = rd32(hw, I40E_PFINT_ICR0_ENA);
11279 reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
11280 wr32(hw, I40E_PFINT_ICR0_ENA, reg);
11281 i40e_flush(hw);
11282 }
11283
11284 /**
11285 * i40e_service_task - Run the driver's async subtasks
11286 * @work: pointer to work_struct containing our data
11287 **/
i40e_service_task(struct work_struct * work)11288 static void i40e_service_task(struct work_struct *work)
11289 {
11290 struct i40e_pf *pf = container_of(work,
11291 struct i40e_pf,
11292 service_task);
11293 unsigned long start_time = jiffies;
11294
11295 /* don't bother with service tasks if a reset is in progress */
11296 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) ||
11297 test_bit(__I40E_SUSPENDED, pf->state))
11298 return;
11299
11300 if (test_and_set_bit(__I40E_SERVICE_SCHED, pf->state))
11301 return;
11302
11303 if (!test_bit(__I40E_RECOVERY_MODE, pf->state)) {
11304 i40e_detect_recover_hung(pf);
11305 i40e_sync_filters_subtask(pf);
11306 i40e_reset_subtask(pf);
11307 i40e_handle_mdd_event(pf);
11308 i40e_vc_process_vflr_event(pf);
11309 i40e_watchdog_subtask(pf);
11310 i40e_fdir_reinit_subtask(pf);
11311 if (test_and_clear_bit(__I40E_CLIENT_RESET, pf->state)) {
11312 /* Client subtask will reopen next time through. */
11313 i40e_notify_client_of_netdev_close(pf, true);
11314 } else {
11315 i40e_client_subtask(pf);
11316 if (test_and_clear_bit(__I40E_CLIENT_L2_CHANGE,
11317 pf->state))
11318 i40e_notify_client_of_l2_param_changes(pf);
11319 }
11320 i40e_sync_filters_subtask(pf);
11321 } else {
11322 i40e_reset_subtask(pf);
11323 }
11324
11325 i40e_clean_adminq_subtask(pf);
11326
11327 /* flush memory to make sure state is correct before next watchdog */
11328 smp_mb__before_atomic();
11329 clear_bit(__I40E_SERVICE_SCHED, pf->state);
11330
11331 /* If the tasks have taken longer than one timer cycle or there
11332 * is more work to be done, reschedule the service task now
11333 * rather than wait for the timer to tick again.
11334 */
11335 if (time_after(jiffies, (start_time + pf->service_timer_period)) ||
11336 test_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state) ||
11337 test_bit(__I40E_MDD_EVENT_PENDING, pf->state) ||
11338 test_bit(__I40E_VFLR_EVENT_PENDING, pf->state))
11339 i40e_service_event_schedule(pf);
11340 }
11341
11342 /**
11343 * i40e_service_timer - timer callback
11344 * @t: timer list pointer
11345 **/
i40e_service_timer(struct timer_list * t)11346 static void i40e_service_timer(struct timer_list *t)
11347 {
11348 struct i40e_pf *pf = from_timer(pf, t, service_timer);
11349
11350 mod_timer(&pf->service_timer,
11351 round_jiffies(jiffies + pf->service_timer_period));
11352 i40e_service_event_schedule(pf);
11353 }
11354
11355 /**
11356 * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI
11357 * @vsi: the VSI being configured
11358 **/
i40e_set_num_rings_in_vsi(struct i40e_vsi * vsi)11359 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi)
11360 {
11361 struct i40e_pf *pf = vsi->back;
11362
11363 switch (vsi->type) {
11364 case I40E_VSI_MAIN:
11365 vsi->alloc_queue_pairs = pf->num_lan_qps;
11366 if (!vsi->num_tx_desc)
11367 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11368 I40E_REQ_DESCRIPTOR_MULTIPLE);
11369 if (!vsi->num_rx_desc)
11370 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11371 I40E_REQ_DESCRIPTOR_MULTIPLE);
11372 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
11373 vsi->num_q_vectors = pf->num_lan_msix;
11374 else
11375 vsi->num_q_vectors = 1;
11376
11377 break;
11378
11379 case I40E_VSI_FDIR:
11380 vsi->alloc_queue_pairs = 1;
11381 vsi->num_tx_desc = ALIGN(I40E_FDIR_RING_COUNT,
11382 I40E_REQ_DESCRIPTOR_MULTIPLE);
11383 vsi->num_rx_desc = ALIGN(I40E_FDIR_RING_COUNT,
11384 I40E_REQ_DESCRIPTOR_MULTIPLE);
11385 vsi->num_q_vectors = pf->num_fdsb_msix;
11386 break;
11387
11388 case I40E_VSI_VMDQ2:
11389 vsi->alloc_queue_pairs = pf->num_vmdq_qps;
11390 if (!vsi->num_tx_desc)
11391 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11392 I40E_REQ_DESCRIPTOR_MULTIPLE);
11393 if (!vsi->num_rx_desc)
11394 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11395 I40E_REQ_DESCRIPTOR_MULTIPLE);
11396 vsi->num_q_vectors = pf->num_vmdq_msix;
11397 break;
11398
11399 case I40E_VSI_SRIOV:
11400 vsi->alloc_queue_pairs = pf->num_vf_qps;
11401 if (!vsi->num_tx_desc)
11402 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11403 I40E_REQ_DESCRIPTOR_MULTIPLE);
11404 if (!vsi->num_rx_desc)
11405 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11406 I40E_REQ_DESCRIPTOR_MULTIPLE);
11407 break;
11408
11409 default:
11410 WARN_ON(1);
11411 return -ENODATA;
11412 }
11413
11414 if (is_kdump_kernel()) {
11415 vsi->num_tx_desc = I40E_MIN_NUM_DESCRIPTORS;
11416 vsi->num_rx_desc = I40E_MIN_NUM_DESCRIPTORS;
11417 }
11418
11419 return 0;
11420 }
11421
11422 /**
11423 * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi
11424 * @vsi: VSI pointer
11425 * @alloc_qvectors: a bool to specify if q_vectors need to be allocated.
11426 *
11427 * On error: returns error code (negative)
11428 * On success: returns 0
11429 **/
i40e_vsi_alloc_arrays(struct i40e_vsi * vsi,bool alloc_qvectors)11430 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors)
11431 {
11432 struct i40e_ring **next_rings;
11433 int size;
11434 int ret = 0;
11435
11436 /* allocate memory for both Tx, XDP Tx and Rx ring pointers */
11437 size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs *
11438 (i40e_enabled_xdp_vsi(vsi) ? 3 : 2);
11439 vsi->tx_rings = kzalloc(size, GFP_KERNEL);
11440 if (!vsi->tx_rings)
11441 return -ENOMEM;
11442 next_rings = vsi->tx_rings + vsi->alloc_queue_pairs;
11443 if (i40e_enabled_xdp_vsi(vsi)) {
11444 vsi->xdp_rings = next_rings;
11445 next_rings += vsi->alloc_queue_pairs;
11446 }
11447 vsi->rx_rings = next_rings;
11448
11449 if (alloc_qvectors) {
11450 /* allocate memory for q_vector pointers */
11451 size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors;
11452 vsi->q_vectors = kzalloc(size, GFP_KERNEL);
11453 if (!vsi->q_vectors) {
11454 ret = -ENOMEM;
11455 goto err_vectors;
11456 }
11457 }
11458 return ret;
11459
11460 err_vectors:
11461 kfree(vsi->tx_rings);
11462 return ret;
11463 }
11464
11465 /**
11466 * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF
11467 * @pf: board private structure
11468 * @type: type of VSI
11469 *
11470 * On error: returns error code (negative)
11471 * On success: returns vsi index in PF (positive)
11472 **/
i40e_vsi_mem_alloc(struct i40e_pf * pf,enum i40e_vsi_type type)11473 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type)
11474 {
11475 int ret = -ENODEV;
11476 struct i40e_vsi *vsi;
11477 int vsi_idx;
11478 int i;
11479
11480 /* Need to protect the allocation of the VSIs at the PF level */
11481 mutex_lock(&pf->switch_mutex);
11482
11483 /* VSI list may be fragmented if VSI creation/destruction has
11484 * been happening. We can afford to do a quick scan to look
11485 * for any free VSIs in the list.
11486 *
11487 * find next empty vsi slot, looping back around if necessary
11488 */
11489 i = pf->next_vsi;
11490 while (i < pf->num_alloc_vsi && pf->vsi[i])
11491 i++;
11492 if (i >= pf->num_alloc_vsi) {
11493 i = 0;
11494 while (i < pf->next_vsi && pf->vsi[i])
11495 i++;
11496 }
11497
11498 if (i < pf->num_alloc_vsi && !pf->vsi[i]) {
11499 vsi_idx = i; /* Found one! */
11500 } else {
11501 ret = -ENODEV;
11502 goto unlock_pf; /* out of VSI slots! */
11503 }
11504 pf->next_vsi = ++i;
11505
11506 vsi = kzalloc(sizeof(*vsi), GFP_KERNEL);
11507 if (!vsi) {
11508 ret = -ENOMEM;
11509 goto unlock_pf;
11510 }
11511 vsi->type = type;
11512 vsi->back = pf;
11513 set_bit(__I40E_VSI_DOWN, vsi->state);
11514 vsi->flags = 0;
11515 vsi->idx = vsi_idx;
11516 vsi->int_rate_limit = 0;
11517 vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ?
11518 pf->rss_table_size : 64;
11519 vsi->netdev_registered = false;
11520 vsi->work_limit = I40E_DEFAULT_IRQ_WORK;
11521 hash_init(vsi->mac_filter_hash);
11522 vsi->irqs_ready = false;
11523
11524 if (type == I40E_VSI_MAIN) {
11525 vsi->af_xdp_zc_qps = bitmap_zalloc(pf->num_lan_qps, GFP_KERNEL);
11526 if (!vsi->af_xdp_zc_qps)
11527 goto err_rings;
11528 }
11529
11530 ret = i40e_set_num_rings_in_vsi(vsi);
11531 if (ret)
11532 goto err_rings;
11533
11534 ret = i40e_vsi_alloc_arrays(vsi, true);
11535 if (ret)
11536 goto err_rings;
11537
11538 /* Setup default MSIX irq handler for VSI */
11539 i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings);
11540
11541 /* Initialize VSI lock */
11542 spin_lock_init(&vsi->mac_filter_hash_lock);
11543 pf->vsi[vsi_idx] = vsi;
11544 ret = vsi_idx;
11545 goto unlock_pf;
11546
11547 err_rings:
11548 bitmap_free(vsi->af_xdp_zc_qps);
11549 pf->next_vsi = i - 1;
11550 kfree(vsi);
11551 unlock_pf:
11552 mutex_unlock(&pf->switch_mutex);
11553 return ret;
11554 }
11555
11556 /**
11557 * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI
11558 * @vsi: VSI pointer
11559 * @free_qvectors: a bool to specify if q_vectors need to be freed.
11560 *
11561 * On error: returns error code (negative)
11562 * On success: returns 0
11563 **/
i40e_vsi_free_arrays(struct i40e_vsi * vsi,bool free_qvectors)11564 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors)
11565 {
11566 /* free the ring and vector containers */
11567 if (free_qvectors) {
11568 kfree(vsi->q_vectors);
11569 vsi->q_vectors = NULL;
11570 }
11571 kfree(vsi->tx_rings);
11572 vsi->tx_rings = NULL;
11573 vsi->rx_rings = NULL;
11574 vsi->xdp_rings = NULL;
11575 }
11576
11577 /**
11578 * i40e_clear_rss_config_user - clear the user configured RSS hash keys
11579 * and lookup table
11580 * @vsi: Pointer to VSI structure
11581 */
i40e_clear_rss_config_user(struct i40e_vsi * vsi)11582 static void i40e_clear_rss_config_user(struct i40e_vsi *vsi)
11583 {
11584 if (!vsi)
11585 return;
11586
11587 kfree(vsi->rss_hkey_user);
11588 vsi->rss_hkey_user = NULL;
11589
11590 kfree(vsi->rss_lut_user);
11591 vsi->rss_lut_user = NULL;
11592 }
11593
11594 /**
11595 * i40e_vsi_clear - Deallocate the VSI provided
11596 * @vsi: the VSI being un-configured
11597 **/
i40e_vsi_clear(struct i40e_vsi * vsi)11598 static int i40e_vsi_clear(struct i40e_vsi *vsi)
11599 {
11600 struct i40e_pf *pf;
11601
11602 if (!vsi)
11603 return 0;
11604
11605 if (!vsi->back)
11606 goto free_vsi;
11607 pf = vsi->back;
11608
11609 mutex_lock(&pf->switch_mutex);
11610 if (!pf->vsi[vsi->idx]) {
11611 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](type %d)\n",
11612 vsi->idx, vsi->idx, vsi->type);
11613 goto unlock_vsi;
11614 }
11615
11616 if (pf->vsi[vsi->idx] != vsi) {
11617 dev_err(&pf->pdev->dev,
11618 "pf->vsi[%d](type %d) != vsi[%d](type %d): no free!\n",
11619 pf->vsi[vsi->idx]->idx,
11620 pf->vsi[vsi->idx]->type,
11621 vsi->idx, vsi->type);
11622 goto unlock_vsi;
11623 }
11624
11625 /* updates the PF for this cleared vsi */
11626 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
11627 i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx);
11628
11629 bitmap_free(vsi->af_xdp_zc_qps);
11630 i40e_vsi_free_arrays(vsi, true);
11631 i40e_clear_rss_config_user(vsi);
11632
11633 pf->vsi[vsi->idx] = NULL;
11634 if (vsi->idx < pf->next_vsi)
11635 pf->next_vsi = vsi->idx;
11636
11637 unlock_vsi:
11638 mutex_unlock(&pf->switch_mutex);
11639 free_vsi:
11640 kfree(vsi);
11641
11642 return 0;
11643 }
11644
11645 /**
11646 * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI
11647 * @vsi: the VSI being cleaned
11648 **/
i40e_vsi_clear_rings(struct i40e_vsi * vsi)11649 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi)
11650 {
11651 int i;
11652
11653 if (vsi->tx_rings && vsi->tx_rings[0]) {
11654 for (i = 0; i < vsi->alloc_queue_pairs; i++) {
11655 kfree_rcu(vsi->tx_rings[i], rcu);
11656 WRITE_ONCE(vsi->tx_rings[i], NULL);
11657 WRITE_ONCE(vsi->rx_rings[i], NULL);
11658 if (vsi->xdp_rings)
11659 WRITE_ONCE(vsi->xdp_rings[i], NULL);
11660 }
11661 }
11662 }
11663
11664 /**
11665 * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI
11666 * @vsi: the VSI being configured
11667 **/
i40e_alloc_rings(struct i40e_vsi * vsi)11668 static int i40e_alloc_rings(struct i40e_vsi *vsi)
11669 {
11670 int i, qpv = i40e_enabled_xdp_vsi(vsi) ? 3 : 2;
11671 struct i40e_pf *pf = vsi->back;
11672 struct i40e_ring *ring;
11673
11674 /* Set basic values in the rings to be used later during open() */
11675 for (i = 0; i < vsi->alloc_queue_pairs; i++) {
11676 /* allocate space for both Tx and Rx in one shot */
11677 ring = kcalloc(qpv, sizeof(struct i40e_ring), GFP_KERNEL);
11678 if (!ring)
11679 goto err_out;
11680
11681 ring->queue_index = i;
11682 ring->reg_idx = vsi->base_queue + i;
11683 ring->ring_active = false;
11684 ring->vsi = vsi;
11685 ring->netdev = vsi->netdev;
11686 ring->dev = &pf->pdev->dev;
11687 ring->count = vsi->num_tx_desc;
11688 ring->size = 0;
11689 ring->dcb_tc = 0;
11690 if (test_bit(I40E_HW_CAP_WB_ON_ITR, vsi->back->hw.caps))
11691 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
11692 ring->itr_setting = pf->tx_itr_default;
11693 WRITE_ONCE(vsi->tx_rings[i], ring++);
11694
11695 if (!i40e_enabled_xdp_vsi(vsi))
11696 goto setup_rx;
11697
11698 ring->queue_index = vsi->alloc_queue_pairs + i;
11699 ring->reg_idx = vsi->base_queue + ring->queue_index;
11700 ring->ring_active = false;
11701 ring->vsi = vsi;
11702 ring->netdev = NULL;
11703 ring->dev = &pf->pdev->dev;
11704 ring->count = vsi->num_tx_desc;
11705 ring->size = 0;
11706 ring->dcb_tc = 0;
11707 if (test_bit(I40E_HW_CAP_WB_ON_ITR, vsi->back->hw.caps))
11708 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
11709 set_ring_xdp(ring);
11710 ring->itr_setting = pf->tx_itr_default;
11711 WRITE_ONCE(vsi->xdp_rings[i], ring++);
11712
11713 setup_rx:
11714 ring->queue_index = i;
11715 ring->reg_idx = vsi->base_queue + i;
11716 ring->ring_active = false;
11717 ring->vsi = vsi;
11718 ring->netdev = vsi->netdev;
11719 ring->dev = &pf->pdev->dev;
11720 ring->count = vsi->num_rx_desc;
11721 ring->size = 0;
11722 ring->dcb_tc = 0;
11723 ring->itr_setting = pf->rx_itr_default;
11724 WRITE_ONCE(vsi->rx_rings[i], ring);
11725 }
11726
11727 return 0;
11728
11729 err_out:
11730 i40e_vsi_clear_rings(vsi);
11731 return -ENOMEM;
11732 }
11733
11734 /**
11735 * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel
11736 * @pf: board private structure
11737 * @vectors: the number of MSI-X vectors to request
11738 *
11739 * Returns the number of vectors reserved, or error
11740 **/
i40e_reserve_msix_vectors(struct i40e_pf * pf,int vectors)11741 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors)
11742 {
11743 vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries,
11744 I40E_MIN_MSIX, vectors);
11745 if (vectors < 0) {
11746 dev_info(&pf->pdev->dev,
11747 "MSI-X vector reservation failed: %d\n", vectors);
11748 vectors = 0;
11749 }
11750
11751 return vectors;
11752 }
11753
11754 /**
11755 * i40e_init_msix - Setup the MSIX capability
11756 * @pf: board private structure
11757 *
11758 * Work with the OS to set up the MSIX vectors needed.
11759 *
11760 * Returns the number of vectors reserved or negative on failure
11761 **/
i40e_init_msix(struct i40e_pf * pf)11762 static int i40e_init_msix(struct i40e_pf *pf)
11763 {
11764 struct i40e_hw *hw = &pf->hw;
11765 int cpus, extra_vectors;
11766 int vectors_left;
11767 int v_budget, i;
11768 int v_actual;
11769 int iwarp_requested = 0;
11770
11771 if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
11772 return -ENODEV;
11773
11774 /* The number of vectors we'll request will be comprised of:
11775 * - Add 1 for "other" cause for Admin Queue events, etc.
11776 * - The number of LAN queue pairs
11777 * - Queues being used for RSS.
11778 * We don't need as many as max_rss_size vectors.
11779 * use rss_size instead in the calculation since that
11780 * is governed by number of cpus in the system.
11781 * - assumes symmetric Tx/Rx pairing
11782 * - The number of VMDq pairs
11783 * - The CPU count within the NUMA node if iWARP is enabled
11784 * Once we count this up, try the request.
11785 *
11786 * If we can't get what we want, we'll simplify to nearly nothing
11787 * and try again. If that still fails, we punt.
11788 */
11789 vectors_left = hw->func_caps.num_msix_vectors;
11790 v_budget = 0;
11791
11792 /* reserve one vector for miscellaneous handler */
11793 if (vectors_left) {
11794 v_budget++;
11795 vectors_left--;
11796 }
11797
11798 /* reserve some vectors for the main PF traffic queues. Initially we
11799 * only reserve at most 50% of the available vectors, in the case that
11800 * the number of online CPUs is large. This ensures that we can enable
11801 * extra features as well. Once we've enabled the other features, we
11802 * will use any remaining vectors to reach as close as we can to the
11803 * number of online CPUs.
11804 */
11805 cpus = num_online_cpus();
11806 pf->num_lan_msix = min_t(int, cpus, vectors_left / 2);
11807 vectors_left -= pf->num_lan_msix;
11808
11809 /* reserve one vector for sideband flow director */
11810 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {
11811 if (vectors_left) {
11812 pf->num_fdsb_msix = 1;
11813 v_budget++;
11814 vectors_left--;
11815 } else {
11816 pf->num_fdsb_msix = 0;
11817 }
11818 }
11819
11820 /* can we reserve enough for iWARP? */
11821 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {
11822 iwarp_requested = pf->num_iwarp_msix;
11823
11824 if (!vectors_left)
11825 pf->num_iwarp_msix = 0;
11826 else if (vectors_left < pf->num_iwarp_msix)
11827 pf->num_iwarp_msix = 1;
11828 v_budget += pf->num_iwarp_msix;
11829 vectors_left -= pf->num_iwarp_msix;
11830 }
11831
11832 /* any vectors left over go for VMDq support */
11833 if (test_bit(I40E_FLAG_VMDQ_ENA, pf->flags)) {
11834 if (!vectors_left) {
11835 pf->num_vmdq_msix = 0;
11836 pf->num_vmdq_qps = 0;
11837 } else {
11838 int vmdq_vecs_wanted =
11839 pf->num_vmdq_vsis * pf->num_vmdq_qps;
11840 int vmdq_vecs =
11841 min_t(int, vectors_left, vmdq_vecs_wanted);
11842
11843 /* if we're short on vectors for what's desired, we limit
11844 * the queues per vmdq. If this is still more than are
11845 * available, the user will need to change the number of
11846 * queues/vectors used by the PF later with the ethtool
11847 * channels command
11848 */
11849 if (vectors_left < vmdq_vecs_wanted) {
11850 pf->num_vmdq_qps = 1;
11851 vmdq_vecs_wanted = pf->num_vmdq_vsis;
11852 vmdq_vecs = min_t(int,
11853 vectors_left,
11854 vmdq_vecs_wanted);
11855 }
11856 pf->num_vmdq_msix = pf->num_vmdq_qps;
11857
11858 v_budget += vmdq_vecs;
11859 vectors_left -= vmdq_vecs;
11860 }
11861 }
11862
11863 /* On systems with a large number of SMP cores, we previously limited
11864 * the number of vectors for num_lan_msix to be at most 50% of the
11865 * available vectors, to allow for other features. Now, we add back
11866 * the remaining vectors. However, we ensure that the total
11867 * num_lan_msix will not exceed num_online_cpus(). To do this, we
11868 * calculate the number of vectors we can add without going over the
11869 * cap of CPUs. For systems with a small number of CPUs this will be
11870 * zero.
11871 */
11872 extra_vectors = min_t(int, cpus - pf->num_lan_msix, vectors_left);
11873 pf->num_lan_msix += extra_vectors;
11874 vectors_left -= extra_vectors;
11875
11876 WARN(vectors_left < 0,
11877 "Calculation of remaining vectors underflowed. This is an accounting bug when determining total MSI-X vectors.\n");
11878
11879 v_budget += pf->num_lan_msix;
11880 pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry),
11881 GFP_KERNEL);
11882 if (!pf->msix_entries)
11883 return -ENOMEM;
11884
11885 for (i = 0; i < v_budget; i++)
11886 pf->msix_entries[i].entry = i;
11887 v_actual = i40e_reserve_msix_vectors(pf, v_budget);
11888
11889 if (v_actual < I40E_MIN_MSIX) {
11890 clear_bit(I40E_FLAG_MSIX_ENA, pf->flags);
11891 kfree(pf->msix_entries);
11892 pf->msix_entries = NULL;
11893 pci_disable_msix(pf->pdev);
11894 return -ENODEV;
11895
11896 } else if (v_actual == I40E_MIN_MSIX) {
11897 /* Adjust for minimal MSIX use */
11898 pf->num_vmdq_vsis = 0;
11899 pf->num_vmdq_qps = 0;
11900 pf->num_lan_qps = 1;
11901 pf->num_lan_msix = 1;
11902
11903 } else if (v_actual != v_budget) {
11904 /* If we have limited resources, we will start with no vectors
11905 * for the special features and then allocate vectors to some
11906 * of these features based on the policy and at the end disable
11907 * the features that did not get any vectors.
11908 */
11909 int vec;
11910
11911 dev_info(&pf->pdev->dev,
11912 "MSI-X vector limit reached with %d, wanted %d, attempting to redistribute vectors\n",
11913 v_actual, v_budget);
11914 /* reserve the misc vector */
11915 vec = v_actual - 1;
11916
11917 /* Scale vector usage down */
11918 pf->num_vmdq_msix = 1; /* force VMDqs to only one vector */
11919 pf->num_vmdq_vsis = 1;
11920 pf->num_vmdq_qps = 1;
11921
11922 /* partition out the remaining vectors */
11923 switch (vec) {
11924 case 2:
11925 pf->num_lan_msix = 1;
11926 break;
11927 case 3:
11928 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {
11929 pf->num_lan_msix = 1;
11930 pf->num_iwarp_msix = 1;
11931 } else {
11932 pf->num_lan_msix = 2;
11933 }
11934 break;
11935 default:
11936 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {
11937 pf->num_iwarp_msix = min_t(int, (vec / 3),
11938 iwarp_requested);
11939 pf->num_vmdq_vsis = min_t(int, (vec / 3),
11940 I40E_DEFAULT_NUM_VMDQ_VSI);
11941 } else {
11942 pf->num_vmdq_vsis = min_t(int, (vec / 2),
11943 I40E_DEFAULT_NUM_VMDQ_VSI);
11944 }
11945 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {
11946 pf->num_fdsb_msix = 1;
11947 vec--;
11948 }
11949 pf->num_lan_msix = min_t(int,
11950 (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)),
11951 pf->num_lan_msix);
11952 pf->num_lan_qps = pf->num_lan_msix;
11953 break;
11954 }
11955 }
11956
11957 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) && pf->num_fdsb_msix == 0) {
11958 dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n");
11959 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
11960 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
11961 }
11962 if (test_bit(I40E_FLAG_VMDQ_ENA, pf->flags) && pf->num_vmdq_msix == 0) {
11963 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n");
11964 clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags);
11965 }
11966
11967 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags) &&
11968 pf->num_iwarp_msix == 0) {
11969 dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n");
11970 clear_bit(I40E_FLAG_IWARP_ENA, pf->flags);
11971 }
11972 i40e_debug(&pf->hw, I40E_DEBUG_INIT,
11973 "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n",
11974 pf->num_lan_msix,
11975 pf->num_vmdq_msix * pf->num_vmdq_vsis,
11976 pf->num_fdsb_msix,
11977 pf->num_iwarp_msix);
11978
11979 return v_actual;
11980 }
11981
11982 /**
11983 * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector
11984 * @vsi: the VSI being configured
11985 * @v_idx: index of the vector in the vsi struct
11986 *
11987 * We allocate one q_vector. If allocation fails we return -ENOMEM.
11988 **/
i40e_vsi_alloc_q_vector(struct i40e_vsi * vsi,int v_idx)11989 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx)
11990 {
11991 struct i40e_q_vector *q_vector;
11992
11993 /* allocate q_vector */
11994 q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL);
11995 if (!q_vector)
11996 return -ENOMEM;
11997
11998 q_vector->vsi = vsi;
11999 q_vector->v_idx = v_idx;
12000 cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask);
12001
12002 if (vsi->netdev)
12003 netif_napi_add(vsi->netdev, &q_vector->napi, i40e_napi_poll);
12004
12005 /* tie q_vector and vsi together */
12006 vsi->q_vectors[v_idx] = q_vector;
12007
12008 return 0;
12009 }
12010
12011 /**
12012 * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors
12013 * @vsi: the VSI being configured
12014 *
12015 * We allocate one q_vector per queue interrupt. If allocation fails we
12016 * return -ENOMEM.
12017 **/
i40e_vsi_alloc_q_vectors(struct i40e_vsi * vsi)12018 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi)
12019 {
12020 struct i40e_pf *pf = vsi->back;
12021 int err, v_idx, num_q_vectors;
12022
12023 /* if not MSIX, give the one vector only to the LAN VSI */
12024 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
12025 num_q_vectors = vsi->num_q_vectors;
12026 else if (vsi->type == I40E_VSI_MAIN)
12027 num_q_vectors = 1;
12028 else
12029 return -EINVAL;
12030
12031 for (v_idx = 0; v_idx < num_q_vectors; v_idx++) {
12032 err = i40e_vsi_alloc_q_vector(vsi, v_idx);
12033 if (err)
12034 goto err_out;
12035 }
12036
12037 return 0;
12038
12039 err_out:
12040 while (v_idx--)
12041 i40e_free_q_vector(vsi, v_idx);
12042
12043 return err;
12044 }
12045
12046 /**
12047 * i40e_init_interrupt_scheme - Determine proper interrupt scheme
12048 * @pf: board private structure to initialize
12049 **/
i40e_init_interrupt_scheme(struct i40e_pf * pf)12050 static int i40e_init_interrupt_scheme(struct i40e_pf *pf)
12051 {
12052 int vectors = 0;
12053 ssize_t size;
12054
12055 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
12056 vectors = i40e_init_msix(pf);
12057 if (vectors < 0) {
12058 clear_bit(I40E_FLAG_MSIX_ENA, pf->flags);
12059 clear_bit(I40E_FLAG_IWARP_ENA, pf->flags);
12060 clear_bit(I40E_FLAG_RSS_ENA, pf->flags);
12061 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
12062 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
12063 clear_bit(I40E_FLAG_SRIOV_ENA, pf->flags);
12064 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
12065 clear_bit(I40E_FLAG_FD_ATR_ENA, pf->flags);
12066 clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags);
12067 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
12068
12069 /* rework the queue expectations without MSIX */
12070 i40e_determine_queue_usage(pf);
12071 }
12072 }
12073
12074 if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags) &&
12075 test_bit(I40E_FLAG_MSI_ENA, pf->flags)) {
12076 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n");
12077 vectors = pci_enable_msi(pf->pdev);
12078 if (vectors < 0) {
12079 dev_info(&pf->pdev->dev, "MSI init failed - %d\n",
12080 vectors);
12081 clear_bit(I40E_FLAG_MSI_ENA, pf->flags);
12082 }
12083 vectors = 1; /* one MSI or Legacy vector */
12084 }
12085
12086 if (!test_bit(I40E_FLAG_MSI_ENA, pf->flags) &&
12087 !test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
12088 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n");
12089
12090 /* set up vector assignment tracking */
12091 size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors);
12092 pf->irq_pile = kzalloc(size, GFP_KERNEL);
12093 if (!pf->irq_pile)
12094 return -ENOMEM;
12095
12096 pf->irq_pile->num_entries = vectors;
12097
12098 /* track first vector for misc interrupts, ignore return */
12099 (void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1);
12100
12101 return 0;
12102 }
12103
12104 /**
12105 * i40e_restore_interrupt_scheme - Restore the interrupt scheme
12106 * @pf: private board data structure
12107 *
12108 * Restore the interrupt scheme that was cleared when we suspended the
12109 * device. This should be called during resume to re-allocate the q_vectors
12110 * and reacquire IRQs.
12111 */
i40e_restore_interrupt_scheme(struct i40e_pf * pf)12112 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf)
12113 {
12114 struct i40e_vsi *vsi;
12115 int err, i;
12116
12117 /* We cleared the MSI and MSI-X flags when disabling the old interrupt
12118 * scheme. We need to re-enabled them here in order to attempt to
12119 * re-acquire the MSI or MSI-X vectors
12120 */
12121 set_bit(I40E_FLAG_MSI_ENA, pf->flags);
12122 set_bit(I40E_FLAG_MSIX_ENA, pf->flags);
12123
12124 err = i40e_init_interrupt_scheme(pf);
12125 if (err)
12126 return err;
12127
12128 /* Now that we've re-acquired IRQs, we need to remap the vectors and
12129 * rings together again.
12130 */
12131 i40e_pf_for_each_vsi(pf, i, vsi) {
12132 err = i40e_vsi_alloc_q_vectors(vsi);
12133 if (err)
12134 goto err_unwind;
12135
12136 i40e_vsi_map_rings_to_vectors(vsi);
12137 }
12138
12139 err = i40e_setup_misc_vector(pf);
12140 if (err)
12141 goto err_unwind;
12142
12143 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags))
12144 i40e_client_update_msix_info(pf);
12145
12146 return 0;
12147
12148 err_unwind:
12149 while (i--) {
12150 if (pf->vsi[i])
12151 i40e_vsi_free_q_vectors(pf->vsi[i]);
12152 }
12153
12154 return err;
12155 }
12156
12157 /**
12158 * i40e_setup_misc_vector_for_recovery_mode - Setup the misc vector to handle
12159 * non queue events in recovery mode
12160 * @pf: board private structure
12161 *
12162 * This sets up the handler for MSIX 0 or MSI/legacy, which is used to manage
12163 * the non-queue interrupts, e.g. AdminQ and errors in recovery mode.
12164 * This is handled differently than in recovery mode since no Tx/Rx resources
12165 * are being allocated.
12166 **/
i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf * pf)12167 static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf)
12168 {
12169 int err;
12170
12171 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
12172 err = i40e_setup_misc_vector(pf);
12173
12174 if (err) {
12175 dev_info(&pf->pdev->dev,
12176 "MSI-X misc vector request failed, error %d\n",
12177 err);
12178 return err;
12179 }
12180 } else {
12181 u32 flags = test_bit(I40E_FLAG_MSI_ENA, pf->flags) ? 0 : IRQF_SHARED;
12182
12183 err = request_irq(pf->pdev->irq, i40e_intr, flags,
12184 pf->int_name, pf);
12185
12186 if (err) {
12187 dev_info(&pf->pdev->dev,
12188 "MSI/legacy misc vector request failed, error %d\n",
12189 err);
12190 return err;
12191 }
12192 i40e_enable_misc_int_causes(pf);
12193 i40e_irq_dynamic_enable_icr0(pf);
12194 }
12195
12196 return 0;
12197 }
12198
12199 /**
12200 * i40e_setup_misc_vector - Setup the misc vector to handle non queue events
12201 * @pf: board private structure
12202 *
12203 * This sets up the handler for MSIX 0, which is used to manage the
12204 * non-queue interrupts, e.g. AdminQ and errors. This is not used
12205 * when in MSI or Legacy interrupt mode.
12206 **/
i40e_setup_misc_vector(struct i40e_pf * pf)12207 static int i40e_setup_misc_vector(struct i40e_pf *pf)
12208 {
12209 struct i40e_hw *hw = &pf->hw;
12210 int err = 0;
12211
12212 /* Only request the IRQ once, the first time through. */
12213 if (!test_and_set_bit(__I40E_MISC_IRQ_REQUESTED, pf->state)) {
12214 err = request_irq(pf->msix_entries[0].vector,
12215 i40e_intr, 0, pf->int_name, pf);
12216 if (err) {
12217 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);
12218 dev_info(&pf->pdev->dev,
12219 "request_irq for %s failed: %d\n",
12220 pf->int_name, err);
12221 return -EFAULT;
12222 }
12223 }
12224
12225 i40e_enable_misc_int_causes(pf);
12226
12227 /* associate no queues to the misc vector */
12228 wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST);
12229 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K >> 1);
12230
12231 i40e_flush(hw);
12232
12233 i40e_irq_dynamic_enable_icr0(pf);
12234
12235 return err;
12236 }
12237
12238 /**
12239 * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands
12240 * @vsi: Pointer to vsi structure
12241 * @seed: Buffter to store the hash keys
12242 * @lut: Buffer to store the lookup table entries
12243 * @lut_size: Size of buffer to store the lookup table entries
12244 *
12245 * Return 0 on success, negative on failure
12246 */
i40e_get_rss_aq(struct i40e_vsi * vsi,const u8 * seed,u8 * lut,u16 lut_size)12247 static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
12248 u8 *lut, u16 lut_size)
12249 {
12250 struct i40e_pf *pf = vsi->back;
12251 struct i40e_hw *hw = &pf->hw;
12252 int ret = 0;
12253
12254 if (seed) {
12255 ret = i40e_aq_get_rss_key(hw, vsi->id,
12256 (struct i40e_aqc_get_set_rss_key_data *)seed);
12257 if (ret) {
12258 dev_info(&pf->pdev->dev,
12259 "Cannot get RSS key, err %pe aq_err %s\n",
12260 ERR_PTR(ret),
12261 i40e_aq_str(&pf->hw,
12262 pf->hw.aq.asq_last_status));
12263 return ret;
12264 }
12265 }
12266
12267 if (lut) {
12268 bool pf_lut = vsi->type == I40E_VSI_MAIN;
12269
12270 ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
12271 if (ret) {
12272 dev_info(&pf->pdev->dev,
12273 "Cannot get RSS lut, err %pe aq_err %s\n",
12274 ERR_PTR(ret),
12275 i40e_aq_str(&pf->hw,
12276 pf->hw.aq.asq_last_status));
12277 return ret;
12278 }
12279 }
12280
12281 return ret;
12282 }
12283
12284 /**
12285 * i40e_config_rss_reg - Configure RSS keys and lut by writing registers
12286 * @vsi: Pointer to vsi structure
12287 * @seed: RSS hash seed
12288 * @lut: Lookup table
12289 * @lut_size: Lookup table size
12290 *
12291 * Returns 0 on success, negative on failure
12292 **/
i40e_config_rss_reg(struct i40e_vsi * vsi,const u8 * seed,const u8 * lut,u16 lut_size)12293 static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed,
12294 const u8 *lut, u16 lut_size)
12295 {
12296 struct i40e_pf *pf = vsi->back;
12297 struct i40e_hw *hw = &pf->hw;
12298 u16 vf_id = vsi->vf_id;
12299 u8 i;
12300
12301 /* Fill out hash function seed */
12302 if (seed) {
12303 u32 *seed_dw = (u32 *)seed;
12304
12305 if (vsi->type == I40E_VSI_MAIN) {
12306 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
12307 wr32(hw, I40E_PFQF_HKEY(i), seed_dw[i]);
12308 } else if (vsi->type == I40E_VSI_SRIOV) {
12309 for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++)
12310 wr32(hw, I40E_VFQF_HKEY1(i, vf_id), seed_dw[i]);
12311 } else {
12312 dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n");
12313 }
12314 }
12315
12316 if (lut) {
12317 u32 *lut_dw = (u32 *)lut;
12318
12319 if (vsi->type == I40E_VSI_MAIN) {
12320 if (lut_size != I40E_HLUT_ARRAY_SIZE)
12321 return -EINVAL;
12322 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
12323 wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]);
12324 } else if (vsi->type == I40E_VSI_SRIOV) {
12325 if (lut_size != I40E_VF_HLUT_ARRAY_SIZE)
12326 return -EINVAL;
12327 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
12328 wr32(hw, I40E_VFQF_HLUT1(i, vf_id), lut_dw[i]);
12329 } else {
12330 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
12331 }
12332 }
12333 i40e_flush(hw);
12334
12335 return 0;
12336 }
12337
12338 /**
12339 * i40e_get_rss_reg - Get the RSS keys and lut by reading registers
12340 * @vsi: Pointer to VSI structure
12341 * @seed: Buffer to store the keys
12342 * @lut: Buffer to store the lookup table entries
12343 * @lut_size: Size of buffer to store the lookup table entries
12344 *
12345 * Returns 0 on success, negative on failure
12346 */
i40e_get_rss_reg(struct i40e_vsi * vsi,u8 * seed,u8 * lut,u16 lut_size)12347 static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed,
12348 u8 *lut, u16 lut_size)
12349 {
12350 struct i40e_pf *pf = vsi->back;
12351 struct i40e_hw *hw = &pf->hw;
12352 u16 i;
12353
12354 if (seed) {
12355 u32 *seed_dw = (u32 *)seed;
12356
12357 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
12358 seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i));
12359 }
12360 if (lut) {
12361 u32 *lut_dw = (u32 *)lut;
12362
12363 if (lut_size != I40E_HLUT_ARRAY_SIZE)
12364 return -EINVAL;
12365 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
12366 lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i));
12367 }
12368
12369 return 0;
12370 }
12371
12372 /**
12373 * i40e_config_rss - Configure RSS keys and lut
12374 * @vsi: Pointer to VSI structure
12375 * @seed: RSS hash seed
12376 * @lut: Lookup table
12377 * @lut_size: Lookup table size
12378 *
12379 * Returns 0 on success, negative on failure
12380 */
i40e_config_rss(struct i40e_vsi * vsi,u8 * seed,u8 * lut,u16 lut_size)12381 int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
12382 {
12383 struct i40e_pf *pf = vsi->back;
12384
12385 if (test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps))
12386 return i40e_config_rss_aq(vsi, seed, lut, lut_size);
12387 else
12388 return i40e_config_rss_reg(vsi, seed, lut, lut_size);
12389 }
12390
12391 /**
12392 * i40e_get_rss - Get RSS keys and lut
12393 * @vsi: Pointer to VSI structure
12394 * @seed: Buffer to store the keys
12395 * @lut: Buffer to store the lookup table entries
12396 * @lut_size: Size of buffer to store the lookup table entries
12397 *
12398 * Returns 0 on success, negative on failure
12399 */
i40e_get_rss(struct i40e_vsi * vsi,u8 * seed,u8 * lut,u16 lut_size)12400 int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
12401 {
12402 struct i40e_pf *pf = vsi->back;
12403
12404 if (test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps))
12405 return i40e_get_rss_aq(vsi, seed, lut, lut_size);
12406 else
12407 return i40e_get_rss_reg(vsi, seed, lut, lut_size);
12408 }
12409
12410 /**
12411 * i40e_fill_rss_lut - Fill the RSS lookup table with default values
12412 * @pf: Pointer to board private structure
12413 * @lut: Lookup table
12414 * @rss_table_size: Lookup table size
12415 * @rss_size: Range of queue number for hashing
12416 */
i40e_fill_rss_lut(struct i40e_pf * pf,u8 * lut,u16 rss_table_size,u16 rss_size)12417 void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut,
12418 u16 rss_table_size, u16 rss_size)
12419 {
12420 u16 i;
12421
12422 for (i = 0; i < rss_table_size; i++)
12423 lut[i] = i % rss_size;
12424 }
12425
12426 /**
12427 * i40e_pf_config_rss - Prepare for RSS if used
12428 * @pf: board private structure
12429 **/
i40e_pf_config_rss(struct i40e_pf * pf)12430 static int i40e_pf_config_rss(struct i40e_pf *pf)
12431 {
12432 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);
12433 u8 seed[I40E_HKEY_ARRAY_SIZE];
12434 u8 *lut;
12435 struct i40e_hw *hw = &pf->hw;
12436 u32 reg_val;
12437 u64 hena;
12438 int ret;
12439
12440 /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */
12441 hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) |
12442 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32);
12443 hena |= i40e_pf_get_default_rss_hena(pf);
12444
12445 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena);
12446 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
12447
12448 /* Determine the RSS table size based on the hardware capabilities */
12449 reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0);
12450 reg_val = (pf->rss_table_size == 512) ?
12451 (reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) :
12452 (reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512);
12453 i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val);
12454
12455 /* Determine the RSS size of the VSI */
12456 if (!vsi->rss_size) {
12457 u16 qcount;
12458 /* If the firmware does something weird during VSI init, we
12459 * could end up with zero TCs. Check for that to avoid
12460 * divide-by-zero. It probably won't pass traffic, but it also
12461 * won't panic.
12462 */
12463 qcount = vsi->num_queue_pairs /
12464 (vsi->tc_config.numtc ? vsi->tc_config.numtc : 1);
12465 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
12466 }
12467 if (!vsi->rss_size)
12468 return -EINVAL;
12469
12470 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
12471 if (!lut)
12472 return -ENOMEM;
12473
12474 /* Use user configured lut if there is one, otherwise use default */
12475 if (vsi->rss_lut_user)
12476 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
12477 else
12478 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
12479
12480 /* Use user configured hash key if there is one, otherwise
12481 * use default.
12482 */
12483 if (vsi->rss_hkey_user)
12484 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
12485 else
12486 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
12487 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);
12488 kfree(lut);
12489
12490 return ret;
12491 }
12492
12493 /**
12494 * i40e_reconfig_rss_queues - change number of queues for rss and rebuild
12495 * @pf: board private structure
12496 * @queue_count: the requested queue count for rss.
12497 *
12498 * returns 0 if rss is not enabled, if enabled returns the final rss queue
12499 * count which may be different from the requested queue count.
12500 * Note: expects to be called while under rtnl_lock()
12501 **/
i40e_reconfig_rss_queues(struct i40e_pf * pf,int queue_count)12502 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count)
12503 {
12504 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);
12505 int new_rss_size;
12506
12507 if (!test_bit(I40E_FLAG_RSS_ENA, pf->flags))
12508 return 0;
12509
12510 queue_count = min_t(int, queue_count, num_online_cpus());
12511 new_rss_size = min_t(int, queue_count, pf->rss_size_max);
12512
12513 if (queue_count != vsi->num_queue_pairs) {
12514 u16 qcount;
12515
12516 vsi->req_queue_pairs = queue_count;
12517 i40e_prep_for_reset(pf);
12518 if (test_bit(__I40E_IN_REMOVE, pf->state))
12519 return pf->alloc_rss_size;
12520
12521 pf->alloc_rss_size = new_rss_size;
12522
12523 i40e_reset_and_rebuild(pf, true, true);
12524
12525 /* Discard the user configured hash keys and lut, if less
12526 * queues are enabled.
12527 */
12528 if (queue_count < vsi->rss_size) {
12529 i40e_clear_rss_config_user(vsi);
12530 dev_dbg(&pf->pdev->dev,
12531 "discard user configured hash keys and lut\n");
12532 }
12533
12534 /* Reset vsi->rss_size, as number of enabled queues changed */
12535 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc;
12536 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
12537
12538 i40e_pf_config_rss(pf);
12539 }
12540 dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count: %d/%d\n",
12541 vsi->req_queue_pairs, pf->rss_size_max);
12542 return pf->alloc_rss_size;
12543 }
12544
12545 /**
12546 * i40e_get_partition_bw_setting - Retrieve BW settings for this PF partition
12547 * @pf: board private structure
12548 **/
i40e_get_partition_bw_setting(struct i40e_pf * pf)12549 int i40e_get_partition_bw_setting(struct i40e_pf *pf)
12550 {
12551 bool min_valid, max_valid;
12552 u32 max_bw, min_bw;
12553 int status;
12554
12555 status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw,
12556 &min_valid, &max_valid);
12557
12558 if (!status) {
12559 if (min_valid)
12560 pf->min_bw = min_bw;
12561 if (max_valid)
12562 pf->max_bw = max_bw;
12563 }
12564
12565 return status;
12566 }
12567
12568 /**
12569 * i40e_set_partition_bw_setting - Set BW settings for this PF partition
12570 * @pf: board private structure
12571 **/
i40e_set_partition_bw_setting(struct i40e_pf * pf)12572 int i40e_set_partition_bw_setting(struct i40e_pf *pf)
12573 {
12574 struct i40e_aqc_configure_partition_bw_data bw_data;
12575 int status;
12576
12577 memset(&bw_data, 0, sizeof(bw_data));
12578
12579 /* Set the valid bit for this PF */
12580 bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id));
12581 bw_data.max_bw[pf->hw.pf_id] = pf->max_bw & I40E_ALT_BW_VALUE_MASK;
12582 bw_data.min_bw[pf->hw.pf_id] = pf->min_bw & I40E_ALT_BW_VALUE_MASK;
12583
12584 /* Set the new bandwidths */
12585 status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL);
12586
12587 return status;
12588 }
12589
12590 /**
12591 * i40e_commit_partition_bw_setting - Commit BW settings for this PF partition
12592 * @pf: board private structure
12593 **/
i40e_commit_partition_bw_setting(struct i40e_pf * pf)12594 int i40e_commit_partition_bw_setting(struct i40e_pf *pf)
12595 {
12596 /* Commit temporary BW setting to permanent NVM image */
12597 enum i40e_admin_queue_err last_aq_status;
12598 u16 nvm_word;
12599 int ret;
12600
12601 if (pf->hw.partition_id != 1) {
12602 dev_info(&pf->pdev->dev,
12603 "Commit BW only works on partition 1! This is partition %d",
12604 pf->hw.partition_id);
12605 ret = -EOPNOTSUPP;
12606 goto bw_commit_out;
12607 }
12608
12609 /* Acquire NVM for read access */
12610 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_READ);
12611 last_aq_status = pf->hw.aq.asq_last_status;
12612 if (ret) {
12613 dev_info(&pf->pdev->dev,
12614 "Cannot acquire NVM for read access, err %pe aq_err %s\n",
12615 ERR_PTR(ret),
12616 i40e_aq_str(&pf->hw, last_aq_status));
12617 goto bw_commit_out;
12618 }
12619
12620 /* Read word 0x10 of NVM - SW compatibility word 1 */
12621 ret = i40e_aq_read_nvm(&pf->hw,
12622 I40E_SR_NVM_CONTROL_WORD,
12623 0x10, sizeof(nvm_word), &nvm_word,
12624 false, NULL);
12625 /* Save off last admin queue command status before releasing
12626 * the NVM
12627 */
12628 last_aq_status = pf->hw.aq.asq_last_status;
12629 i40e_release_nvm(&pf->hw);
12630 if (ret) {
12631 dev_info(&pf->pdev->dev, "NVM read error, err %pe aq_err %s\n",
12632 ERR_PTR(ret),
12633 i40e_aq_str(&pf->hw, last_aq_status));
12634 goto bw_commit_out;
12635 }
12636
12637 /* Wait a bit for NVM release to complete */
12638 msleep(50);
12639
12640 /* Acquire NVM for write access */
12641 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_WRITE);
12642 last_aq_status = pf->hw.aq.asq_last_status;
12643 if (ret) {
12644 dev_info(&pf->pdev->dev,
12645 "Cannot acquire NVM for write access, err %pe aq_err %s\n",
12646 ERR_PTR(ret),
12647 i40e_aq_str(&pf->hw, last_aq_status));
12648 goto bw_commit_out;
12649 }
12650 /* Write it back out unchanged to initiate update NVM,
12651 * which will force a write of the shadow (alt) RAM to
12652 * the NVM - thus storing the bandwidth values permanently.
12653 */
12654 ret = i40e_aq_update_nvm(&pf->hw,
12655 I40E_SR_NVM_CONTROL_WORD,
12656 0x10, sizeof(nvm_word),
12657 &nvm_word, true, 0, NULL);
12658 /* Save off last admin queue command status before releasing
12659 * the NVM
12660 */
12661 last_aq_status = pf->hw.aq.asq_last_status;
12662 i40e_release_nvm(&pf->hw);
12663 if (ret)
12664 dev_info(&pf->pdev->dev,
12665 "BW settings NOT SAVED, err %pe aq_err %s\n",
12666 ERR_PTR(ret),
12667 i40e_aq_str(&pf->hw, last_aq_status));
12668 bw_commit_out:
12669
12670 return ret;
12671 }
12672
12673 /**
12674 * i40e_is_total_port_shutdown_enabled - read NVM and return value
12675 * if total port shutdown feature is enabled for this PF
12676 * @pf: board private structure
12677 **/
i40e_is_total_port_shutdown_enabled(struct i40e_pf * pf)12678 static bool i40e_is_total_port_shutdown_enabled(struct i40e_pf *pf)
12679 {
12680 #define I40E_TOTAL_PORT_SHUTDOWN_ENABLED BIT(4)
12681 #define I40E_FEATURES_ENABLE_PTR 0x2A
12682 #define I40E_CURRENT_SETTING_PTR 0x2B
12683 #define I40E_LINK_BEHAVIOR_WORD_OFFSET 0x2D
12684 #define I40E_LINK_BEHAVIOR_WORD_LENGTH 0x1
12685 #define I40E_LINK_BEHAVIOR_OS_FORCED_ENABLED BIT(0)
12686 #define I40E_LINK_BEHAVIOR_PORT_BIT_LENGTH 4
12687 u16 sr_emp_sr_settings_ptr = 0;
12688 u16 features_enable = 0;
12689 u16 link_behavior = 0;
12690 int read_status = 0;
12691 bool ret = false;
12692
12693 read_status = i40e_read_nvm_word(&pf->hw,
12694 I40E_SR_EMP_SR_SETTINGS_PTR,
12695 &sr_emp_sr_settings_ptr);
12696 if (read_status)
12697 goto err_nvm;
12698 read_status = i40e_read_nvm_word(&pf->hw,
12699 sr_emp_sr_settings_ptr +
12700 I40E_FEATURES_ENABLE_PTR,
12701 &features_enable);
12702 if (read_status)
12703 goto err_nvm;
12704 if (I40E_TOTAL_PORT_SHUTDOWN_ENABLED & features_enable) {
12705 read_status = i40e_read_nvm_module_data(&pf->hw,
12706 I40E_SR_EMP_SR_SETTINGS_PTR,
12707 I40E_CURRENT_SETTING_PTR,
12708 I40E_LINK_BEHAVIOR_WORD_OFFSET,
12709 I40E_LINK_BEHAVIOR_WORD_LENGTH,
12710 &link_behavior);
12711 if (read_status)
12712 goto err_nvm;
12713 link_behavior >>= (pf->hw.port * I40E_LINK_BEHAVIOR_PORT_BIT_LENGTH);
12714 ret = I40E_LINK_BEHAVIOR_OS_FORCED_ENABLED & link_behavior;
12715 }
12716 return ret;
12717
12718 err_nvm:
12719 dev_warn(&pf->pdev->dev,
12720 "total-port-shutdown feature is off due to read nvm error: %pe\n",
12721 ERR_PTR(read_status));
12722 return ret;
12723 }
12724
12725 /**
12726 * i40e_sw_init - Initialize general software structures (struct i40e_pf)
12727 * @pf: board private structure to initialize
12728 *
12729 * i40e_sw_init initializes the Adapter private data structure.
12730 * Fields are initialized based on PCI device information and
12731 * OS network device settings (MTU size).
12732 **/
i40e_sw_init(struct i40e_pf * pf)12733 static int i40e_sw_init(struct i40e_pf *pf)
12734 {
12735 int err = 0;
12736 int size;
12737 u16 pow;
12738
12739 /* Set default capability flags */
12740 bitmap_zero(pf->flags, I40E_PF_FLAGS_NBITS);
12741 set_bit(I40E_FLAG_MSI_ENA, pf->flags);
12742 set_bit(I40E_FLAG_MSIX_ENA, pf->flags);
12743
12744 /* Set default ITR */
12745 pf->rx_itr_default = I40E_ITR_RX_DEF;
12746 pf->tx_itr_default = I40E_ITR_TX_DEF;
12747
12748 /* Depending on PF configurations, it is possible that the RSS
12749 * maximum might end up larger than the available queues
12750 */
12751 pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width);
12752 pf->alloc_rss_size = 1;
12753 pf->rss_table_size = pf->hw.func_caps.rss_table_size;
12754 pf->rss_size_max = min_t(int, pf->rss_size_max,
12755 pf->hw.func_caps.num_tx_qp);
12756
12757 /* find the next higher power-of-2 of num cpus */
12758 pow = roundup_pow_of_two(num_online_cpus());
12759 pf->rss_size_max = min_t(int, pf->rss_size_max, pow);
12760
12761 if (pf->hw.func_caps.rss) {
12762 set_bit(I40E_FLAG_RSS_ENA, pf->flags);
12763 pf->alloc_rss_size = min_t(int, pf->rss_size_max,
12764 num_online_cpus());
12765 }
12766
12767 /* MFP mode enabled */
12768 if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) {
12769 set_bit(I40E_FLAG_MFP_ENA, pf->flags);
12770 dev_info(&pf->pdev->dev, "MFP mode Enabled\n");
12771 if (i40e_get_partition_bw_setting(pf)) {
12772 dev_warn(&pf->pdev->dev,
12773 "Could not get partition bw settings\n");
12774 } else {
12775 dev_info(&pf->pdev->dev,
12776 "Partition BW Min = %8.8x, Max = %8.8x\n",
12777 pf->min_bw, pf->max_bw);
12778
12779 /* nudge the Tx scheduler */
12780 i40e_set_partition_bw_setting(pf);
12781 }
12782 }
12783
12784 if ((pf->hw.func_caps.fd_filters_guaranteed > 0) ||
12785 (pf->hw.func_caps.fd_filters_best_effort > 0)) {
12786 set_bit(I40E_FLAG_FD_ATR_ENA, pf->flags);
12787 if (test_bit(I40E_FLAG_MFP_ENA, pf->flags) &&
12788 pf->hw.num_partitions > 1)
12789 dev_info(&pf->pdev->dev,
12790 "Flow Director Sideband mode Disabled in MFP mode\n");
12791 else
12792 set_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
12793 pf->fdir_pf_filter_count =
12794 pf->hw.func_caps.fd_filters_guaranteed;
12795 pf->hw.fdir_shared_filter_count =
12796 pf->hw.func_caps.fd_filters_best_effort;
12797 }
12798
12799 /* Enable HW ATR eviction if possible */
12800 if (test_bit(I40E_HW_CAP_ATR_EVICT, pf->hw.caps))
12801 set_bit(I40E_FLAG_HW_ATR_EVICT_ENA, pf->flags);
12802
12803 if (pf->hw.func_caps.vmdq && num_online_cpus() != 1) {
12804 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI;
12805 set_bit(I40E_FLAG_VMDQ_ENA, pf->flags);
12806 pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf);
12807 }
12808
12809 if (pf->hw.func_caps.iwarp && num_online_cpus() != 1) {
12810 set_bit(I40E_FLAG_IWARP_ENA, pf->flags);
12811 /* IWARP needs one extra vector for CQP just like MISC.*/
12812 pf->num_iwarp_msix = (int)num_online_cpus() + 1;
12813 }
12814 /* Stopping FW LLDP engine is supported on XL710 and X722
12815 * starting from FW versions determined in i40e_init_adminq.
12816 * Stopping the FW LLDP engine is not supported on XL710
12817 * if NPAR is functioning so unset this hw flag in this case.
12818 */
12819 if (pf->hw.mac.type == I40E_MAC_XL710 &&
12820 pf->hw.func_caps.npar_enable)
12821 clear_bit(I40E_HW_CAP_FW_LLDP_STOPPABLE, pf->hw.caps);
12822
12823 #ifdef CONFIG_PCI_IOV
12824 if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) {
12825 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF;
12826 set_bit(I40E_FLAG_SRIOV_ENA, pf->flags);
12827 pf->num_req_vfs = min_t(int,
12828 pf->hw.func_caps.num_vfs,
12829 I40E_MAX_VF_COUNT);
12830 }
12831 #endif /* CONFIG_PCI_IOV */
12832 pf->lan_veb = I40E_NO_VEB;
12833 pf->lan_vsi = I40E_NO_VSI;
12834
12835 /* By default FW has this off for performance reasons */
12836 clear_bit(I40E_FLAG_VEB_STATS_ENA, pf->flags);
12837
12838 /* set up queue assignment tracking */
12839 size = sizeof(struct i40e_lump_tracking)
12840 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp);
12841 pf->qp_pile = kzalloc(size, GFP_KERNEL);
12842 if (!pf->qp_pile) {
12843 err = -ENOMEM;
12844 goto sw_init_done;
12845 }
12846 pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp;
12847
12848 pf->tx_timeout_recovery_level = 1;
12849
12850 if (pf->hw.mac.type != I40E_MAC_X722 &&
12851 i40e_is_total_port_shutdown_enabled(pf)) {
12852 /* Link down on close must be on when total port shutdown
12853 * is enabled for a given port
12854 */
12855 set_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags);
12856 set_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags);
12857 dev_info(&pf->pdev->dev,
12858 "total-port-shutdown was enabled, link-down-on-close is forced on\n");
12859 }
12860 mutex_init(&pf->switch_mutex);
12861
12862 sw_init_done:
12863 return err;
12864 }
12865
12866 /**
12867 * i40e_set_ntuple - set the ntuple feature flag and take action
12868 * @pf: board private structure to initialize
12869 * @features: the feature set that the stack is suggesting
12870 *
12871 * returns a bool to indicate if reset needs to happen
12872 **/
i40e_set_ntuple(struct i40e_pf * pf,netdev_features_t features)12873 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features)
12874 {
12875 bool need_reset = false;
12876
12877 /* Check if Flow Director n-tuple support was enabled or disabled. If
12878 * the state changed, we need to reset.
12879 */
12880 if (features & NETIF_F_NTUPLE) {
12881 /* Enable filters and mark for reset */
12882 if (!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags))
12883 need_reset = true;
12884 /* enable FD_SB only if there is MSI-X vector and no cloud
12885 * filters exist
12886 */
12887 if (pf->num_fdsb_msix > 0 && !pf->num_cloud_filters) {
12888 set_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
12889 clear_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
12890 }
12891 } else {
12892 /* turn off filters, mark for reset and clear SW filter list */
12893 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {
12894 need_reset = true;
12895 i40e_fdir_filter_exit(pf);
12896 }
12897 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
12898 clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state);
12899 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
12900
12901 /* reset fd counters */
12902 pf->fd_add_err = 0;
12903 pf->fd_atr_cnt = 0;
12904 /* if ATR was auto disabled it can be re-enabled. */
12905 if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))
12906 if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) &&
12907 (I40E_DEBUG_FD & pf->hw.debug_mask))
12908 dev_info(&pf->pdev->dev, "ATR re-enabled.\n");
12909 }
12910 return need_reset;
12911 }
12912
12913 /**
12914 * i40e_clear_rss_lut - clear the rx hash lookup table
12915 * @vsi: the VSI being configured
12916 **/
i40e_clear_rss_lut(struct i40e_vsi * vsi)12917 static void i40e_clear_rss_lut(struct i40e_vsi *vsi)
12918 {
12919 struct i40e_pf *pf = vsi->back;
12920 struct i40e_hw *hw = &pf->hw;
12921 u16 vf_id = vsi->vf_id;
12922 u8 i;
12923
12924 if (vsi->type == I40E_VSI_MAIN) {
12925 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
12926 wr32(hw, I40E_PFQF_HLUT(i), 0);
12927 } else if (vsi->type == I40E_VSI_SRIOV) {
12928 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
12929 i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0);
12930 } else {
12931 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
12932 }
12933 }
12934
12935 /**
12936 * i40e_set_loopback - turn on/off loopback mode on underlying PF
12937 * @vsi: ptr to VSI
12938 * @ena: flag to indicate the on/off setting
12939 */
i40e_set_loopback(struct i40e_vsi * vsi,bool ena)12940 static int i40e_set_loopback(struct i40e_vsi *vsi, bool ena)
12941 {
12942 bool if_running = netif_running(vsi->netdev) &&
12943 !test_and_set_bit(__I40E_VSI_DOWN, vsi->state);
12944 int ret;
12945
12946 if (if_running)
12947 i40e_down(vsi);
12948
12949 ret = i40e_aq_set_mac_loopback(&vsi->back->hw, ena, NULL);
12950 if (ret)
12951 netdev_err(vsi->netdev, "Failed to toggle loopback state\n");
12952 if (if_running)
12953 i40e_up(vsi);
12954
12955 return ret;
12956 }
12957
12958 /**
12959 * i40e_set_features - set the netdev feature flags
12960 * @netdev: ptr to the netdev being adjusted
12961 * @features: the feature set that the stack is suggesting
12962 * Note: expects to be called while under rtnl_lock()
12963 **/
i40e_set_features(struct net_device * netdev,netdev_features_t features)12964 static int i40e_set_features(struct net_device *netdev,
12965 netdev_features_t features)
12966 {
12967 struct i40e_netdev_priv *np = netdev_priv(netdev);
12968 struct i40e_vsi *vsi = np->vsi;
12969 struct i40e_pf *pf = vsi->back;
12970 bool need_reset;
12971
12972 if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH))
12973 i40e_pf_config_rss(pf);
12974 else if (!(features & NETIF_F_RXHASH) &&
12975 netdev->features & NETIF_F_RXHASH)
12976 i40e_clear_rss_lut(vsi);
12977
12978 if (features & NETIF_F_HW_VLAN_CTAG_RX)
12979 i40e_vlan_stripping_enable(vsi);
12980 else
12981 i40e_vlan_stripping_disable(vsi);
12982
12983 if (!(features & NETIF_F_HW_TC) &&
12984 (netdev->features & NETIF_F_HW_TC) && pf->num_cloud_filters) {
12985 dev_err(&pf->pdev->dev,
12986 "Offloaded tc filters active, can't turn hw_tc_offload off");
12987 return -EINVAL;
12988 }
12989
12990 if (!(features & NETIF_F_HW_L2FW_DOFFLOAD) && vsi->macvlan_cnt)
12991 i40e_del_all_macvlans(vsi);
12992
12993 need_reset = i40e_set_ntuple(pf, features);
12994
12995 if (need_reset)
12996 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
12997
12998 if ((features ^ netdev->features) & NETIF_F_LOOPBACK)
12999 return i40e_set_loopback(vsi, !!(features & NETIF_F_LOOPBACK));
13000
13001 return 0;
13002 }
13003
i40e_udp_tunnel_set_port(struct net_device * netdev,unsigned int table,unsigned int idx,struct udp_tunnel_info * ti)13004 static int i40e_udp_tunnel_set_port(struct net_device *netdev,
13005 unsigned int table, unsigned int idx,
13006 struct udp_tunnel_info *ti)
13007 {
13008 struct i40e_netdev_priv *np = netdev_priv(netdev);
13009 struct i40e_hw *hw = &np->vsi->back->hw;
13010 u8 type, filter_index;
13011 int ret;
13012
13013 type = ti->type == UDP_TUNNEL_TYPE_VXLAN ? I40E_AQC_TUNNEL_TYPE_VXLAN :
13014 I40E_AQC_TUNNEL_TYPE_NGE;
13015
13016 ret = i40e_aq_add_udp_tunnel(hw, ntohs(ti->port), type, &filter_index,
13017 NULL);
13018 if (ret) {
13019 netdev_info(netdev, "add UDP port failed, err %pe aq_err %s\n",
13020 ERR_PTR(ret),
13021 i40e_aq_str(hw, hw->aq.asq_last_status));
13022 return -EIO;
13023 }
13024
13025 udp_tunnel_nic_set_port_priv(netdev, table, idx, filter_index);
13026 return 0;
13027 }
13028
i40e_udp_tunnel_unset_port(struct net_device * netdev,unsigned int table,unsigned int idx,struct udp_tunnel_info * ti)13029 static int i40e_udp_tunnel_unset_port(struct net_device *netdev,
13030 unsigned int table, unsigned int idx,
13031 struct udp_tunnel_info *ti)
13032 {
13033 struct i40e_netdev_priv *np = netdev_priv(netdev);
13034 struct i40e_hw *hw = &np->vsi->back->hw;
13035 int ret;
13036
13037 ret = i40e_aq_del_udp_tunnel(hw, ti->hw_priv, NULL);
13038 if (ret) {
13039 netdev_info(netdev, "delete UDP port failed, err %pe aq_err %s\n",
13040 ERR_PTR(ret),
13041 i40e_aq_str(hw, hw->aq.asq_last_status));
13042 return -EIO;
13043 }
13044
13045 return 0;
13046 }
13047
i40e_get_phys_port_id(struct net_device * netdev,struct netdev_phys_item_id * ppid)13048 static int i40e_get_phys_port_id(struct net_device *netdev,
13049 struct netdev_phys_item_id *ppid)
13050 {
13051 struct i40e_netdev_priv *np = netdev_priv(netdev);
13052 struct i40e_pf *pf = np->vsi->back;
13053 struct i40e_hw *hw = &pf->hw;
13054
13055 if (!test_bit(I40E_HW_CAP_PORT_ID_VALID, pf->hw.caps))
13056 return -EOPNOTSUPP;
13057
13058 ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id));
13059 memcpy(ppid->id, hw->mac.port_addr, ppid->id_len);
13060
13061 return 0;
13062 }
13063
13064 /**
13065 * i40e_ndo_fdb_add - add an entry to the hardware database
13066 * @ndm: the input from the stack
13067 * @tb: pointer to array of nladdr (unused)
13068 * @dev: the net device pointer
13069 * @addr: the MAC address entry being added
13070 * @vid: VLAN ID
13071 * @flags: instructions from stack about fdb operation
13072 * @extack: netlink extended ack, unused currently
13073 */
i40e_ndo_fdb_add(struct ndmsg * ndm,struct nlattr * tb[],struct net_device * dev,const unsigned char * addr,u16 vid,u16 flags,struct netlink_ext_ack * extack)13074 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
13075 struct net_device *dev,
13076 const unsigned char *addr, u16 vid,
13077 u16 flags,
13078 struct netlink_ext_ack *extack)
13079 {
13080 struct i40e_netdev_priv *np = netdev_priv(dev);
13081 struct i40e_pf *pf = np->vsi->back;
13082 int err = 0;
13083
13084 if (!test_bit(I40E_FLAG_SRIOV_ENA, pf->flags))
13085 return -EOPNOTSUPP;
13086
13087 if (vid) {
13088 pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name);
13089 return -EINVAL;
13090 }
13091
13092 /* Hardware does not support aging addresses so if a
13093 * ndm_state is given only allow permanent addresses
13094 */
13095 if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) {
13096 netdev_info(dev, "FDB only supports static addresses\n");
13097 return -EINVAL;
13098 }
13099
13100 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr))
13101 err = dev_uc_add_excl(dev, addr);
13102 else if (is_multicast_ether_addr(addr))
13103 err = dev_mc_add_excl(dev, addr);
13104 else
13105 err = -EINVAL;
13106
13107 /* Only return duplicate errors if NLM_F_EXCL is set */
13108 if (err == -EEXIST && !(flags & NLM_F_EXCL))
13109 err = 0;
13110
13111 return err;
13112 }
13113
13114 /**
13115 * i40e_ndo_bridge_setlink - Set the hardware bridge mode
13116 * @dev: the netdev being configured
13117 * @nlh: RTNL message
13118 * @flags: bridge flags
13119 * @extack: netlink extended ack
13120 *
13121 * Inserts a new hardware bridge if not already created and
13122 * enables the bridging mode requested (VEB or VEPA). If the
13123 * hardware bridge has already been inserted and the request
13124 * is to change the mode then that requires a PF reset to
13125 * allow rebuild of the components with required hardware
13126 * bridge mode enabled.
13127 *
13128 * Note: expects to be called while under rtnl_lock()
13129 **/
i40e_ndo_bridge_setlink(struct net_device * dev,struct nlmsghdr * nlh,u16 flags,struct netlink_ext_ack * extack)13130 static int i40e_ndo_bridge_setlink(struct net_device *dev,
13131 struct nlmsghdr *nlh,
13132 u16 flags,
13133 struct netlink_ext_ack *extack)
13134 {
13135 struct i40e_netdev_priv *np = netdev_priv(dev);
13136 struct i40e_vsi *vsi = np->vsi;
13137 struct i40e_pf *pf = vsi->back;
13138 struct nlattr *attr, *br_spec;
13139 struct i40e_veb *veb;
13140 int rem;
13141
13142 /* Only for PF VSI for now */
13143 if (vsi->type != I40E_VSI_MAIN)
13144 return -EOPNOTSUPP;
13145
13146 /* Find the HW bridge for PF VSI */
13147 veb = i40e_pf_get_veb_by_seid(pf, vsi->uplink_seid);
13148
13149 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
13150 if (!br_spec)
13151 return -EINVAL;
13152
13153 nla_for_each_nested_type(attr, IFLA_BRIDGE_MODE, br_spec, rem) {
13154 __u16 mode = nla_get_u16(attr);
13155
13156 if ((mode != BRIDGE_MODE_VEPA) &&
13157 (mode != BRIDGE_MODE_VEB))
13158 return -EINVAL;
13159
13160 /* Insert a new HW bridge */
13161 if (!veb) {
13162 veb = i40e_veb_setup(pf, vsi->uplink_seid, vsi->seid,
13163 vsi->tc_config.enabled_tc);
13164 if (veb) {
13165 veb->bridge_mode = mode;
13166 i40e_config_bridge_mode(veb);
13167 } else {
13168 /* No Bridge HW offload available */
13169 return -ENOENT;
13170 }
13171 break;
13172 } else if (mode != veb->bridge_mode) {
13173 /* Existing HW bridge but different mode needs reset */
13174 veb->bridge_mode = mode;
13175 /* TODO: If no VFs or VMDq VSIs, disallow VEB mode */
13176 if (mode == BRIDGE_MODE_VEB)
13177 set_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);
13178 else
13179 clear_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);
13180 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
13181 break;
13182 }
13183 }
13184
13185 return 0;
13186 }
13187
13188 /**
13189 * i40e_ndo_bridge_getlink - Get the hardware bridge mode
13190 * @skb: skb buff
13191 * @pid: process id
13192 * @seq: RTNL message seq #
13193 * @dev: the netdev being configured
13194 * @filter_mask: unused
13195 * @nlflags: netlink flags passed in
13196 *
13197 * Return the mode in which the hardware bridge is operating in
13198 * i.e VEB or VEPA.
13199 **/
i40e_ndo_bridge_getlink(struct sk_buff * skb,u32 pid,u32 seq,struct net_device * dev,u32 __always_unused filter_mask,int nlflags)13200 static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
13201 struct net_device *dev,
13202 u32 __always_unused filter_mask,
13203 int nlflags)
13204 {
13205 struct i40e_netdev_priv *np = netdev_priv(dev);
13206 struct i40e_vsi *vsi = np->vsi;
13207 struct i40e_pf *pf = vsi->back;
13208 struct i40e_veb *veb;
13209
13210 /* Only for PF VSI for now */
13211 if (vsi->type != I40E_VSI_MAIN)
13212 return -EOPNOTSUPP;
13213
13214 /* Find the HW bridge for the PF VSI */
13215 veb = i40e_pf_get_veb_by_seid(pf, vsi->uplink_seid);
13216 if (!veb)
13217 return 0;
13218
13219 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode,
13220 0, 0, nlflags, filter_mask, NULL);
13221 }
13222
13223 /**
13224 * i40e_features_check - Validate encapsulated packet conforms to limits
13225 * @skb: skb buff
13226 * @dev: This physical port's netdev
13227 * @features: Offload features that the stack believes apply
13228 **/
i40e_features_check(struct sk_buff * skb,struct net_device * dev,netdev_features_t features)13229 static netdev_features_t i40e_features_check(struct sk_buff *skb,
13230 struct net_device *dev,
13231 netdev_features_t features)
13232 {
13233 size_t len;
13234
13235 /* No point in doing any of this if neither checksum nor GSO are
13236 * being requested for this frame. We can rule out both by just
13237 * checking for CHECKSUM_PARTIAL
13238 */
13239 if (skb->ip_summed != CHECKSUM_PARTIAL)
13240 return features;
13241
13242 /* We cannot support GSO if the MSS is going to be less than
13243 * 64 bytes. If it is then we need to drop support for GSO.
13244 */
13245 if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))
13246 features &= ~NETIF_F_GSO_MASK;
13247
13248 /* MACLEN can support at most 63 words */
13249 len = skb_network_offset(skb);
13250 if (len & ~(63 * 2))
13251 goto out_err;
13252
13253 /* IPLEN and EIPLEN can support at most 127 dwords */
13254 len = skb_network_header_len(skb);
13255 if (len & ~(127 * 4))
13256 goto out_err;
13257
13258 if (skb->encapsulation) {
13259 /* L4TUNLEN can support 127 words */
13260 len = skb_inner_network_header(skb) - skb_transport_header(skb);
13261 if (len & ~(127 * 2))
13262 goto out_err;
13263
13264 /* IPLEN can support at most 127 dwords */
13265 len = skb_inner_transport_header(skb) -
13266 skb_inner_network_header(skb);
13267 if (len & ~(127 * 4))
13268 goto out_err;
13269 }
13270
13271 /* No need to validate L4LEN as TCP is the only protocol with a
13272 * flexible value and we support all possible values supported
13273 * by TCP, which is at most 15 dwords
13274 */
13275
13276 return features;
13277 out_err:
13278 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
13279 }
13280
13281 /**
13282 * i40e_xdp_setup - add/remove an XDP program
13283 * @vsi: VSI to changed
13284 * @prog: XDP program
13285 * @extack: netlink extended ack
13286 **/
i40e_xdp_setup(struct i40e_vsi * vsi,struct bpf_prog * prog,struct netlink_ext_ack * extack)13287 static int i40e_xdp_setup(struct i40e_vsi *vsi, struct bpf_prog *prog,
13288 struct netlink_ext_ack *extack)
13289 {
13290 int frame_size = i40e_max_vsi_frame_size(vsi, prog);
13291 struct i40e_pf *pf = vsi->back;
13292 struct bpf_prog *old_prog;
13293 bool need_reset;
13294 int i;
13295
13296 /* Don't allow frames that span over multiple buffers */
13297 if (vsi->netdev->mtu > frame_size - I40E_PACKET_HDR_PAD) {
13298 NL_SET_ERR_MSG_MOD(extack, "MTU too large for linear frames and XDP prog does not support frags");
13299 return -EINVAL;
13300 }
13301
13302 /* When turning XDP on->off/off->on we reset and rebuild the rings. */
13303 need_reset = (i40e_enabled_xdp_vsi(vsi) != !!prog);
13304
13305 if (need_reset)
13306 i40e_prep_for_reset(pf);
13307
13308 /* VSI shall be deleted in a moment, just return EINVAL */
13309 if (test_bit(__I40E_IN_REMOVE, pf->state))
13310 return -EINVAL;
13311
13312 old_prog = xchg(&vsi->xdp_prog, prog);
13313
13314 if (need_reset) {
13315 if (!prog) {
13316 xdp_features_clear_redirect_target(vsi->netdev);
13317 /* Wait until ndo_xsk_wakeup completes. */
13318 synchronize_rcu();
13319 }
13320 i40e_reset_and_rebuild(pf, true, true);
13321 }
13322
13323 if (!i40e_enabled_xdp_vsi(vsi) && prog) {
13324 if (i40e_realloc_rx_bi_zc(vsi, true))
13325 return -ENOMEM;
13326 } else if (i40e_enabled_xdp_vsi(vsi) && !prog) {
13327 if (i40e_realloc_rx_bi_zc(vsi, false))
13328 return -ENOMEM;
13329 }
13330
13331 for (i = 0; i < vsi->num_queue_pairs; i++)
13332 WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog);
13333
13334 if (old_prog)
13335 bpf_prog_put(old_prog);
13336
13337 /* Kick start the NAPI context if there is an AF_XDP socket open
13338 * on that queue id. This so that receiving will start.
13339 */
13340 if (need_reset && prog) {
13341 for (i = 0; i < vsi->num_queue_pairs; i++)
13342 if (vsi->xdp_rings[i]->xsk_pool)
13343 (void)i40e_xsk_wakeup(vsi->netdev, i,
13344 XDP_WAKEUP_RX);
13345 xdp_features_set_redirect_target(vsi->netdev, true);
13346 }
13347
13348 return 0;
13349 }
13350
13351 /**
13352 * i40e_enter_busy_conf - Enters busy config state
13353 * @vsi: vsi
13354 *
13355 * Returns 0 on success, <0 for failure.
13356 **/
i40e_enter_busy_conf(struct i40e_vsi * vsi)13357 static int i40e_enter_busy_conf(struct i40e_vsi *vsi)
13358 {
13359 struct i40e_pf *pf = vsi->back;
13360 int timeout = 50;
13361
13362 while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) {
13363 timeout--;
13364 if (!timeout)
13365 return -EBUSY;
13366 usleep_range(1000, 2000);
13367 }
13368
13369 return 0;
13370 }
13371
13372 /**
13373 * i40e_exit_busy_conf - Exits busy config state
13374 * @vsi: vsi
13375 **/
i40e_exit_busy_conf(struct i40e_vsi * vsi)13376 static void i40e_exit_busy_conf(struct i40e_vsi *vsi)
13377 {
13378 struct i40e_pf *pf = vsi->back;
13379
13380 clear_bit(__I40E_CONFIG_BUSY, pf->state);
13381 }
13382
13383 /**
13384 * i40e_queue_pair_reset_stats - Resets all statistics for a queue pair
13385 * @vsi: vsi
13386 * @queue_pair: queue pair
13387 **/
i40e_queue_pair_reset_stats(struct i40e_vsi * vsi,int queue_pair)13388 static void i40e_queue_pair_reset_stats(struct i40e_vsi *vsi, int queue_pair)
13389 {
13390 memset(&vsi->rx_rings[queue_pair]->rx_stats, 0,
13391 sizeof(vsi->rx_rings[queue_pair]->rx_stats));
13392 memset(&vsi->tx_rings[queue_pair]->stats, 0,
13393 sizeof(vsi->tx_rings[queue_pair]->stats));
13394 if (i40e_enabled_xdp_vsi(vsi)) {
13395 memset(&vsi->xdp_rings[queue_pair]->stats, 0,
13396 sizeof(vsi->xdp_rings[queue_pair]->stats));
13397 }
13398 }
13399
13400 /**
13401 * i40e_queue_pair_clean_rings - Cleans all the rings of a queue pair
13402 * @vsi: vsi
13403 * @queue_pair: queue pair
13404 **/
i40e_queue_pair_clean_rings(struct i40e_vsi * vsi,int queue_pair)13405 static void i40e_queue_pair_clean_rings(struct i40e_vsi *vsi, int queue_pair)
13406 {
13407 i40e_clean_tx_ring(vsi->tx_rings[queue_pair]);
13408 if (i40e_enabled_xdp_vsi(vsi)) {
13409 /* Make sure that in-progress ndo_xdp_xmit calls are
13410 * completed.
13411 */
13412 synchronize_rcu();
13413 i40e_clean_tx_ring(vsi->xdp_rings[queue_pair]);
13414 }
13415 i40e_clean_rx_ring(vsi->rx_rings[queue_pair]);
13416 }
13417
13418 /**
13419 * i40e_queue_pair_toggle_napi - Enables/disables NAPI for a queue pair
13420 * @vsi: vsi
13421 * @queue_pair: queue pair
13422 * @enable: true for enable, false for disable
13423 **/
i40e_queue_pair_toggle_napi(struct i40e_vsi * vsi,int queue_pair,bool enable)13424 static void i40e_queue_pair_toggle_napi(struct i40e_vsi *vsi, int queue_pair,
13425 bool enable)
13426 {
13427 struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
13428 struct i40e_q_vector *q_vector = rxr->q_vector;
13429
13430 if (!vsi->netdev)
13431 return;
13432
13433 /* All rings in a qp belong to the same qvector. */
13434 if (q_vector->rx.ring || q_vector->tx.ring) {
13435 if (enable)
13436 napi_enable(&q_vector->napi);
13437 else
13438 napi_disable(&q_vector->napi);
13439 }
13440 }
13441
13442 /**
13443 * i40e_queue_pair_toggle_rings - Enables/disables all rings for a queue pair
13444 * @vsi: vsi
13445 * @queue_pair: queue pair
13446 * @enable: true for enable, false for disable
13447 *
13448 * Returns 0 on success, <0 on failure.
13449 **/
i40e_queue_pair_toggle_rings(struct i40e_vsi * vsi,int queue_pair,bool enable)13450 static int i40e_queue_pair_toggle_rings(struct i40e_vsi *vsi, int queue_pair,
13451 bool enable)
13452 {
13453 struct i40e_pf *pf = vsi->back;
13454 int pf_q, ret = 0;
13455
13456 pf_q = vsi->base_queue + queue_pair;
13457 ret = i40e_control_wait_tx_q(vsi->seid, pf, pf_q,
13458 false /*is xdp*/, enable);
13459 if (ret) {
13460 dev_info(&pf->pdev->dev,
13461 "VSI seid %d Tx ring %d %sable timeout\n",
13462 vsi->seid, pf_q, (enable ? "en" : "dis"));
13463 return ret;
13464 }
13465
13466 i40e_control_rx_q(pf, pf_q, enable);
13467 ret = i40e_pf_rxq_wait(pf, pf_q, enable);
13468 if (ret) {
13469 dev_info(&pf->pdev->dev,
13470 "VSI seid %d Rx ring %d %sable timeout\n",
13471 vsi->seid, pf_q, (enable ? "en" : "dis"));
13472 return ret;
13473 }
13474
13475 /* Due to HW errata, on Rx disable only, the register can
13476 * indicate done before it really is. Needs 50ms to be sure
13477 */
13478 if (!enable)
13479 mdelay(50);
13480
13481 if (!i40e_enabled_xdp_vsi(vsi))
13482 return ret;
13483
13484 ret = i40e_control_wait_tx_q(vsi->seid, pf,
13485 pf_q + vsi->alloc_queue_pairs,
13486 true /*is xdp*/, enable);
13487 if (ret) {
13488 dev_info(&pf->pdev->dev,
13489 "VSI seid %d XDP Tx ring %d %sable timeout\n",
13490 vsi->seid, pf_q, (enable ? "en" : "dis"));
13491 }
13492
13493 return ret;
13494 }
13495
13496 /**
13497 * i40e_queue_pair_enable_irq - Enables interrupts for a queue pair
13498 * @vsi: vsi
13499 * @queue_pair: queue_pair
13500 **/
i40e_queue_pair_enable_irq(struct i40e_vsi * vsi,int queue_pair)13501 static void i40e_queue_pair_enable_irq(struct i40e_vsi *vsi, int queue_pair)
13502 {
13503 struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
13504 struct i40e_pf *pf = vsi->back;
13505 struct i40e_hw *hw = &pf->hw;
13506
13507 /* All rings in a qp belong to the same qvector. */
13508 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
13509 i40e_irq_dynamic_enable(vsi, rxr->q_vector->v_idx);
13510 else
13511 i40e_irq_dynamic_enable_icr0(pf);
13512
13513 i40e_flush(hw);
13514 }
13515
13516 /**
13517 * i40e_queue_pair_disable_irq - Disables interrupts for a queue pair
13518 * @vsi: vsi
13519 * @queue_pair: queue_pair
13520 **/
i40e_queue_pair_disable_irq(struct i40e_vsi * vsi,int queue_pair)13521 static void i40e_queue_pair_disable_irq(struct i40e_vsi *vsi, int queue_pair)
13522 {
13523 struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
13524 struct i40e_pf *pf = vsi->back;
13525 struct i40e_hw *hw = &pf->hw;
13526
13527 /* For simplicity, instead of removing the qp interrupt causes
13528 * from the interrupt linked list, we simply disable the interrupt, and
13529 * leave the list intact.
13530 *
13531 * All rings in a qp belong to the same qvector.
13532 */
13533 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
13534 u32 intpf = vsi->base_vector + rxr->q_vector->v_idx;
13535
13536 wr32(hw, I40E_PFINT_DYN_CTLN(intpf - 1), 0);
13537 i40e_flush(hw);
13538 synchronize_irq(pf->msix_entries[intpf].vector);
13539 } else {
13540 /* Legacy and MSI mode - this stops all interrupt handling */
13541 wr32(hw, I40E_PFINT_ICR0_ENA, 0);
13542 wr32(hw, I40E_PFINT_DYN_CTL0, 0);
13543 i40e_flush(hw);
13544 synchronize_irq(pf->pdev->irq);
13545 }
13546 }
13547
13548 /**
13549 * i40e_queue_pair_disable - Disables a queue pair
13550 * @vsi: vsi
13551 * @queue_pair: queue pair
13552 *
13553 * Returns 0 on success, <0 on failure.
13554 **/
i40e_queue_pair_disable(struct i40e_vsi * vsi,int queue_pair)13555 int i40e_queue_pair_disable(struct i40e_vsi *vsi, int queue_pair)
13556 {
13557 int err;
13558
13559 err = i40e_enter_busy_conf(vsi);
13560 if (err)
13561 return err;
13562
13563 i40e_queue_pair_disable_irq(vsi, queue_pair);
13564 i40e_queue_pair_toggle_napi(vsi, queue_pair, false /* off */);
13565 err = i40e_queue_pair_toggle_rings(vsi, queue_pair, false /* off */);
13566 i40e_clean_rx_ring(vsi->rx_rings[queue_pair]);
13567 i40e_queue_pair_clean_rings(vsi, queue_pair);
13568 i40e_queue_pair_reset_stats(vsi, queue_pair);
13569
13570 return err;
13571 }
13572
13573 /**
13574 * i40e_queue_pair_enable - Enables a queue pair
13575 * @vsi: vsi
13576 * @queue_pair: queue pair
13577 *
13578 * Returns 0 on success, <0 on failure.
13579 **/
i40e_queue_pair_enable(struct i40e_vsi * vsi,int queue_pair)13580 int i40e_queue_pair_enable(struct i40e_vsi *vsi, int queue_pair)
13581 {
13582 int err;
13583
13584 err = i40e_configure_tx_ring(vsi->tx_rings[queue_pair]);
13585 if (err)
13586 return err;
13587
13588 if (i40e_enabled_xdp_vsi(vsi)) {
13589 err = i40e_configure_tx_ring(vsi->xdp_rings[queue_pair]);
13590 if (err)
13591 return err;
13592 }
13593
13594 err = i40e_configure_rx_ring(vsi->rx_rings[queue_pair]);
13595 if (err)
13596 return err;
13597
13598 err = i40e_queue_pair_toggle_rings(vsi, queue_pair, true /* on */);
13599 i40e_queue_pair_toggle_napi(vsi, queue_pair, true /* on */);
13600 i40e_queue_pair_enable_irq(vsi, queue_pair);
13601
13602 i40e_exit_busy_conf(vsi);
13603
13604 return err;
13605 }
13606
13607 /**
13608 * i40e_xdp - implements ndo_bpf for i40e
13609 * @dev: netdevice
13610 * @xdp: XDP command
13611 **/
i40e_xdp(struct net_device * dev,struct netdev_bpf * xdp)13612 static int i40e_xdp(struct net_device *dev,
13613 struct netdev_bpf *xdp)
13614 {
13615 struct i40e_netdev_priv *np = netdev_priv(dev);
13616 struct i40e_vsi *vsi = np->vsi;
13617
13618 if (vsi->type != I40E_VSI_MAIN)
13619 return -EINVAL;
13620
13621 switch (xdp->command) {
13622 case XDP_SETUP_PROG:
13623 return i40e_xdp_setup(vsi, xdp->prog, xdp->extack);
13624 case XDP_SETUP_XSK_POOL:
13625 return i40e_xsk_pool_setup(vsi, xdp->xsk.pool,
13626 xdp->xsk.queue_id);
13627 default:
13628 return -EINVAL;
13629 }
13630 }
13631
13632 static const struct net_device_ops i40e_netdev_ops = {
13633 .ndo_open = i40e_open,
13634 .ndo_stop = i40e_close,
13635 .ndo_start_xmit = i40e_lan_xmit_frame,
13636 .ndo_get_stats64 = i40e_get_netdev_stats_struct,
13637 .ndo_set_rx_mode = i40e_set_rx_mode,
13638 .ndo_validate_addr = eth_validate_addr,
13639 .ndo_set_mac_address = i40e_set_mac,
13640 .ndo_change_mtu = i40e_change_mtu,
13641 .ndo_eth_ioctl = i40e_ioctl,
13642 .ndo_tx_timeout = i40e_tx_timeout,
13643 .ndo_vlan_rx_add_vid = i40e_vlan_rx_add_vid,
13644 .ndo_vlan_rx_kill_vid = i40e_vlan_rx_kill_vid,
13645 #ifdef CONFIG_NET_POLL_CONTROLLER
13646 .ndo_poll_controller = i40e_netpoll,
13647 #endif
13648 .ndo_setup_tc = __i40e_setup_tc,
13649 .ndo_select_queue = i40e_lan_select_queue,
13650 .ndo_set_features = i40e_set_features,
13651 .ndo_set_vf_mac = i40e_ndo_set_vf_mac,
13652 .ndo_set_vf_vlan = i40e_ndo_set_vf_port_vlan,
13653 .ndo_get_vf_stats = i40e_get_vf_stats,
13654 .ndo_set_vf_rate = i40e_ndo_set_vf_bw,
13655 .ndo_get_vf_config = i40e_ndo_get_vf_config,
13656 .ndo_set_vf_link_state = i40e_ndo_set_vf_link_state,
13657 .ndo_set_vf_spoofchk = i40e_ndo_set_vf_spoofchk,
13658 .ndo_set_vf_trust = i40e_ndo_set_vf_trust,
13659 .ndo_get_phys_port_id = i40e_get_phys_port_id,
13660 .ndo_fdb_add = i40e_ndo_fdb_add,
13661 .ndo_features_check = i40e_features_check,
13662 .ndo_bridge_getlink = i40e_ndo_bridge_getlink,
13663 .ndo_bridge_setlink = i40e_ndo_bridge_setlink,
13664 .ndo_bpf = i40e_xdp,
13665 .ndo_xdp_xmit = i40e_xdp_xmit,
13666 .ndo_xsk_wakeup = i40e_xsk_wakeup,
13667 .ndo_dfwd_add_station = i40e_fwd_add,
13668 .ndo_dfwd_del_station = i40e_fwd_del,
13669 };
13670
13671 /**
13672 * i40e_config_netdev - Setup the netdev flags
13673 * @vsi: the VSI being configured
13674 *
13675 * Returns 0 on success, negative value on failure
13676 **/
i40e_config_netdev(struct i40e_vsi * vsi)13677 static int i40e_config_netdev(struct i40e_vsi *vsi)
13678 {
13679 struct i40e_pf *pf = vsi->back;
13680 struct i40e_hw *hw = &pf->hw;
13681 struct i40e_netdev_priv *np;
13682 struct net_device *netdev;
13683 u8 broadcast[ETH_ALEN];
13684 u8 mac_addr[ETH_ALEN];
13685 int etherdev_size;
13686 netdev_features_t hw_enc_features;
13687 netdev_features_t hw_features;
13688
13689 etherdev_size = sizeof(struct i40e_netdev_priv);
13690 netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs);
13691 if (!netdev)
13692 return -ENOMEM;
13693
13694 vsi->netdev = netdev;
13695 np = netdev_priv(netdev);
13696 np->vsi = vsi;
13697
13698 hw_enc_features = NETIF_F_SG |
13699 NETIF_F_HW_CSUM |
13700 NETIF_F_HIGHDMA |
13701 NETIF_F_SOFT_FEATURES |
13702 NETIF_F_TSO |
13703 NETIF_F_TSO_ECN |
13704 NETIF_F_TSO6 |
13705 NETIF_F_GSO_GRE |
13706 NETIF_F_GSO_GRE_CSUM |
13707 NETIF_F_GSO_PARTIAL |
13708 NETIF_F_GSO_IPXIP4 |
13709 NETIF_F_GSO_IPXIP6 |
13710 NETIF_F_GSO_UDP_TUNNEL |
13711 NETIF_F_GSO_UDP_TUNNEL_CSUM |
13712 NETIF_F_GSO_UDP_L4 |
13713 NETIF_F_SCTP_CRC |
13714 NETIF_F_RXHASH |
13715 NETIF_F_RXCSUM |
13716 0;
13717
13718 if (!test_bit(I40E_HW_CAP_OUTER_UDP_CSUM, pf->hw.caps))
13719 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
13720
13721 netdev->udp_tunnel_nic_info = &pf->udp_tunnel_nic;
13722
13723 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
13724
13725 netdev->hw_enc_features |= hw_enc_features;
13726
13727 /* record features VLANs can make use of */
13728 netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID;
13729
13730 #define I40E_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \
13731 NETIF_F_GSO_GRE_CSUM | \
13732 NETIF_F_GSO_IPXIP4 | \
13733 NETIF_F_GSO_IPXIP6 | \
13734 NETIF_F_GSO_UDP_TUNNEL | \
13735 NETIF_F_GSO_UDP_TUNNEL_CSUM)
13736
13737 netdev->gso_partial_features = I40E_GSO_PARTIAL_FEATURES;
13738 netdev->features |= NETIF_F_GSO_PARTIAL |
13739 I40E_GSO_PARTIAL_FEATURES;
13740
13741 netdev->mpls_features |= NETIF_F_SG;
13742 netdev->mpls_features |= NETIF_F_HW_CSUM;
13743 netdev->mpls_features |= NETIF_F_TSO;
13744 netdev->mpls_features |= NETIF_F_TSO6;
13745 netdev->mpls_features |= I40E_GSO_PARTIAL_FEATURES;
13746
13747 /* enable macvlan offloads */
13748 netdev->hw_features |= NETIF_F_HW_L2FW_DOFFLOAD;
13749
13750 hw_features = hw_enc_features |
13751 NETIF_F_HW_VLAN_CTAG_TX |
13752 NETIF_F_HW_VLAN_CTAG_RX;
13753
13754 if (!test_bit(I40E_FLAG_MFP_ENA, pf->flags))
13755 hw_features |= NETIF_F_NTUPLE | NETIF_F_HW_TC;
13756
13757 netdev->hw_features |= hw_features | NETIF_F_LOOPBACK;
13758
13759 netdev->features |= hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;
13760 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
13761
13762 netdev->features &= ~NETIF_F_HW_TC;
13763
13764 if (vsi->type == I40E_VSI_MAIN) {
13765 SET_NETDEV_DEV(netdev, &pf->pdev->dev);
13766 ether_addr_copy(mac_addr, hw->mac.perm_addr);
13767 /* The following steps are necessary for two reasons. First,
13768 * some older NVM configurations load a default MAC-VLAN
13769 * filter that will accept any tagged packet, and we want to
13770 * replace this with a normal filter. Additionally, it is
13771 * possible our MAC address was provided by the platform using
13772 * Open Firmware or similar.
13773 *
13774 * Thus, we need to remove the default filter and install one
13775 * specific to the MAC address.
13776 */
13777 i40e_rm_default_mac_filter(vsi, mac_addr);
13778 spin_lock_bh(&vsi->mac_filter_hash_lock);
13779 i40e_add_mac_filter(vsi, mac_addr);
13780 spin_unlock_bh(&vsi->mac_filter_hash_lock);
13781
13782 netdev->xdp_features = NETDEV_XDP_ACT_BASIC |
13783 NETDEV_XDP_ACT_REDIRECT |
13784 NETDEV_XDP_ACT_XSK_ZEROCOPY |
13785 NETDEV_XDP_ACT_RX_SG;
13786 netdev->xdp_zc_max_segs = I40E_MAX_BUFFER_TXD;
13787 } else {
13788 /* Relate the VSI_VMDQ name to the VSI_MAIN name. Note that we
13789 * are still limited by IFNAMSIZ, but we're adding 'v%d\0' to
13790 * the end, which is 4 bytes long, so force truncation of the
13791 * original name by IFNAMSIZ - 4
13792 */
13793 struct i40e_vsi *main_vsi = i40e_pf_get_main_vsi(pf);
13794
13795 snprintf(netdev->name, IFNAMSIZ, "%.*sv%%d", IFNAMSIZ - 4,
13796 main_vsi->netdev->name);
13797 eth_random_addr(mac_addr);
13798
13799 spin_lock_bh(&vsi->mac_filter_hash_lock);
13800 i40e_add_mac_filter(vsi, mac_addr);
13801 spin_unlock_bh(&vsi->mac_filter_hash_lock);
13802 }
13803
13804 /* Add the broadcast filter so that we initially will receive
13805 * broadcast packets. Note that when a new VLAN is first added the
13806 * driver will convert all filters marked I40E_VLAN_ANY into VLAN
13807 * specific filters as part of transitioning into "vlan" operation.
13808 * When more VLANs are added, the driver will copy each existing MAC
13809 * filter and add it for the new VLAN.
13810 *
13811 * Broadcast filters are handled specially by
13812 * i40e_sync_filters_subtask, as the driver must to set the broadcast
13813 * promiscuous bit instead of adding this directly as a MAC/VLAN
13814 * filter. The subtask will update the correct broadcast promiscuous
13815 * bits as VLANs become active or inactive.
13816 */
13817 eth_broadcast_addr(broadcast);
13818 spin_lock_bh(&vsi->mac_filter_hash_lock);
13819 i40e_add_mac_filter(vsi, broadcast);
13820 spin_unlock_bh(&vsi->mac_filter_hash_lock);
13821
13822 eth_hw_addr_set(netdev, mac_addr);
13823 ether_addr_copy(netdev->perm_addr, mac_addr);
13824
13825 /* i40iw_net_event() reads 16 bytes from neigh->primary_key */
13826 netdev->neigh_priv_len = sizeof(u32) * 4;
13827
13828 netdev->priv_flags |= IFF_UNICAST_FLT;
13829 netdev->priv_flags |= IFF_SUPP_NOFCS;
13830 /* Setup netdev TC information */
13831 i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc);
13832
13833 netdev->netdev_ops = &i40e_netdev_ops;
13834 netdev->watchdog_timeo = 5 * HZ;
13835 i40e_set_ethtool_ops(netdev);
13836
13837 /* MTU range: 68 - 9706 */
13838 netdev->min_mtu = ETH_MIN_MTU;
13839 netdev->max_mtu = I40E_MAX_RXBUFFER - I40E_PACKET_HDR_PAD;
13840
13841 return 0;
13842 }
13843
13844 /**
13845 * i40e_vsi_delete - Delete a VSI from the switch
13846 * @vsi: the VSI being removed
13847 *
13848 * Returns 0 on success, negative value on failure
13849 **/
i40e_vsi_delete(struct i40e_vsi * vsi)13850 static void i40e_vsi_delete(struct i40e_vsi *vsi)
13851 {
13852 /* remove default VSI is not allowed */
13853 if (vsi == vsi->back->vsi[vsi->back->lan_vsi])
13854 return;
13855
13856 i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL);
13857 }
13858
13859 /**
13860 * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB
13861 * @vsi: the VSI being queried
13862 *
13863 * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode
13864 **/
i40e_is_vsi_uplink_mode_veb(struct i40e_vsi * vsi)13865 int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi)
13866 {
13867 struct i40e_veb *veb;
13868 struct i40e_pf *pf = vsi->back;
13869
13870 /* Uplink is not a bridge so default to VEB */
13871 if (vsi->veb_idx >= I40E_MAX_VEB)
13872 return 1;
13873
13874 veb = pf->veb[vsi->veb_idx];
13875 if (!veb) {
13876 dev_info(&pf->pdev->dev,
13877 "There is no veb associated with the bridge\n");
13878 return -ENOENT;
13879 }
13880
13881 /* Uplink is a bridge in VEPA mode */
13882 if (veb->bridge_mode & BRIDGE_MODE_VEPA) {
13883 return 0;
13884 } else {
13885 /* Uplink is a bridge in VEB mode */
13886 return 1;
13887 }
13888
13889 /* VEPA is now default bridge, so return 0 */
13890 return 0;
13891 }
13892
13893 /**
13894 * i40e_add_vsi - Add a VSI to the switch
13895 * @vsi: the VSI being configured
13896 *
13897 * This initializes a VSI context depending on the VSI type to be added and
13898 * passes it down to the add_vsi aq command.
13899 **/
i40e_add_vsi(struct i40e_vsi * vsi)13900 static int i40e_add_vsi(struct i40e_vsi *vsi)
13901 {
13902 int ret = -ENODEV;
13903 struct i40e_pf *pf = vsi->back;
13904 struct i40e_hw *hw = &pf->hw;
13905 struct i40e_vsi_context ctxt;
13906 struct i40e_mac_filter *f;
13907 struct hlist_node *h;
13908 int bkt;
13909
13910 u8 enabled_tc = 0x1; /* TC0 enabled */
13911 int f_count = 0;
13912
13913 memset(&ctxt, 0, sizeof(ctxt));
13914 switch (vsi->type) {
13915 case I40E_VSI_MAIN:
13916 /* The PF's main VSI is already setup as part of the
13917 * device initialization, so we'll not bother with
13918 * the add_vsi call, but we will retrieve the current
13919 * VSI context.
13920 */
13921 ctxt.seid = pf->main_vsi_seid;
13922 ctxt.pf_num = pf->hw.pf_id;
13923 ctxt.vf_num = 0;
13924 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
13925 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
13926 if (ret) {
13927 dev_info(&pf->pdev->dev,
13928 "couldn't get PF vsi config, err %pe aq_err %s\n",
13929 ERR_PTR(ret),
13930 i40e_aq_str(&pf->hw,
13931 pf->hw.aq.asq_last_status));
13932 return -ENOENT;
13933 }
13934 vsi->info = ctxt.info;
13935 vsi->info.valid_sections = 0;
13936
13937 vsi->seid = ctxt.seid;
13938 vsi->id = ctxt.vsi_number;
13939
13940 enabled_tc = i40e_pf_get_tc_map(pf);
13941
13942 /* Source pruning is enabled by default, so the flag is
13943 * negative logic - if it's set, we need to fiddle with
13944 * the VSI to disable source pruning.
13945 */
13946 if (test_bit(I40E_FLAG_SOURCE_PRUNING_DIS, pf->flags)) {
13947 memset(&ctxt, 0, sizeof(ctxt));
13948 ctxt.seid = pf->main_vsi_seid;
13949 ctxt.pf_num = pf->hw.pf_id;
13950 ctxt.vf_num = 0;
13951 ctxt.info.valid_sections |=
13952 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
13953 ctxt.info.switch_id =
13954 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB);
13955 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
13956 if (ret) {
13957 dev_info(&pf->pdev->dev,
13958 "update vsi failed, err %d aq_err %s\n",
13959 ret,
13960 i40e_aq_str(&pf->hw,
13961 pf->hw.aq.asq_last_status));
13962 ret = -ENOENT;
13963 goto err;
13964 }
13965 }
13966
13967 /* MFP mode setup queue map and update VSI */
13968 if (test_bit(I40E_FLAG_MFP_ENA, pf->flags) &&
13969 !(pf->hw.func_caps.iscsi)) { /* NIC type PF */
13970 memset(&ctxt, 0, sizeof(ctxt));
13971 ctxt.seid = pf->main_vsi_seid;
13972 ctxt.pf_num = pf->hw.pf_id;
13973 ctxt.vf_num = 0;
13974 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
13975 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
13976 if (ret) {
13977 dev_info(&pf->pdev->dev,
13978 "update vsi failed, err %pe aq_err %s\n",
13979 ERR_PTR(ret),
13980 i40e_aq_str(&pf->hw,
13981 pf->hw.aq.asq_last_status));
13982 ret = -ENOENT;
13983 goto err;
13984 }
13985 /* update the local VSI info queue map */
13986 i40e_vsi_update_queue_map(vsi, &ctxt);
13987 vsi->info.valid_sections = 0;
13988 } else {
13989 /* Default/Main VSI is only enabled for TC0
13990 * reconfigure it to enable all TCs that are
13991 * available on the port in SFP mode.
13992 * For MFP case the iSCSI PF would use this
13993 * flow to enable LAN+iSCSI TC.
13994 */
13995 ret = i40e_vsi_config_tc(vsi, enabled_tc);
13996 if (ret) {
13997 /* Single TC condition is not fatal,
13998 * message and continue
13999 */
14000 dev_info(&pf->pdev->dev,
14001 "failed to configure TCs for main VSI tc_map 0x%08x, err %pe aq_err %s\n",
14002 enabled_tc,
14003 ERR_PTR(ret),
14004 i40e_aq_str(&pf->hw,
14005 pf->hw.aq.asq_last_status));
14006 }
14007 }
14008 break;
14009
14010 case I40E_VSI_FDIR:
14011 ctxt.pf_num = hw->pf_id;
14012 ctxt.vf_num = 0;
14013 ctxt.uplink_seid = vsi->uplink_seid;
14014 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
14015 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
14016 if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags) &&
14017 (i40e_is_vsi_uplink_mode_veb(vsi))) {
14018 ctxt.info.valid_sections |=
14019 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
14020 ctxt.info.switch_id =
14021 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
14022 }
14023 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
14024 break;
14025
14026 case I40E_VSI_VMDQ2:
14027 ctxt.pf_num = hw->pf_id;
14028 ctxt.vf_num = 0;
14029 ctxt.uplink_seid = vsi->uplink_seid;
14030 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
14031 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
14032
14033 /* This VSI is connected to VEB so the switch_id
14034 * should be set to zero by default.
14035 */
14036 if (i40e_is_vsi_uplink_mode_veb(vsi)) {
14037 ctxt.info.valid_sections |=
14038 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
14039 ctxt.info.switch_id =
14040 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
14041 }
14042
14043 /* Setup the VSI tx/rx queue map for TC0 only for now */
14044 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
14045 break;
14046
14047 case I40E_VSI_SRIOV:
14048 ctxt.pf_num = hw->pf_id;
14049 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id;
14050 ctxt.uplink_seid = vsi->uplink_seid;
14051 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
14052 ctxt.flags = I40E_AQ_VSI_TYPE_VF;
14053
14054 /* This VSI is connected to VEB so the switch_id
14055 * should be set to zero by default.
14056 */
14057 if (i40e_is_vsi_uplink_mode_veb(vsi)) {
14058 ctxt.info.valid_sections |=
14059 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
14060 ctxt.info.switch_id =
14061 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
14062 }
14063
14064 if (test_bit(I40E_FLAG_IWARP_ENA, vsi->back->flags)) {
14065 ctxt.info.valid_sections |=
14066 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
14067 ctxt.info.queueing_opt_flags |=
14068 (I40E_AQ_VSI_QUE_OPT_TCP_ENA |
14069 I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI);
14070 }
14071
14072 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
14073 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
14074 if (pf->vf[vsi->vf_id].spoofchk) {
14075 ctxt.info.valid_sections |=
14076 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID);
14077 ctxt.info.sec_flags |=
14078 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK |
14079 I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK);
14080 }
14081 /* Setup the VSI tx/rx queue map for TC0 only for now */
14082 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
14083 break;
14084
14085 case I40E_VSI_IWARP:
14086 /* send down message to iWARP */
14087 break;
14088
14089 default:
14090 return -ENODEV;
14091 }
14092
14093 if (vsi->type != I40E_VSI_MAIN) {
14094 ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
14095 if (ret) {
14096 dev_info(&vsi->back->pdev->dev,
14097 "add vsi failed, err %pe aq_err %s\n",
14098 ERR_PTR(ret),
14099 i40e_aq_str(&pf->hw,
14100 pf->hw.aq.asq_last_status));
14101 ret = -ENOENT;
14102 goto err;
14103 }
14104 vsi->info = ctxt.info;
14105 vsi->info.valid_sections = 0;
14106 vsi->seid = ctxt.seid;
14107 vsi->id = ctxt.vsi_number;
14108 }
14109
14110 spin_lock_bh(&vsi->mac_filter_hash_lock);
14111 vsi->active_filters = 0;
14112 /* If macvlan filters already exist, force them to get loaded */
14113 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
14114 f->state = I40E_FILTER_NEW;
14115 f_count++;
14116 }
14117 spin_unlock_bh(&vsi->mac_filter_hash_lock);
14118 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
14119
14120 if (f_count) {
14121 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
14122 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state);
14123 }
14124
14125 /* Update VSI BW information */
14126 ret = i40e_vsi_get_bw_info(vsi);
14127 if (ret) {
14128 dev_info(&pf->pdev->dev,
14129 "couldn't get vsi bw info, err %pe aq_err %s\n",
14130 ERR_PTR(ret),
14131 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14132 /* VSI is already added so not tearing that up */
14133 ret = 0;
14134 }
14135
14136 err:
14137 return ret;
14138 }
14139
14140 /**
14141 * i40e_vsi_release - Delete a VSI and free its resources
14142 * @vsi: the VSI being removed
14143 *
14144 * Returns 0 on success or < 0 on error
14145 **/
i40e_vsi_release(struct i40e_vsi * vsi)14146 int i40e_vsi_release(struct i40e_vsi *vsi)
14147 {
14148 struct i40e_mac_filter *f;
14149 struct hlist_node *h;
14150 struct i40e_veb *veb;
14151 struct i40e_pf *pf;
14152 u16 uplink_seid;
14153 int i, n, bkt;
14154
14155 pf = vsi->back;
14156
14157 /* release of a VEB-owner or last VSI is not allowed */
14158 if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) {
14159 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n",
14160 vsi->seid, vsi->uplink_seid);
14161 return -ENODEV;
14162 }
14163 if (vsi->type == I40E_VSI_MAIN && !test_bit(__I40E_DOWN, pf->state)) {
14164 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n");
14165 return -ENODEV;
14166 }
14167 set_bit(__I40E_VSI_RELEASING, vsi->state);
14168 uplink_seid = vsi->uplink_seid;
14169
14170 if (vsi->type != I40E_VSI_SRIOV) {
14171 if (vsi->netdev_registered) {
14172 vsi->netdev_registered = false;
14173 if (vsi->netdev) {
14174 /* results in a call to i40e_close() */
14175 unregister_netdev(vsi->netdev);
14176 }
14177 } else {
14178 i40e_vsi_close(vsi);
14179 }
14180 i40e_vsi_disable_irq(vsi);
14181 }
14182
14183 if (vsi->type == I40E_VSI_MAIN)
14184 i40e_devlink_destroy_port(pf);
14185
14186 spin_lock_bh(&vsi->mac_filter_hash_lock);
14187
14188 /* clear the sync flag on all filters */
14189 if (vsi->netdev) {
14190 __dev_uc_unsync(vsi->netdev, NULL);
14191 __dev_mc_unsync(vsi->netdev, NULL);
14192 }
14193
14194 /* make sure any remaining filters are marked for deletion */
14195 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist)
14196 __i40e_del_filter(vsi, f);
14197
14198 spin_unlock_bh(&vsi->mac_filter_hash_lock);
14199
14200 i40e_sync_vsi_filters(vsi);
14201
14202 i40e_vsi_delete(vsi);
14203 i40e_vsi_free_q_vectors(vsi);
14204 if (vsi->netdev) {
14205 free_netdev(vsi->netdev);
14206 vsi->netdev = NULL;
14207 }
14208 i40e_vsi_clear_rings(vsi);
14209 i40e_vsi_clear(vsi);
14210
14211 /* If this was the last thing on the VEB, except for the
14212 * controlling VSI, remove the VEB, which puts the controlling
14213 * VSI onto the uplink port.
14214 *
14215 * Well, okay, there's one more exception here: don't remove
14216 * the floating VEBs yet. We'll wait for an explicit remove request
14217 * from up the network stack.
14218 */
14219 veb = i40e_pf_get_veb_by_seid(pf, uplink_seid);
14220 if (veb && veb->uplink_seid) {
14221 n = 0;
14222
14223 /* Count non-controlling VSIs present on the VEB */
14224 i40e_pf_for_each_vsi(pf, i, vsi)
14225 if (vsi->uplink_seid == uplink_seid &&
14226 (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)
14227 n++;
14228
14229 /* If there is no VSI except the control one then release
14230 * the VEB and put the control VSI onto VEB uplink.
14231 */
14232 if (!n)
14233 i40e_veb_release(veb);
14234 }
14235
14236 return 0;
14237 }
14238
14239 /**
14240 * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI
14241 * @vsi: ptr to the VSI
14242 *
14243 * This should only be called after i40e_vsi_mem_alloc() which allocates the
14244 * corresponding SW VSI structure and initializes num_queue_pairs for the
14245 * newly allocated VSI.
14246 *
14247 * Returns 0 on success or negative on failure
14248 **/
i40e_vsi_setup_vectors(struct i40e_vsi * vsi)14249 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi)
14250 {
14251 int ret = -ENOENT;
14252 struct i40e_pf *pf = vsi->back;
14253
14254 if (vsi->q_vectors[0]) {
14255 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n",
14256 vsi->seid);
14257 return -EEXIST;
14258 }
14259
14260 if (vsi->base_vector) {
14261 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n",
14262 vsi->seid, vsi->base_vector);
14263 return -EEXIST;
14264 }
14265
14266 ret = i40e_vsi_alloc_q_vectors(vsi);
14267 if (ret) {
14268 dev_info(&pf->pdev->dev,
14269 "failed to allocate %d q_vector for VSI %d, ret=%d\n",
14270 vsi->num_q_vectors, vsi->seid, ret);
14271 vsi->num_q_vectors = 0;
14272 goto vector_setup_out;
14273 }
14274
14275 /* In Legacy mode, we do not have to get any other vector since we
14276 * piggyback on the misc/ICR0 for queue interrupts.
14277 */
14278 if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
14279 return ret;
14280 if (vsi->num_q_vectors)
14281 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile,
14282 vsi->num_q_vectors, vsi->idx);
14283 if (vsi->base_vector < 0) {
14284 dev_info(&pf->pdev->dev,
14285 "failed to get tracking for %d vectors for VSI %d, err=%d\n",
14286 vsi->num_q_vectors, vsi->seid, vsi->base_vector);
14287 i40e_vsi_free_q_vectors(vsi);
14288 ret = -ENOENT;
14289 goto vector_setup_out;
14290 }
14291
14292 vector_setup_out:
14293 return ret;
14294 }
14295
14296 /**
14297 * i40e_vsi_reinit_setup - return and reallocate resources for a VSI
14298 * @vsi: pointer to the vsi.
14299 *
14300 * This re-allocates a vsi's queue resources.
14301 *
14302 * Returns pointer to the successfully allocated and configured VSI sw struct
14303 * on success, otherwise returns NULL on failure.
14304 **/
i40e_vsi_reinit_setup(struct i40e_vsi * vsi)14305 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi)
14306 {
14307 struct i40e_vsi *main_vsi;
14308 u16 alloc_queue_pairs;
14309 struct i40e_pf *pf;
14310 int ret;
14311
14312 if (!vsi)
14313 return NULL;
14314
14315 pf = vsi->back;
14316
14317 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
14318 i40e_vsi_clear_rings(vsi);
14319
14320 i40e_vsi_free_arrays(vsi, false);
14321 i40e_set_num_rings_in_vsi(vsi);
14322 ret = i40e_vsi_alloc_arrays(vsi, false);
14323 if (ret)
14324 goto err_vsi;
14325
14326 alloc_queue_pairs = vsi->alloc_queue_pairs *
14327 (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
14328
14329 ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);
14330 if (ret < 0) {
14331 dev_info(&pf->pdev->dev,
14332 "failed to get tracking for %d queues for VSI %d err %d\n",
14333 alloc_queue_pairs, vsi->seid, ret);
14334 goto err_vsi;
14335 }
14336 vsi->base_queue = ret;
14337
14338 /* Update the FW view of the VSI. Force a reset of TC and queue
14339 * layout configurations.
14340 */
14341 main_vsi = i40e_pf_get_main_vsi(pf);
14342 main_vsi->seid = pf->main_vsi_seid;
14343 i40e_vsi_reconfig_tc(main_vsi);
14344
14345 if (vsi->type == I40E_VSI_MAIN)
14346 i40e_rm_default_mac_filter(vsi, pf->hw.mac.perm_addr);
14347
14348 /* assign it some queues */
14349 ret = i40e_alloc_rings(vsi);
14350 if (ret)
14351 goto err_rings;
14352
14353 /* map all of the rings to the q_vectors */
14354 i40e_vsi_map_rings_to_vectors(vsi);
14355 return vsi;
14356
14357 err_rings:
14358 i40e_vsi_free_q_vectors(vsi);
14359 if (vsi->netdev_registered) {
14360 vsi->netdev_registered = false;
14361 unregister_netdev(vsi->netdev);
14362 free_netdev(vsi->netdev);
14363 vsi->netdev = NULL;
14364 }
14365 if (vsi->type == I40E_VSI_MAIN)
14366 i40e_devlink_destroy_port(pf);
14367 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
14368 err_vsi:
14369 i40e_vsi_clear(vsi);
14370 return NULL;
14371 }
14372
14373 /**
14374 * i40e_vsi_setup - Set up a VSI by a given type
14375 * @pf: board private structure
14376 * @type: VSI type
14377 * @uplink_seid: the switch element to link to
14378 * @param1: usage depends upon VSI type. For VF types, indicates VF id
14379 *
14380 * This allocates the sw VSI structure and its queue resources, then add a VSI
14381 * to the identified VEB.
14382 *
14383 * Returns pointer to the successfully allocated and configure VSI sw struct on
14384 * success, otherwise returns NULL on failure.
14385 **/
i40e_vsi_setup(struct i40e_pf * pf,u8 type,u16 uplink_seid,u32 param1)14386 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type,
14387 u16 uplink_seid, u32 param1)
14388 {
14389 struct i40e_vsi *vsi = NULL;
14390 struct i40e_veb *veb = NULL;
14391 u16 alloc_queue_pairs;
14392 int v_idx;
14393 int ret;
14394
14395 /* The requested uplink_seid must be either
14396 * - the PF's port seid
14397 * no VEB is needed because this is the PF
14398 * or this is a Flow Director special case VSI
14399 * - seid of an existing VEB
14400 * - seid of a VSI that owns an existing VEB
14401 * - seid of a VSI that doesn't own a VEB
14402 * a new VEB is created and the VSI becomes the owner
14403 * - seid of the PF VSI, which is what creates the first VEB
14404 * this is a special case of the previous
14405 *
14406 * Find which uplink_seid we were given and create a new VEB if needed
14407 */
14408 veb = i40e_pf_get_veb_by_seid(pf, uplink_seid);
14409 if (!veb && uplink_seid != pf->mac_seid) {
14410 vsi = i40e_pf_get_vsi_by_seid(pf, uplink_seid);
14411 if (!vsi) {
14412 dev_info(&pf->pdev->dev, "no such uplink_seid %d\n",
14413 uplink_seid);
14414 return NULL;
14415 }
14416
14417 if (vsi->uplink_seid == pf->mac_seid)
14418 veb = i40e_veb_setup(pf, pf->mac_seid, vsi->seid,
14419 vsi->tc_config.enabled_tc);
14420 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)
14421 veb = i40e_veb_setup(pf, vsi->uplink_seid, vsi->seid,
14422 vsi->tc_config.enabled_tc);
14423 if (veb) {
14424 if (vsi->type != I40E_VSI_MAIN) {
14425 dev_info(&vsi->back->pdev->dev,
14426 "New VSI creation error, uplink seid of LAN VSI expected.\n");
14427 return NULL;
14428 }
14429 /* We come up by default in VEPA mode if SRIOV is not
14430 * already enabled, in which case we can't force VEPA
14431 * mode.
14432 */
14433 if (!test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) {
14434 veb->bridge_mode = BRIDGE_MODE_VEPA;
14435 clear_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);
14436 }
14437 i40e_config_bridge_mode(veb);
14438 }
14439 veb = i40e_pf_get_veb_by_seid(pf, vsi->uplink_seid);
14440 if (!veb) {
14441 dev_info(&pf->pdev->dev, "couldn't add VEB\n");
14442 return NULL;
14443 }
14444
14445 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
14446 uplink_seid = veb->seid;
14447 }
14448
14449 /* get vsi sw struct */
14450 v_idx = i40e_vsi_mem_alloc(pf, type);
14451 if (v_idx < 0)
14452 goto err_alloc;
14453 vsi = pf->vsi[v_idx];
14454 if (!vsi)
14455 goto err_alloc;
14456 vsi->type = type;
14457 vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB);
14458
14459 if (type == I40E_VSI_MAIN)
14460 pf->lan_vsi = v_idx;
14461 else if (type == I40E_VSI_SRIOV)
14462 vsi->vf_id = param1;
14463 /* assign it some queues */
14464 alloc_queue_pairs = vsi->alloc_queue_pairs *
14465 (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
14466
14467 ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);
14468 if (ret < 0) {
14469 dev_info(&pf->pdev->dev,
14470 "failed to get tracking for %d queues for VSI %d err=%d\n",
14471 alloc_queue_pairs, vsi->seid, ret);
14472 goto err_vsi;
14473 }
14474 vsi->base_queue = ret;
14475
14476 /* get a VSI from the hardware */
14477 vsi->uplink_seid = uplink_seid;
14478 ret = i40e_add_vsi(vsi);
14479 if (ret)
14480 goto err_vsi;
14481
14482 switch (vsi->type) {
14483 /* setup the netdev if needed */
14484 case I40E_VSI_MAIN:
14485 case I40E_VSI_VMDQ2:
14486 ret = i40e_config_netdev(vsi);
14487 if (ret)
14488 goto err_netdev;
14489 ret = i40e_netif_set_realnum_tx_rx_queues(vsi);
14490 if (ret)
14491 goto err_netdev;
14492 if (vsi->type == I40E_VSI_MAIN) {
14493 ret = i40e_devlink_create_port(pf);
14494 if (ret)
14495 goto err_netdev;
14496 SET_NETDEV_DEVLINK_PORT(vsi->netdev, &pf->devlink_port);
14497 }
14498 ret = register_netdev(vsi->netdev);
14499 if (ret)
14500 goto err_dl_port;
14501 vsi->netdev_registered = true;
14502 netif_carrier_off(vsi->netdev);
14503 #ifdef CONFIG_I40E_DCB
14504 /* Setup DCB netlink interface */
14505 i40e_dcbnl_setup(vsi);
14506 #endif /* CONFIG_I40E_DCB */
14507 fallthrough;
14508 case I40E_VSI_FDIR:
14509 /* set up vectors and rings if needed */
14510 ret = i40e_vsi_setup_vectors(vsi);
14511 if (ret)
14512 goto err_msix;
14513
14514 ret = i40e_alloc_rings(vsi);
14515 if (ret)
14516 goto err_rings;
14517
14518 /* map all of the rings to the q_vectors */
14519 i40e_vsi_map_rings_to_vectors(vsi);
14520
14521 i40e_vsi_reset_stats(vsi);
14522 break;
14523 default:
14524 /* no netdev or rings for the other VSI types */
14525 break;
14526 }
14527
14528 if (test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps) &&
14529 vsi->type == I40E_VSI_VMDQ2) {
14530 ret = i40e_vsi_config_rss(vsi);
14531 if (ret)
14532 goto err_config;
14533 }
14534 return vsi;
14535
14536 err_config:
14537 i40e_vsi_clear_rings(vsi);
14538 err_rings:
14539 i40e_vsi_free_q_vectors(vsi);
14540 err_msix:
14541 if (vsi->netdev_registered) {
14542 vsi->netdev_registered = false;
14543 unregister_netdev(vsi->netdev);
14544 free_netdev(vsi->netdev);
14545 vsi->netdev = NULL;
14546 }
14547 err_dl_port:
14548 if (vsi->type == I40E_VSI_MAIN)
14549 i40e_devlink_destroy_port(pf);
14550 err_netdev:
14551 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
14552 err_vsi:
14553 i40e_vsi_clear(vsi);
14554 err_alloc:
14555 return NULL;
14556 }
14557
14558 /**
14559 * i40e_veb_get_bw_info - Query VEB BW information
14560 * @veb: the veb to query
14561 *
14562 * Query the Tx scheduler BW configuration data for given VEB
14563 **/
i40e_veb_get_bw_info(struct i40e_veb * veb)14564 static int i40e_veb_get_bw_info(struct i40e_veb *veb)
14565 {
14566 struct i40e_aqc_query_switching_comp_ets_config_resp ets_data;
14567 struct i40e_aqc_query_switching_comp_bw_config_resp bw_data;
14568 struct i40e_pf *pf = veb->pf;
14569 struct i40e_hw *hw = &pf->hw;
14570 u32 tc_bw_max;
14571 int ret = 0;
14572 int i;
14573
14574 ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid,
14575 &bw_data, NULL);
14576 if (ret) {
14577 dev_info(&pf->pdev->dev,
14578 "query veb bw config failed, err %pe aq_err %s\n",
14579 ERR_PTR(ret),
14580 i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
14581 goto out;
14582 }
14583
14584 ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid,
14585 &ets_data, NULL);
14586 if (ret) {
14587 dev_info(&pf->pdev->dev,
14588 "query veb bw ets config failed, err %pe aq_err %s\n",
14589 ERR_PTR(ret),
14590 i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
14591 goto out;
14592 }
14593
14594 veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit);
14595 veb->bw_max_quanta = ets_data.tc_bw_max;
14596 veb->is_abs_credits = bw_data.absolute_credits_enable;
14597 veb->enabled_tc = ets_data.tc_valid_bits;
14598 tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) |
14599 (le16_to_cpu(bw_data.tc_bw_max[1]) << 16);
14600 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
14601 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i];
14602 veb->bw_tc_limit_credits[i] =
14603 le16_to_cpu(bw_data.tc_bw_limits[i]);
14604 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7);
14605 }
14606
14607 out:
14608 return ret;
14609 }
14610
14611 /**
14612 * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF
14613 * @pf: board private structure
14614 *
14615 * On error: returns error code (negative)
14616 * On success: returns vsi index in PF (positive)
14617 **/
i40e_veb_mem_alloc(struct i40e_pf * pf)14618 static int i40e_veb_mem_alloc(struct i40e_pf *pf)
14619 {
14620 int ret = -ENOENT;
14621 struct i40e_veb *veb;
14622 int i;
14623
14624 /* Need to protect the allocation of switch elements at the PF level */
14625 mutex_lock(&pf->switch_mutex);
14626
14627 /* VEB list may be fragmented if VEB creation/destruction has
14628 * been happening. We can afford to do a quick scan to look
14629 * for any free slots in the list.
14630 *
14631 * find next empty veb slot, looping back around if necessary
14632 */
14633 i = 0;
14634 while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL))
14635 i++;
14636 if (i >= I40E_MAX_VEB) {
14637 ret = -ENOMEM;
14638 goto err_alloc_veb; /* out of VEB slots! */
14639 }
14640
14641 veb = kzalloc(sizeof(*veb), GFP_KERNEL);
14642 if (!veb) {
14643 ret = -ENOMEM;
14644 goto err_alloc_veb;
14645 }
14646 veb->pf = pf;
14647 veb->idx = i;
14648 veb->enabled_tc = 1;
14649
14650 pf->veb[i] = veb;
14651 ret = i;
14652 err_alloc_veb:
14653 mutex_unlock(&pf->switch_mutex);
14654 return ret;
14655 }
14656
14657 /**
14658 * i40e_switch_branch_release - Delete a branch of the switch tree
14659 * @branch: where to start deleting
14660 *
14661 * This uses recursion to find the tips of the branch to be
14662 * removed, deleting until we get back to and can delete this VEB.
14663 **/
i40e_switch_branch_release(struct i40e_veb * branch)14664 static void i40e_switch_branch_release(struct i40e_veb *branch)
14665 {
14666 struct i40e_pf *pf = branch->pf;
14667 u16 branch_seid = branch->seid;
14668 u16 veb_idx = branch->idx;
14669 struct i40e_vsi *vsi;
14670 struct i40e_veb *veb;
14671 int i;
14672
14673 /* release any VEBs on this VEB - RECURSION */
14674 i40e_pf_for_each_veb(pf, i, veb)
14675 if (veb->uplink_seid == branch->seid)
14676 i40e_switch_branch_release(veb);
14677
14678 /* Release the VSIs on this VEB, but not the owner VSI.
14679 *
14680 * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing
14681 * the VEB itself, so don't use (*branch) after this loop.
14682 */
14683 i40e_pf_for_each_vsi(pf, i, vsi)
14684 if (vsi->uplink_seid == branch_seid &&
14685 (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)
14686 i40e_vsi_release(vsi);
14687
14688 /* There's one corner case where the VEB might not have been
14689 * removed, so double check it here and remove it if needed.
14690 * This case happens if the veb was created from the debugfs
14691 * commands and no VSIs were added to it.
14692 */
14693 if (pf->veb[veb_idx])
14694 i40e_veb_release(pf->veb[veb_idx]);
14695 }
14696
14697 /**
14698 * i40e_veb_clear - remove veb struct
14699 * @veb: the veb to remove
14700 **/
i40e_veb_clear(struct i40e_veb * veb)14701 static void i40e_veb_clear(struct i40e_veb *veb)
14702 {
14703 if (!veb)
14704 return;
14705
14706 if (veb->pf) {
14707 struct i40e_pf *pf = veb->pf;
14708
14709 mutex_lock(&pf->switch_mutex);
14710 if (pf->veb[veb->idx] == veb)
14711 pf->veb[veb->idx] = NULL;
14712 mutex_unlock(&pf->switch_mutex);
14713 }
14714
14715 kfree(veb);
14716 }
14717
14718 /**
14719 * i40e_veb_release - Delete a VEB and free its resources
14720 * @veb: the VEB being removed
14721 **/
i40e_veb_release(struct i40e_veb * veb)14722 void i40e_veb_release(struct i40e_veb *veb)
14723 {
14724 struct i40e_vsi *vsi, *vsi_it;
14725 struct i40e_pf *pf;
14726 int i, n = 0;
14727
14728 pf = veb->pf;
14729
14730 /* find the remaining VSI and check for extras */
14731 i40e_pf_for_each_vsi(pf, i, vsi_it)
14732 if (vsi_it->uplink_seid == veb->seid) {
14733 if (vsi_it->flags & I40E_VSI_FLAG_VEB_OWNER)
14734 vsi = vsi_it;
14735 n++;
14736 }
14737
14738 /* Floating VEB has to be empty and regular one must have
14739 * single owner VSI.
14740 */
14741 if ((veb->uplink_seid && n != 1) || (!veb->uplink_seid && n != 0)) {
14742 dev_info(&pf->pdev->dev,
14743 "can't remove VEB %d with %d VSIs left\n",
14744 veb->seid, n);
14745 return;
14746 }
14747
14748 /* For regular VEB move the owner VSI to uplink port */
14749 if (veb->uplink_seid) {
14750 vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER;
14751 vsi->uplink_seid = veb->uplink_seid;
14752 vsi->veb_idx = I40E_NO_VEB;
14753 }
14754
14755 i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
14756 i40e_veb_clear(veb);
14757 }
14758
14759 /**
14760 * i40e_add_veb - create the VEB in the switch
14761 * @veb: the VEB to be instantiated
14762 * @vsi: the controlling VSI
14763 **/
i40e_add_veb(struct i40e_veb * veb,struct i40e_vsi * vsi)14764 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi)
14765 {
14766 struct i40e_pf *pf = veb->pf;
14767 bool enable_stats = !!test_bit(I40E_FLAG_VEB_STATS_ENA, pf->flags);
14768 int ret;
14769
14770 ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi ? vsi->seid : 0,
14771 veb->enabled_tc, vsi ? false : true,
14772 &veb->seid, enable_stats, NULL);
14773
14774 /* get a VEB from the hardware */
14775 if (ret) {
14776 dev_info(&pf->pdev->dev,
14777 "couldn't add VEB, err %pe aq_err %s\n",
14778 ERR_PTR(ret),
14779 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14780 return -EPERM;
14781 }
14782
14783 /* get statistics counter */
14784 ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL,
14785 &veb->stats_idx, NULL, NULL, NULL);
14786 if (ret) {
14787 dev_info(&pf->pdev->dev,
14788 "couldn't get VEB statistics idx, err %pe aq_err %s\n",
14789 ERR_PTR(ret),
14790 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14791 return -EPERM;
14792 }
14793 ret = i40e_veb_get_bw_info(veb);
14794 if (ret) {
14795 dev_info(&pf->pdev->dev,
14796 "couldn't get VEB bw info, err %pe aq_err %s\n",
14797 ERR_PTR(ret),
14798 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14799 i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
14800 return -ENOENT;
14801 }
14802
14803 if (vsi) {
14804 vsi->uplink_seid = veb->seid;
14805 vsi->veb_idx = veb->idx;
14806 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
14807 }
14808
14809 return 0;
14810 }
14811
14812 /**
14813 * i40e_veb_setup - Set up a VEB
14814 * @pf: board private structure
14815 * @uplink_seid: the switch element to link to
14816 * @vsi_seid: the initial VSI seid
14817 * @enabled_tc: Enabled TC bit-map
14818 *
14819 * This allocates the sw VEB structure and links it into the switch
14820 * It is possible and legal for this to be a duplicate of an already
14821 * existing VEB. It is also possible for both uplink and vsi seids
14822 * to be zero, in order to create a floating VEB.
14823 *
14824 * Returns pointer to the successfully allocated VEB sw struct on
14825 * success, otherwise returns NULL on failure.
14826 **/
i40e_veb_setup(struct i40e_pf * pf,u16 uplink_seid,u16 vsi_seid,u8 enabled_tc)14827 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 uplink_seid,
14828 u16 vsi_seid, u8 enabled_tc)
14829 {
14830 struct i40e_vsi *vsi = NULL;
14831 struct i40e_veb *veb;
14832 int veb_idx;
14833 int ret;
14834
14835 /* if one seid is 0, the other must be 0 to create a floating relay */
14836 if ((uplink_seid == 0 || vsi_seid == 0) &&
14837 (uplink_seid + vsi_seid != 0)) {
14838 dev_info(&pf->pdev->dev,
14839 "one, not both seid's are 0: uplink=%d vsi=%d\n",
14840 uplink_seid, vsi_seid);
14841 return NULL;
14842 }
14843
14844 /* make sure there is such a vsi and uplink */
14845 if (vsi_seid) {
14846 vsi = i40e_pf_get_vsi_by_seid(pf, vsi_seid);
14847 if (!vsi) {
14848 dev_err(&pf->pdev->dev, "vsi seid %d not found\n",
14849 vsi_seid);
14850 return NULL;
14851 }
14852 }
14853
14854 /* get veb sw struct */
14855 veb_idx = i40e_veb_mem_alloc(pf);
14856 if (veb_idx < 0)
14857 goto err_alloc;
14858 veb = pf->veb[veb_idx];
14859 veb->uplink_seid = uplink_seid;
14860 veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1);
14861
14862 /* create the VEB in the switch */
14863 ret = i40e_add_veb(veb, vsi);
14864 if (ret)
14865 goto err_veb;
14866
14867 if (vsi && vsi->idx == pf->lan_vsi)
14868 pf->lan_veb = veb->idx;
14869
14870 return veb;
14871
14872 err_veb:
14873 i40e_veb_clear(veb);
14874 err_alloc:
14875 return NULL;
14876 }
14877
14878 /**
14879 * i40e_setup_pf_switch_element - set PF vars based on switch type
14880 * @pf: board private structure
14881 * @ele: element we are building info from
14882 * @num_reported: total number of elements
14883 * @printconfig: should we print the contents
14884 *
14885 * helper function to assist in extracting a few useful SEID values.
14886 **/
i40e_setup_pf_switch_element(struct i40e_pf * pf,struct i40e_aqc_switch_config_element_resp * ele,u16 num_reported,bool printconfig)14887 static void i40e_setup_pf_switch_element(struct i40e_pf *pf,
14888 struct i40e_aqc_switch_config_element_resp *ele,
14889 u16 num_reported, bool printconfig)
14890 {
14891 u16 downlink_seid = le16_to_cpu(ele->downlink_seid);
14892 u16 uplink_seid = le16_to_cpu(ele->uplink_seid);
14893 u8 element_type = ele->element_type;
14894 u16 seid = le16_to_cpu(ele->seid);
14895 struct i40e_veb *veb;
14896
14897 if (printconfig)
14898 dev_info(&pf->pdev->dev,
14899 "type=%d seid=%d uplink=%d downlink=%d\n",
14900 element_type, seid, uplink_seid, downlink_seid);
14901
14902 switch (element_type) {
14903 case I40E_SWITCH_ELEMENT_TYPE_MAC:
14904 pf->mac_seid = seid;
14905 break;
14906 case I40E_SWITCH_ELEMENT_TYPE_VEB:
14907 /* Main VEB? */
14908 if (uplink_seid != pf->mac_seid)
14909 break;
14910 veb = i40e_pf_get_main_veb(pf);
14911 if (!veb) {
14912 int v;
14913
14914 /* find existing or else empty VEB */
14915 veb = i40e_pf_get_veb_by_seid(pf, seid);
14916 if (veb) {
14917 pf->lan_veb = veb->idx;
14918 } else {
14919 v = i40e_veb_mem_alloc(pf);
14920 if (v < 0)
14921 break;
14922 pf->lan_veb = v;
14923 }
14924 }
14925
14926 /* Try to get again main VEB as pf->lan_veb may have changed */
14927 veb = i40e_pf_get_main_veb(pf);
14928 if (!veb)
14929 break;
14930
14931 veb->seid = seid;
14932 veb->uplink_seid = pf->mac_seid;
14933 veb->pf = pf;
14934 break;
14935 case I40E_SWITCH_ELEMENT_TYPE_VSI:
14936 if (num_reported != 1)
14937 break;
14938 /* This is immediately after a reset so we can assume this is
14939 * the PF's VSI
14940 */
14941 pf->mac_seid = uplink_seid;
14942 pf->main_vsi_seid = seid;
14943 if (printconfig)
14944 dev_info(&pf->pdev->dev,
14945 "pf_seid=%d main_vsi_seid=%d\n",
14946 downlink_seid, pf->main_vsi_seid);
14947 break;
14948 case I40E_SWITCH_ELEMENT_TYPE_PF:
14949 case I40E_SWITCH_ELEMENT_TYPE_VF:
14950 case I40E_SWITCH_ELEMENT_TYPE_EMP:
14951 case I40E_SWITCH_ELEMENT_TYPE_BMC:
14952 case I40E_SWITCH_ELEMENT_TYPE_PE:
14953 case I40E_SWITCH_ELEMENT_TYPE_PA:
14954 /* ignore these for now */
14955 break;
14956 default:
14957 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n",
14958 element_type, seid);
14959 break;
14960 }
14961 }
14962
14963 /**
14964 * i40e_fetch_switch_configuration - Get switch config from firmware
14965 * @pf: board private structure
14966 * @printconfig: should we print the contents
14967 *
14968 * Get the current switch configuration from the device and
14969 * extract a few useful SEID values.
14970 **/
i40e_fetch_switch_configuration(struct i40e_pf * pf,bool printconfig)14971 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig)
14972 {
14973 struct i40e_aqc_get_switch_config_resp *sw_config;
14974 u16 next_seid = 0;
14975 int ret = 0;
14976 u8 *aq_buf;
14977 int i;
14978
14979 aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL);
14980 if (!aq_buf)
14981 return -ENOMEM;
14982
14983 sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
14984 do {
14985 u16 num_reported, num_total;
14986
14987 ret = i40e_aq_get_switch_config(&pf->hw, sw_config,
14988 I40E_AQ_LARGE_BUF,
14989 &next_seid, NULL);
14990 if (ret) {
14991 dev_info(&pf->pdev->dev,
14992 "get switch config failed err %d aq_err %s\n",
14993 ret,
14994 i40e_aq_str(&pf->hw,
14995 pf->hw.aq.asq_last_status));
14996 kfree(aq_buf);
14997 return -ENOENT;
14998 }
14999
15000 num_reported = le16_to_cpu(sw_config->header.num_reported);
15001 num_total = le16_to_cpu(sw_config->header.num_total);
15002
15003 if (printconfig)
15004 dev_info(&pf->pdev->dev,
15005 "header: %d reported %d total\n",
15006 num_reported, num_total);
15007
15008 for (i = 0; i < num_reported; i++) {
15009 struct i40e_aqc_switch_config_element_resp *ele =
15010 &sw_config->element[i];
15011
15012 i40e_setup_pf_switch_element(pf, ele, num_reported,
15013 printconfig);
15014 }
15015 } while (next_seid != 0);
15016
15017 kfree(aq_buf);
15018 return ret;
15019 }
15020
15021 /**
15022 * i40e_setup_pf_switch - Setup the HW switch on startup or after reset
15023 * @pf: board private structure
15024 * @reinit: if the Main VSI needs to re-initialized.
15025 * @lock_acquired: indicates whether or not the lock has been acquired
15026 *
15027 * Returns 0 on success, negative value on failure
15028 **/
i40e_setup_pf_switch(struct i40e_pf * pf,bool reinit,bool lock_acquired)15029 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit, bool lock_acquired)
15030 {
15031 struct i40e_vsi *main_vsi;
15032 u16 flags = 0;
15033 int ret;
15034
15035 /* find out what's out there already */
15036 ret = i40e_fetch_switch_configuration(pf, false);
15037 if (ret) {
15038 dev_info(&pf->pdev->dev,
15039 "couldn't fetch switch config, err %pe aq_err %s\n",
15040 ERR_PTR(ret),
15041 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
15042 return ret;
15043 }
15044 i40e_pf_reset_stats(pf);
15045
15046 /* set the switch config bit for the whole device to
15047 * support limited promisc or true promisc
15048 * when user requests promisc. The default is limited
15049 * promisc.
15050 */
15051
15052 if ((pf->hw.pf_id == 0) &&
15053 !test_bit(I40E_FLAG_TRUE_PROMISC_ENA, pf->flags)) {
15054 flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
15055 pf->last_sw_conf_flags = flags;
15056 }
15057
15058 if (pf->hw.pf_id == 0) {
15059 u16 valid_flags;
15060
15061 valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
15062 ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags, 0,
15063 NULL);
15064 if (ret && pf->hw.aq.asq_last_status != I40E_AQ_RC_ESRCH) {
15065 dev_info(&pf->pdev->dev,
15066 "couldn't set switch config bits, err %pe aq_err %s\n",
15067 ERR_PTR(ret),
15068 i40e_aq_str(&pf->hw,
15069 pf->hw.aq.asq_last_status));
15070 /* not a fatal problem, just keep going */
15071 }
15072 pf->last_sw_conf_valid_flags = valid_flags;
15073 }
15074
15075 /* first time setup */
15076 main_vsi = i40e_pf_get_main_vsi(pf);
15077 if (!main_vsi || reinit) {
15078 struct i40e_veb *veb;
15079 u16 uplink_seid;
15080
15081 /* Set up the PF VSI associated with the PF's main VSI
15082 * that is already in the HW switch
15083 */
15084 veb = i40e_pf_get_main_veb(pf);
15085 if (veb)
15086 uplink_seid = veb->seid;
15087 else
15088 uplink_seid = pf->mac_seid;
15089 if (!main_vsi)
15090 main_vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN,
15091 uplink_seid, 0);
15092 else if (reinit)
15093 main_vsi = i40e_vsi_reinit_setup(main_vsi);
15094 if (!main_vsi) {
15095 dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n");
15096 i40e_cloud_filter_exit(pf);
15097 i40e_fdir_teardown(pf);
15098 return -EAGAIN;
15099 }
15100 } else {
15101 /* force a reset of TC and queue layout configurations */
15102 main_vsi->seid = pf->main_vsi_seid;
15103 i40e_vsi_reconfig_tc(main_vsi);
15104 }
15105 i40e_vlan_stripping_disable(main_vsi);
15106
15107 i40e_fdir_sb_setup(pf);
15108
15109 /* Setup static PF queue filter control settings */
15110 ret = i40e_setup_pf_filter_control(pf);
15111 if (ret) {
15112 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n",
15113 ret);
15114 /* Failure here should not stop continuing other steps */
15115 }
15116
15117 /* enable RSS in the HW, even for only one queue, as the stack can use
15118 * the hash
15119 */
15120 if (test_bit(I40E_FLAG_RSS_ENA, pf->flags))
15121 i40e_pf_config_rss(pf);
15122
15123 /* fill in link information and enable LSE reporting */
15124 i40e_link_event(pf);
15125
15126 i40e_ptp_init(pf);
15127
15128 if (!lock_acquired)
15129 rtnl_lock();
15130
15131 /* repopulate tunnel port filters */
15132 udp_tunnel_nic_reset_ntf(main_vsi->netdev);
15133
15134 if (!lock_acquired)
15135 rtnl_unlock();
15136
15137 return ret;
15138 }
15139
15140 /**
15141 * i40e_determine_queue_usage - Work out queue distribution
15142 * @pf: board private structure
15143 **/
i40e_determine_queue_usage(struct i40e_pf * pf)15144 static void i40e_determine_queue_usage(struct i40e_pf *pf)
15145 {
15146 int queues_left;
15147 int q_max;
15148
15149 pf->num_lan_qps = 0;
15150
15151 /* Find the max queues to be put into basic use. We'll always be
15152 * using TC0, whether or not DCB is running, and TC0 will get the
15153 * big RSS set.
15154 */
15155 queues_left = pf->hw.func_caps.num_tx_qp;
15156
15157 if ((queues_left == 1) ||
15158 !test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
15159 /* one qp for PF, no queues for anything else */
15160 queues_left = 0;
15161 pf->alloc_rss_size = pf->num_lan_qps = 1;
15162
15163 /* make sure all the fancies are disabled */
15164 clear_bit(I40E_FLAG_RSS_ENA, pf->flags);
15165 clear_bit(I40E_FLAG_IWARP_ENA, pf->flags);
15166 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
15167 clear_bit(I40E_FLAG_FD_ATR_ENA, pf->flags);
15168 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
15169 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
15170 clear_bit(I40E_FLAG_SRIOV_ENA, pf->flags);
15171 clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags);
15172 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
15173 } else if (!test_bit(I40E_FLAG_RSS_ENA, pf->flags) &&
15174 !test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) &&
15175 !test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) &&
15176 !test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags)) {
15177 /* one qp for PF */
15178 pf->alloc_rss_size = pf->num_lan_qps = 1;
15179 queues_left -= pf->num_lan_qps;
15180
15181 clear_bit(I40E_FLAG_RSS_ENA, pf->flags);
15182 clear_bit(I40E_FLAG_IWARP_ENA, pf->flags);
15183 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
15184 clear_bit(I40E_FLAG_FD_ATR_ENA, pf->flags);
15185 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
15186 clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags);
15187 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
15188 } else {
15189 /* Not enough queues for all TCs */
15190 if (test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags) &&
15191 queues_left < I40E_MAX_TRAFFIC_CLASS) {
15192 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
15193 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
15194 dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n");
15195 }
15196
15197 /* limit lan qps to the smaller of qps, cpus or msix */
15198 q_max = max_t(int, pf->rss_size_max, num_online_cpus());
15199 q_max = min_t(int, q_max, pf->hw.func_caps.num_tx_qp);
15200 q_max = min_t(int, q_max, pf->hw.func_caps.num_msix_vectors);
15201 pf->num_lan_qps = q_max;
15202
15203 queues_left -= pf->num_lan_qps;
15204 }
15205
15206 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {
15207 if (queues_left > 1) {
15208 queues_left -= 1; /* save 1 queue for FD */
15209 } else {
15210 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
15211 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
15212 dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n");
15213 }
15214 }
15215
15216 if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags) &&
15217 pf->num_vf_qps && pf->num_req_vfs && queues_left) {
15218 pf->num_req_vfs = min_t(int, pf->num_req_vfs,
15219 (queues_left / pf->num_vf_qps));
15220 queues_left -= (pf->num_req_vfs * pf->num_vf_qps);
15221 }
15222
15223 if (test_bit(I40E_FLAG_VMDQ_ENA, pf->flags) &&
15224 pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) {
15225 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis,
15226 (queues_left / pf->num_vmdq_qps));
15227 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps);
15228 }
15229
15230 pf->queues_left = queues_left;
15231 dev_dbg(&pf->pdev->dev,
15232 "qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n",
15233 pf->hw.func_caps.num_tx_qp,
15234 !!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags),
15235 pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs,
15236 pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps,
15237 queues_left);
15238 }
15239
15240 /**
15241 * i40e_setup_pf_filter_control - Setup PF static filter control
15242 * @pf: PF to be setup
15243 *
15244 * i40e_setup_pf_filter_control sets up a PF's initial filter control
15245 * settings. If PE/FCoE are enabled then it will also set the per PF
15246 * based filter sizes required for them. It also enables Flow director,
15247 * ethertype and macvlan type filter settings for the pf.
15248 *
15249 * Returns 0 on success, negative on failure
15250 **/
i40e_setup_pf_filter_control(struct i40e_pf * pf)15251 static int i40e_setup_pf_filter_control(struct i40e_pf *pf)
15252 {
15253 struct i40e_filter_control_settings *settings = &pf->filter_settings;
15254
15255 settings->hash_lut_size = I40E_HASH_LUT_SIZE_128;
15256
15257 /* Flow Director is enabled */
15258 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) ||
15259 test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags))
15260 settings->enable_fdir = true;
15261
15262 /* Ethtype and MACVLAN filters enabled for PF */
15263 settings->enable_ethtype = true;
15264 settings->enable_macvlan = true;
15265
15266 if (i40e_set_filter_control(&pf->hw, settings))
15267 return -ENOENT;
15268
15269 return 0;
15270 }
15271
15272 #define INFO_STRING_LEN 255
15273 #define REMAIN(__x) (INFO_STRING_LEN - (__x))
i40e_print_features(struct i40e_pf * pf)15274 static void i40e_print_features(struct i40e_pf *pf)
15275 {
15276 struct i40e_vsi *main_vsi = i40e_pf_get_main_vsi(pf);
15277 struct i40e_hw *hw = &pf->hw;
15278 char *buf;
15279 int i;
15280
15281 buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL);
15282 if (!buf)
15283 return;
15284
15285 i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id);
15286 #ifdef CONFIG_PCI_IOV
15287 i += scnprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs);
15288 #endif
15289 i += scnprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d",
15290 pf->hw.func_caps.num_vsis, main_vsi->num_queue_pairs);
15291 if (test_bit(I40E_FLAG_RSS_ENA, pf->flags))
15292 i += scnprintf(&buf[i], REMAIN(i), " RSS");
15293 if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags))
15294 i += scnprintf(&buf[i], REMAIN(i), " FD_ATR");
15295 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {
15296 i += scnprintf(&buf[i], REMAIN(i), " FD_SB");
15297 i += scnprintf(&buf[i], REMAIN(i), " NTUPLE");
15298 }
15299 if (test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags))
15300 i += scnprintf(&buf[i], REMAIN(i), " DCB");
15301 i += scnprintf(&buf[i], REMAIN(i), " VxLAN");
15302 i += scnprintf(&buf[i], REMAIN(i), " Geneve");
15303 if (test_bit(I40E_FLAG_PTP_ENA, pf->flags))
15304 i += scnprintf(&buf[i], REMAIN(i), " PTP");
15305 if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags))
15306 i += scnprintf(&buf[i], REMAIN(i), " VEB");
15307 else
15308 i += scnprintf(&buf[i], REMAIN(i), " VEPA");
15309
15310 dev_info(&pf->pdev->dev, "%s\n", buf);
15311 kfree(buf);
15312 WARN_ON(i > INFO_STRING_LEN);
15313 }
15314
15315 /**
15316 * i40e_get_platform_mac_addr - get platform-specific MAC address
15317 * @pdev: PCI device information struct
15318 * @pf: board private structure
15319 *
15320 * Look up the MAC address for the device. First we'll try
15321 * eth_platform_get_mac_address, which will check Open Firmware, or arch
15322 * specific fallback. Otherwise, we'll default to the stored value in
15323 * firmware.
15324 **/
i40e_get_platform_mac_addr(struct pci_dev * pdev,struct i40e_pf * pf)15325 static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf)
15326 {
15327 if (eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr))
15328 i40e_get_mac_addr(&pf->hw, pf->hw.mac.addr);
15329 }
15330
15331 /**
15332 * i40e_set_fec_in_flags - helper function for setting FEC options in flags
15333 * @fec_cfg: FEC option to set in flags
15334 * @flags: ptr to flags in which we set FEC option
15335 **/
i40e_set_fec_in_flags(u8 fec_cfg,unsigned long * flags)15336 void i40e_set_fec_in_flags(u8 fec_cfg, unsigned long *flags)
15337 {
15338 if (fec_cfg & I40E_AQ_SET_FEC_AUTO) {
15339 set_bit(I40E_FLAG_RS_FEC, flags);
15340 set_bit(I40E_FLAG_BASE_R_FEC, flags);
15341 }
15342 if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_RS) ||
15343 (fec_cfg & I40E_AQ_SET_FEC_ABILITY_RS)) {
15344 set_bit(I40E_FLAG_RS_FEC, flags);
15345 clear_bit(I40E_FLAG_BASE_R_FEC, flags);
15346 }
15347 if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_KR) ||
15348 (fec_cfg & I40E_AQ_SET_FEC_ABILITY_KR)) {
15349 set_bit(I40E_FLAG_BASE_R_FEC, flags);
15350 clear_bit(I40E_FLAG_RS_FEC, flags);
15351 }
15352 if (fec_cfg == 0) {
15353 clear_bit(I40E_FLAG_RS_FEC, flags);
15354 clear_bit(I40E_FLAG_BASE_R_FEC, flags);
15355 }
15356 }
15357
15358 /**
15359 * i40e_check_recovery_mode - check if we are running transition firmware
15360 * @pf: board private structure
15361 *
15362 * Check registers indicating the firmware runs in recovery mode. Sets the
15363 * appropriate driver state.
15364 *
15365 * Returns true if the recovery mode was detected, false otherwise
15366 **/
i40e_check_recovery_mode(struct i40e_pf * pf)15367 static bool i40e_check_recovery_mode(struct i40e_pf *pf)
15368 {
15369 u32 val = rd32(&pf->hw, I40E_GL_FWSTS);
15370
15371 if (val & I40E_GL_FWSTS_FWS1B_MASK) {
15372 dev_crit(&pf->pdev->dev, "Firmware recovery mode detected. Limiting functionality.\n");
15373 dev_crit(&pf->pdev->dev, "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n");
15374 set_bit(__I40E_RECOVERY_MODE, pf->state);
15375
15376 return true;
15377 }
15378 if (test_bit(__I40E_RECOVERY_MODE, pf->state))
15379 dev_info(&pf->pdev->dev, "Please do Power-On Reset to initialize adapter in normal mode with full functionality.\n");
15380
15381 return false;
15382 }
15383
15384 /**
15385 * i40e_pf_loop_reset - perform reset in a loop.
15386 * @pf: board private structure
15387 *
15388 * This function is useful when a NIC is about to enter recovery mode.
15389 * When a NIC's internal data structures are corrupted the NIC's
15390 * firmware is going to enter recovery mode.
15391 * Right after a POR it takes about 7 minutes for firmware to enter
15392 * recovery mode. Until that time a NIC is in some kind of intermediate
15393 * state. After that time period the NIC almost surely enters
15394 * recovery mode. The only way for a driver to detect intermediate
15395 * state is to issue a series of pf-resets and check a return value.
15396 * If a PF reset returns success then the firmware could be in recovery
15397 * mode so the caller of this code needs to check for recovery mode
15398 * if this function returns success. There is a little chance that
15399 * firmware will hang in intermediate state forever.
15400 * Since waiting 7 minutes is quite a lot of time this function waits
15401 * 10 seconds and then gives up by returning an error.
15402 *
15403 * Return 0 on success, negative on failure.
15404 **/
i40e_pf_loop_reset(struct i40e_pf * pf)15405 static int i40e_pf_loop_reset(struct i40e_pf *pf)
15406 {
15407 /* wait max 10 seconds for PF reset to succeed */
15408 const unsigned long time_end = jiffies + 10 * HZ;
15409 struct i40e_hw *hw = &pf->hw;
15410 int ret;
15411
15412 ret = i40e_pf_reset(hw);
15413 while (ret != 0 && time_before(jiffies, time_end)) {
15414 usleep_range(10000, 20000);
15415 ret = i40e_pf_reset(hw);
15416 }
15417
15418 if (ret == 0)
15419 pf->pfr_count++;
15420 else
15421 dev_info(&pf->pdev->dev, "PF reset failed: %d\n", ret);
15422
15423 return ret;
15424 }
15425
15426 /**
15427 * i40e_check_fw_empr - check if FW issued unexpected EMP Reset
15428 * @pf: board private structure
15429 *
15430 * Check FW registers to determine if FW issued unexpected EMP Reset.
15431 * Every time when unexpected EMP Reset occurs the FW increments
15432 * a counter of unexpected EMP Resets. When the counter reaches 10
15433 * the FW should enter the Recovery mode
15434 *
15435 * Returns true if FW issued unexpected EMP Reset
15436 **/
i40e_check_fw_empr(struct i40e_pf * pf)15437 static bool i40e_check_fw_empr(struct i40e_pf *pf)
15438 {
15439 const u32 fw_sts = rd32(&pf->hw, I40E_GL_FWSTS) &
15440 I40E_GL_FWSTS_FWS1B_MASK;
15441 return (fw_sts > I40E_GL_FWSTS_FWS1B_EMPR_0) &&
15442 (fw_sts <= I40E_GL_FWSTS_FWS1B_EMPR_10);
15443 }
15444
15445 /**
15446 * i40e_handle_resets - handle EMP resets and PF resets
15447 * @pf: board private structure
15448 *
15449 * Handle both EMP resets and PF resets and conclude whether there are
15450 * any issues regarding these resets. If there are any issues then
15451 * generate log entry.
15452 *
15453 * Return 0 if NIC is healthy or negative value when there are issues
15454 * with resets
15455 **/
i40e_handle_resets(struct i40e_pf * pf)15456 static int i40e_handle_resets(struct i40e_pf *pf)
15457 {
15458 const int pfr = i40e_pf_loop_reset(pf);
15459 const bool is_empr = i40e_check_fw_empr(pf);
15460
15461 if (is_empr || pfr != 0)
15462 dev_crit(&pf->pdev->dev, "Entering recovery mode due to repeated FW resets. This may take several minutes. Refer to the Intel(R) Ethernet Adapters and Devices User Guide.\n");
15463
15464 return is_empr ? -EIO : pfr;
15465 }
15466
15467 /**
15468 * i40e_init_recovery_mode - initialize subsystems needed in recovery mode
15469 * @pf: board private structure
15470 * @hw: ptr to the hardware info
15471 *
15472 * This function does a minimal setup of all subsystems needed for running
15473 * recovery mode.
15474 *
15475 * Returns 0 on success, negative on failure
15476 **/
i40e_init_recovery_mode(struct i40e_pf * pf,struct i40e_hw * hw)15477 static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw)
15478 {
15479 struct i40e_vsi *vsi;
15480 int err;
15481 int v_idx;
15482
15483 pci_set_drvdata(pf->pdev, pf);
15484 pci_save_state(pf->pdev);
15485
15486 /* set up periodic task facility */
15487 timer_setup(&pf->service_timer, i40e_service_timer, 0);
15488 pf->service_timer_period = HZ;
15489
15490 INIT_WORK(&pf->service_task, i40e_service_task);
15491 clear_bit(__I40E_SERVICE_SCHED, pf->state);
15492
15493 err = i40e_init_interrupt_scheme(pf);
15494 if (err)
15495 goto err_switch_setup;
15496
15497 /* The number of VSIs reported by the FW is the minimum guaranteed
15498 * to us; HW supports far more and we share the remaining pool with
15499 * the other PFs. We allocate space for more than the guarantee with
15500 * the understanding that we might not get them all later.
15501 */
15502 if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)
15503 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;
15504 else
15505 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;
15506
15507 /* Set up the vsi struct and our local tracking of the MAIN PF vsi. */
15508 pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *),
15509 GFP_KERNEL);
15510 if (!pf->vsi) {
15511 err = -ENOMEM;
15512 goto err_switch_setup;
15513 }
15514
15515 /* We allocate one VSI which is needed as absolute minimum
15516 * in order to register the netdev
15517 */
15518 v_idx = i40e_vsi_mem_alloc(pf, I40E_VSI_MAIN);
15519 if (v_idx < 0) {
15520 err = v_idx;
15521 goto err_switch_setup;
15522 }
15523 pf->lan_vsi = v_idx;
15524 vsi = pf->vsi[v_idx];
15525 if (!vsi) {
15526 err = -EFAULT;
15527 goto err_switch_setup;
15528 }
15529 vsi->alloc_queue_pairs = 1;
15530 err = i40e_config_netdev(vsi);
15531 if (err)
15532 goto err_switch_setup;
15533 err = register_netdev(vsi->netdev);
15534 if (err)
15535 goto err_switch_setup;
15536 vsi->netdev_registered = true;
15537 i40e_dbg_pf_init(pf);
15538
15539 err = i40e_setup_misc_vector_for_recovery_mode(pf);
15540 if (err)
15541 goto err_switch_setup;
15542
15543 /* tell the firmware that we're starting */
15544 i40e_send_version(pf);
15545
15546 /* since everything's happy, start the service_task timer */
15547 mod_timer(&pf->service_timer,
15548 round_jiffies(jiffies + pf->service_timer_period));
15549
15550 return 0;
15551
15552 err_switch_setup:
15553 i40e_reset_interrupt_capability(pf);
15554 timer_shutdown_sync(&pf->service_timer);
15555 i40e_shutdown_adminq(hw);
15556 iounmap(hw->hw_addr);
15557 pci_release_mem_regions(pf->pdev);
15558 pci_disable_device(pf->pdev);
15559 i40e_free_pf(pf);
15560
15561 return err;
15562 }
15563
15564 /**
15565 * i40e_set_subsystem_device_id - set subsystem device id
15566 * @hw: pointer to the hardware info
15567 *
15568 * Set PCI subsystem device id either from a pci_dev structure or
15569 * a specific FW register.
15570 **/
i40e_set_subsystem_device_id(struct i40e_hw * hw)15571 static inline void i40e_set_subsystem_device_id(struct i40e_hw *hw)
15572 {
15573 struct i40e_pf *pf = i40e_hw_to_pf(hw);
15574
15575 hw->subsystem_device_id = pf->pdev->subsystem_device ?
15576 pf->pdev->subsystem_device :
15577 (ushort)(rd32(hw, I40E_PFPCI_SUBSYSID) & USHRT_MAX);
15578 }
15579
15580 /**
15581 * i40e_probe - Device initialization routine
15582 * @pdev: PCI device information struct
15583 * @ent: entry in i40e_pci_tbl
15584 *
15585 * i40e_probe initializes a PF identified by a pci_dev structure.
15586 * The OS initialization, configuring of the PF private structure,
15587 * and a hardware reset occur.
15588 *
15589 * Returns 0 on success, negative on failure
15590 **/
i40e_probe(struct pci_dev * pdev,const struct pci_device_id * ent)15591 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
15592 {
15593 struct i40e_aq_get_phy_abilities_resp abilities;
15594 #ifdef CONFIG_I40E_DCB
15595 enum i40e_get_fw_lldp_status_resp lldp_status;
15596 #endif /* CONFIG_I40E_DCB */
15597 struct i40e_vsi *vsi;
15598 struct i40e_pf *pf;
15599 struct i40e_hw *hw;
15600 u16 wol_nvm_bits;
15601 char nvm_ver[32];
15602 u16 link_status;
15603 #ifdef CONFIG_I40E_DCB
15604 int status;
15605 #endif /* CONFIG_I40E_DCB */
15606 int err;
15607 u32 val;
15608
15609 err = pci_enable_device_mem(pdev);
15610 if (err)
15611 return err;
15612
15613 /* set up for high or low dma */
15614 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
15615 if (err) {
15616 dev_err(&pdev->dev,
15617 "DMA configuration failed: 0x%x\n", err);
15618 goto err_dma;
15619 }
15620
15621 /* set up pci connections */
15622 err = pci_request_mem_regions(pdev, i40e_driver_name);
15623 if (err) {
15624 dev_info(&pdev->dev,
15625 "pci_request_selected_regions failed %d\n", err);
15626 goto err_pci_reg;
15627 }
15628
15629 pci_set_master(pdev);
15630
15631 /* Now that we have a PCI connection, we need to do the
15632 * low level device setup. This is primarily setting up
15633 * the Admin Queue structures and then querying for the
15634 * device's current profile information.
15635 */
15636 pf = i40e_alloc_pf(&pdev->dev);
15637 if (!pf) {
15638 err = -ENOMEM;
15639 goto err_pf_alloc;
15640 }
15641 pf->next_vsi = 0;
15642 pf->pdev = pdev;
15643 set_bit(__I40E_DOWN, pf->state);
15644
15645 hw = &pf->hw;
15646
15647 pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0),
15648 I40E_MAX_CSR_SPACE);
15649 /* We believe that the highest register to read is
15650 * I40E_GLGEN_STAT_CLEAR, so we check if the BAR size
15651 * is not less than that before mapping to prevent a
15652 * kernel panic.
15653 */
15654 if (pf->ioremap_len < I40E_GLGEN_STAT_CLEAR) {
15655 dev_err(&pdev->dev, "Cannot map registers, bar size 0x%X too small, aborting\n",
15656 pf->ioremap_len);
15657 err = -ENOMEM;
15658 goto err_ioremap;
15659 }
15660 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len);
15661 if (!hw->hw_addr) {
15662 err = -EIO;
15663 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n",
15664 (unsigned int)pci_resource_start(pdev, 0),
15665 pf->ioremap_len, err);
15666 goto err_ioremap;
15667 }
15668 hw->vendor_id = pdev->vendor;
15669 hw->device_id = pdev->device;
15670 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
15671 hw->subsystem_vendor_id = pdev->subsystem_vendor;
15672 i40e_set_subsystem_device_id(hw);
15673 hw->bus.device = PCI_SLOT(pdev->devfn);
15674 hw->bus.func = PCI_FUNC(pdev->devfn);
15675 hw->bus.bus_id = pdev->bus->number;
15676
15677 /* Select something other than the 802.1ad ethertype for the
15678 * switch to use internally and drop on ingress.
15679 */
15680 hw->switch_tag = 0xffff;
15681 hw->first_tag = ETH_P_8021AD;
15682 hw->second_tag = ETH_P_8021Q;
15683
15684 INIT_LIST_HEAD(&pf->l3_flex_pit_list);
15685 INIT_LIST_HEAD(&pf->l4_flex_pit_list);
15686 INIT_LIST_HEAD(&pf->ddp_old_prof);
15687
15688 /* set up the locks for the AQ, do this only once in probe
15689 * and destroy them only once in remove
15690 */
15691 mutex_init(&hw->aq.asq_mutex);
15692 mutex_init(&hw->aq.arq_mutex);
15693
15694 pf->msg_enable = netif_msg_init(debug,
15695 NETIF_MSG_DRV |
15696 NETIF_MSG_PROBE |
15697 NETIF_MSG_LINK);
15698 if (debug < -1)
15699 pf->hw.debug_mask = debug;
15700
15701 /* do a special CORER for clearing PXE mode once at init */
15702 if (hw->revision_id == 0 &&
15703 (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) {
15704 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK);
15705 i40e_flush(hw);
15706 msleep(200);
15707 pf->corer_count++;
15708
15709 i40e_clear_pxe_mode(hw);
15710 }
15711
15712 /* Reset here to make sure all is clean and to define PF 'n' */
15713 i40e_clear_hw(hw);
15714
15715 err = i40e_set_mac_type(hw);
15716 if (err) {
15717 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n",
15718 err);
15719 goto err_pf_reset;
15720 }
15721
15722 err = i40e_handle_resets(pf);
15723 if (err)
15724 goto err_pf_reset;
15725
15726 i40e_check_recovery_mode(pf);
15727
15728 if (is_kdump_kernel()) {
15729 hw->aq.num_arq_entries = I40E_MIN_ARQ_LEN;
15730 hw->aq.num_asq_entries = I40E_MIN_ASQ_LEN;
15731 } else {
15732 hw->aq.num_arq_entries = I40E_AQ_LEN;
15733 hw->aq.num_asq_entries = I40E_AQ_LEN;
15734 }
15735 hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE;
15736 hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE;
15737
15738 snprintf(pf->int_name, sizeof(pf->int_name) - 1,
15739 "%s-%s:misc",
15740 dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev));
15741
15742 err = i40e_init_shared_code(hw);
15743 if (err) {
15744 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n",
15745 err);
15746 goto err_pf_reset;
15747 }
15748
15749 /* set up a default setting for link flow control */
15750 pf->hw.fc.requested_mode = I40E_FC_NONE;
15751
15752 err = i40e_init_adminq(hw);
15753 if (err) {
15754 if (err == -EIO)
15755 dev_info(&pdev->dev,
15756 "The driver for the device stopped because the NVM image v%u.%u is newer than expected v%u.%u. You must install the most recent version of the network driver.\n",
15757 hw->aq.api_maj_ver,
15758 hw->aq.api_min_ver,
15759 I40E_FW_API_VERSION_MAJOR,
15760 I40E_FW_MINOR_VERSION(hw));
15761 else
15762 dev_info(&pdev->dev,
15763 "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n");
15764
15765 goto err_pf_reset;
15766 }
15767 i40e_get_oem_version(hw);
15768 i40e_get_pba_string(hw);
15769
15770 /* provide nvm, fw, api versions, vendor:device id, subsys vendor:device id */
15771 i40e_nvm_version_str(hw, nvm_ver, sizeof(nvm_ver));
15772 dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s [%04x:%04x] [%04x:%04x]\n",
15773 hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build,
15774 hw->aq.api_maj_ver, hw->aq.api_min_ver, nvm_ver,
15775 hw->vendor_id, hw->device_id, hw->subsystem_vendor_id,
15776 hw->subsystem_device_id);
15777
15778 if (i40e_is_aq_api_ver_ge(hw, I40E_FW_API_VERSION_MAJOR,
15779 I40E_FW_MINOR_VERSION(hw) + 1))
15780 dev_dbg(&pdev->dev,
15781 "The driver for the device detected a newer version of the NVM image v%u.%u than v%u.%u.\n",
15782 hw->aq.api_maj_ver,
15783 hw->aq.api_min_ver,
15784 I40E_FW_API_VERSION_MAJOR,
15785 I40E_FW_MINOR_VERSION(hw));
15786 else if (i40e_is_aq_api_ver_lt(hw, 1, 4))
15787 dev_info(&pdev->dev,
15788 "The driver for the device detected an older version of the NVM image v%u.%u than expected v%u.%u. Please update the NVM image.\n",
15789 hw->aq.api_maj_ver,
15790 hw->aq.api_min_ver,
15791 I40E_FW_API_VERSION_MAJOR,
15792 I40E_FW_MINOR_VERSION(hw));
15793
15794 i40e_verify_eeprom(pf);
15795
15796 /* Rev 0 hardware was never productized */
15797 if (hw->revision_id < 1)
15798 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n");
15799
15800 i40e_clear_pxe_mode(hw);
15801
15802 err = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities);
15803 if (err)
15804 goto err_adminq_setup;
15805
15806 err = i40e_sw_init(pf);
15807 if (err) {
15808 dev_info(&pdev->dev, "sw_init failed: %d\n", err);
15809 goto err_sw_init;
15810 }
15811
15812 if (test_bit(__I40E_RECOVERY_MODE, pf->state))
15813 return i40e_init_recovery_mode(pf, hw);
15814
15815 err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
15816 hw->func_caps.num_rx_qp, 0, 0);
15817 if (err) {
15818 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err);
15819 goto err_init_lan_hmc;
15820 }
15821
15822 err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
15823 if (err) {
15824 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err);
15825 err = -ENOENT;
15826 goto err_configure_lan_hmc;
15827 }
15828
15829 /* Disable LLDP for NICs that have firmware versions lower than v4.3.
15830 * Ignore error return codes because if it was already disabled via
15831 * hardware settings this will fail
15832 */
15833 if (test_bit(I40E_HW_CAP_STOP_FW_LLDP, pf->hw.caps)) {
15834 dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n");
15835 i40e_aq_stop_lldp(hw, true, false, NULL);
15836 }
15837
15838 /* allow a platform config to override the HW addr */
15839 i40e_get_platform_mac_addr(pdev, pf);
15840
15841 if (!is_valid_ether_addr(hw->mac.addr)) {
15842 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr);
15843 err = -EIO;
15844 goto err_mac_addr;
15845 }
15846 dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr);
15847 ether_addr_copy(hw->mac.perm_addr, hw->mac.addr);
15848 i40e_get_port_mac_addr(hw, hw->mac.port_addr);
15849 if (is_valid_ether_addr(hw->mac.port_addr))
15850 set_bit(I40E_HW_CAP_PORT_ID_VALID, pf->hw.caps);
15851
15852 i40e_ptp_alloc_pins(pf);
15853 pci_set_drvdata(pdev, pf);
15854 pci_save_state(pdev);
15855
15856 #ifdef CONFIG_I40E_DCB
15857 status = i40e_get_fw_lldp_status(&pf->hw, &lldp_status);
15858 (!status &&
15859 lldp_status == I40E_GET_FW_LLDP_STATUS_ENABLED) ?
15860 (clear_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags)) :
15861 (set_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags));
15862 dev_info(&pdev->dev,
15863 test_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags) ?
15864 "FW LLDP is disabled\n" :
15865 "FW LLDP is enabled\n");
15866
15867 /* Enable FW to write default DCB config on link-up */
15868 i40e_aq_set_dcb_parameters(hw, true, NULL);
15869
15870 err = i40e_init_pf_dcb(pf);
15871 if (err) {
15872 dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err);
15873 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
15874 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
15875 /* Continue without DCB enabled */
15876 }
15877 #endif /* CONFIG_I40E_DCB */
15878
15879 /* set up periodic task facility */
15880 timer_setup(&pf->service_timer, i40e_service_timer, 0);
15881 pf->service_timer_period = HZ;
15882
15883 INIT_WORK(&pf->service_task, i40e_service_task);
15884 clear_bit(__I40E_SERVICE_SCHED, pf->state);
15885
15886 /* NVM bit on means WoL disabled for the port */
15887 i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits);
15888 if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1)
15889 pf->wol_en = false;
15890 else
15891 pf->wol_en = true;
15892 device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en);
15893
15894 /* set up the main switch operations */
15895 i40e_determine_queue_usage(pf);
15896 err = i40e_init_interrupt_scheme(pf);
15897 if (err)
15898 goto err_switch_setup;
15899
15900 /* Reduce Tx and Rx pairs for kdump
15901 * When MSI-X is enabled, it's not allowed to use more TC queue
15902 * pairs than MSI-X vectors (pf->num_lan_msix) exist. Thus
15903 * vsi->num_queue_pairs will be equal to pf->num_lan_msix, i.e., 1.
15904 */
15905 if (is_kdump_kernel())
15906 pf->num_lan_msix = 1;
15907
15908 pf->udp_tunnel_nic.set_port = i40e_udp_tunnel_set_port;
15909 pf->udp_tunnel_nic.unset_port = i40e_udp_tunnel_unset_port;
15910 pf->udp_tunnel_nic.flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP;
15911 pf->udp_tunnel_nic.shared = &pf->udp_tunnel_shared;
15912 pf->udp_tunnel_nic.tables[0].n_entries = I40E_MAX_PF_UDP_OFFLOAD_PORTS;
15913 pf->udp_tunnel_nic.tables[0].tunnel_types = UDP_TUNNEL_TYPE_VXLAN |
15914 UDP_TUNNEL_TYPE_GENEVE;
15915
15916 /* The number of VSIs reported by the FW is the minimum guaranteed
15917 * to us; HW supports far more and we share the remaining pool with
15918 * the other PFs. We allocate space for more than the guarantee with
15919 * the understanding that we might not get them all later.
15920 */
15921 if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)
15922 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;
15923 else
15924 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;
15925 if (pf->num_alloc_vsi > UDP_TUNNEL_NIC_MAX_SHARING_DEVICES) {
15926 dev_warn(&pf->pdev->dev,
15927 "limiting the VSI count due to UDP tunnel limitation %d > %d\n",
15928 pf->num_alloc_vsi, UDP_TUNNEL_NIC_MAX_SHARING_DEVICES);
15929 pf->num_alloc_vsi = UDP_TUNNEL_NIC_MAX_SHARING_DEVICES;
15930 }
15931
15932 /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */
15933 pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *),
15934 GFP_KERNEL);
15935 if (!pf->vsi) {
15936 err = -ENOMEM;
15937 goto err_switch_setup;
15938 }
15939
15940 #ifdef CONFIG_PCI_IOV
15941 /* prep for VF support */
15942 if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags) &&
15943 test_bit(I40E_FLAG_MSIX_ENA, pf->flags) &&
15944 !test_bit(__I40E_BAD_EEPROM, pf->state)) {
15945 if (pci_num_vf(pdev))
15946 set_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);
15947 }
15948 #endif
15949 err = i40e_setup_pf_switch(pf, false, false);
15950 if (err) {
15951 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err);
15952 goto err_vsis;
15953 }
15954
15955 vsi = i40e_pf_get_main_vsi(pf);
15956 INIT_LIST_HEAD(&vsi->ch_list);
15957
15958 /* if FDIR VSI was set up, start it now */
15959 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
15960 if (vsi)
15961 i40e_vsi_open(vsi);
15962
15963 /* The driver only wants link up/down and module qualification
15964 * reports from firmware. Note the negative logic.
15965 */
15966 err = i40e_aq_set_phy_int_mask(&pf->hw,
15967 ~(I40E_AQ_EVENT_LINK_UPDOWN |
15968 I40E_AQ_EVENT_MEDIA_NA |
15969 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
15970 if (err)
15971 dev_info(&pf->pdev->dev, "set phy mask fail, err %pe aq_err %s\n",
15972 ERR_PTR(err),
15973 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
15974
15975 /* Reconfigure hardware for allowing smaller MSS in the case
15976 * of TSO, so that we avoid the MDD being fired and causing
15977 * a reset in the case of small MSS+TSO.
15978 */
15979 val = rd32(hw, I40E_REG_MSS);
15980 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
15981 val &= ~I40E_REG_MSS_MIN_MASK;
15982 val |= I40E_64BYTE_MSS;
15983 wr32(hw, I40E_REG_MSS, val);
15984 }
15985
15986 if (test_bit(I40E_HW_CAP_RESTART_AUTONEG, pf->hw.caps)) {
15987 msleep(75);
15988 err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
15989 if (err)
15990 dev_info(&pf->pdev->dev, "link restart failed, err %pe aq_err %s\n",
15991 ERR_PTR(err),
15992 i40e_aq_str(&pf->hw,
15993 pf->hw.aq.asq_last_status));
15994 }
15995 /* The main driver is (mostly) up and happy. We need to set this state
15996 * before setting up the misc vector or we get a race and the vector
15997 * ends up disabled forever.
15998 */
15999 clear_bit(__I40E_DOWN, pf->state);
16000
16001 /* In case of MSIX we are going to setup the misc vector right here
16002 * to handle admin queue events etc. In case of legacy and MSI
16003 * the misc functionality and queue processing is combined in
16004 * the same vector and that gets setup at open.
16005 */
16006 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
16007 err = i40e_setup_misc_vector(pf);
16008 if (err) {
16009 dev_info(&pdev->dev,
16010 "setup of misc vector failed: %d\n", err);
16011 i40e_cloud_filter_exit(pf);
16012 i40e_fdir_teardown(pf);
16013 goto err_vsis;
16014 }
16015 }
16016
16017 #ifdef CONFIG_PCI_IOV
16018 /* prep for VF support */
16019 if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags) &&
16020 test_bit(I40E_FLAG_MSIX_ENA, pf->flags) &&
16021 !test_bit(__I40E_BAD_EEPROM, pf->state)) {
16022 /* disable link interrupts for VFs */
16023 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM);
16024 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK;
16025 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val);
16026 i40e_flush(hw);
16027
16028 if (pci_num_vf(pdev)) {
16029 dev_info(&pdev->dev,
16030 "Active VFs found, allocating resources.\n");
16031 err = i40e_alloc_vfs(pf, pci_num_vf(pdev));
16032 if (err)
16033 dev_info(&pdev->dev,
16034 "Error %d allocating resources for existing VFs\n",
16035 err);
16036 }
16037 }
16038 #endif /* CONFIG_PCI_IOV */
16039
16040 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {
16041 pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile,
16042 pf->num_iwarp_msix,
16043 I40E_IWARP_IRQ_PILE_ID);
16044 if (pf->iwarp_base_vector < 0) {
16045 dev_info(&pdev->dev,
16046 "failed to get tracking for %d vectors for IWARP err=%d\n",
16047 pf->num_iwarp_msix, pf->iwarp_base_vector);
16048 clear_bit(I40E_FLAG_IWARP_ENA, pf->flags);
16049 }
16050 }
16051
16052 i40e_dbg_pf_init(pf);
16053
16054 /* tell the firmware that we're starting */
16055 i40e_send_version(pf);
16056
16057 /* since everything's happy, start the service_task timer */
16058 mod_timer(&pf->service_timer,
16059 round_jiffies(jiffies + pf->service_timer_period));
16060
16061 /* add this PF to client device list and launch a client service task */
16062 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {
16063 err = i40e_lan_add_device(pf);
16064 if (err)
16065 dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n",
16066 err);
16067 }
16068
16069 #define PCI_SPEED_SIZE 8
16070 #define PCI_WIDTH_SIZE 8
16071 /* Devices on the IOSF bus do not have this information
16072 * and will report PCI Gen 1 x 1 by default so don't bother
16073 * checking them.
16074 */
16075 if (!test_bit(I40E_HW_CAP_NO_PCI_LINK_CHECK, pf->hw.caps)) {
16076 char speed[PCI_SPEED_SIZE] = "Unknown";
16077 char width[PCI_WIDTH_SIZE] = "Unknown";
16078
16079 /* Get the negotiated link width and speed from PCI config
16080 * space
16081 */
16082 pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA,
16083 &link_status);
16084
16085 i40e_set_pci_config_data(hw, link_status);
16086
16087 switch (hw->bus.speed) {
16088 case i40e_bus_speed_8000:
16089 strscpy(speed, "8.0", PCI_SPEED_SIZE); break;
16090 case i40e_bus_speed_5000:
16091 strscpy(speed, "5.0", PCI_SPEED_SIZE); break;
16092 case i40e_bus_speed_2500:
16093 strscpy(speed, "2.5", PCI_SPEED_SIZE); break;
16094 default:
16095 break;
16096 }
16097 switch (hw->bus.width) {
16098 case i40e_bus_width_pcie_x8:
16099 strscpy(width, "8", PCI_WIDTH_SIZE); break;
16100 case i40e_bus_width_pcie_x4:
16101 strscpy(width, "4", PCI_WIDTH_SIZE); break;
16102 case i40e_bus_width_pcie_x2:
16103 strscpy(width, "2", PCI_WIDTH_SIZE); break;
16104 case i40e_bus_width_pcie_x1:
16105 strscpy(width, "1", PCI_WIDTH_SIZE); break;
16106 default:
16107 break;
16108 }
16109
16110 dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n",
16111 speed, width);
16112
16113 if (hw->bus.width < i40e_bus_width_pcie_x8 ||
16114 hw->bus.speed < i40e_bus_speed_8000) {
16115 dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n");
16116 dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n");
16117 }
16118 }
16119
16120 /* get the requested speeds from the fw */
16121 err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL);
16122 if (err)
16123 dev_dbg(&pf->pdev->dev, "get requested speeds ret = %pe last_status = %s\n",
16124 ERR_PTR(err),
16125 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
16126 pf->hw.phy.link_info.requested_speeds = abilities.link_speed;
16127
16128 /* set the FEC config due to the board capabilities */
16129 i40e_set_fec_in_flags(abilities.fec_cfg_curr_mod_ext_info, pf->flags);
16130
16131 /* get the supported phy types from the fw */
16132 err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL);
16133 if (err)
16134 dev_dbg(&pf->pdev->dev, "get supported phy types ret = %pe last_status = %s\n",
16135 ERR_PTR(err),
16136 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
16137
16138 /* make sure the MFS hasn't been set lower than the default */
16139 #define MAX_FRAME_SIZE_DEFAULT 0x2600
16140 val = FIELD_GET(I40E_PRTGL_SAH_MFS_MASK,
16141 rd32(&pf->hw, I40E_PRTGL_SAH));
16142 if (val < MAX_FRAME_SIZE_DEFAULT)
16143 dev_warn(&pdev->dev, "MFS for port %x (%d) has been set below the default (%d)\n",
16144 pf->hw.port, val, MAX_FRAME_SIZE_DEFAULT);
16145
16146 /* Add a filter to drop all Flow control frames from any VSI from being
16147 * transmitted. By doing so we stop a malicious VF from sending out
16148 * PAUSE or PFC frames and potentially controlling traffic for other
16149 * PF/VF VSIs.
16150 * The FW can still send Flow control frames if enabled.
16151 */
16152 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
16153 pf->main_vsi_seid);
16154
16155 if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) ||
16156 (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4))
16157 set_bit(I40E_HW_CAP_PHY_CONTROLS_LEDS, pf->hw.caps);
16158 if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722)
16159 set_bit(I40E_HW_CAP_CRT_RETIMER, pf->hw.caps);
16160 /* print a string summarizing features */
16161 i40e_print_features(pf);
16162
16163 i40e_devlink_register(pf);
16164
16165 return 0;
16166
16167 /* Unwind what we've done if something failed in the setup */
16168 err_vsis:
16169 set_bit(__I40E_DOWN, pf->state);
16170 i40e_clear_interrupt_scheme(pf);
16171 kfree(pf->vsi);
16172 err_switch_setup:
16173 i40e_reset_interrupt_capability(pf);
16174 timer_shutdown_sync(&pf->service_timer);
16175 err_mac_addr:
16176 err_configure_lan_hmc:
16177 (void)i40e_shutdown_lan_hmc(hw);
16178 err_init_lan_hmc:
16179 kfree(pf->qp_pile);
16180 err_sw_init:
16181 err_adminq_setup:
16182 err_pf_reset:
16183 iounmap(hw->hw_addr);
16184 err_ioremap:
16185 i40e_free_pf(pf);
16186 err_pf_alloc:
16187 pci_release_mem_regions(pdev);
16188 err_pci_reg:
16189 err_dma:
16190 pci_disable_device(pdev);
16191 return err;
16192 }
16193
16194 /**
16195 * i40e_remove - Device removal routine
16196 * @pdev: PCI device information struct
16197 *
16198 * i40e_remove is called by the PCI subsystem to alert the driver
16199 * that is should release a PCI device. This could be caused by a
16200 * Hot-Plug event, or because the driver is going to be removed from
16201 * memory.
16202 **/
i40e_remove(struct pci_dev * pdev)16203 static void i40e_remove(struct pci_dev *pdev)
16204 {
16205 struct i40e_pf *pf = pci_get_drvdata(pdev);
16206 struct i40e_hw *hw = &pf->hw;
16207 struct i40e_vsi *vsi;
16208 struct i40e_veb *veb;
16209 int ret_code;
16210 int i;
16211
16212 i40e_devlink_unregister(pf);
16213
16214 i40e_dbg_pf_exit(pf);
16215
16216 i40e_ptp_stop(pf);
16217
16218 /* Disable RSS in hw */
16219 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0);
16220 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0);
16221
16222 /* Grab __I40E_RESET_RECOVERY_PENDING and set __I40E_IN_REMOVE
16223 * flags, once they are set, i40e_rebuild should not be called as
16224 * i40e_prep_for_reset always returns early.
16225 */
16226 while (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
16227 usleep_range(1000, 2000);
16228 set_bit(__I40E_IN_REMOVE, pf->state);
16229
16230 if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags)) {
16231 set_bit(__I40E_VF_RESETS_DISABLED, pf->state);
16232 i40e_free_vfs(pf);
16233 clear_bit(I40E_FLAG_SRIOV_ENA, pf->flags);
16234 }
16235 /* no more scheduling of any task */
16236 set_bit(__I40E_SUSPENDED, pf->state);
16237 set_bit(__I40E_DOWN, pf->state);
16238 if (pf->service_timer.function)
16239 timer_shutdown_sync(&pf->service_timer);
16240 if (pf->service_task.func)
16241 cancel_work_sync(&pf->service_task);
16242
16243 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) {
16244 struct i40e_vsi *vsi = pf->vsi[0];
16245
16246 /* We know that we have allocated only one vsi for this PF,
16247 * it was just for registering netdevice, so the interface
16248 * could be visible in the 'ifconfig' output
16249 */
16250 unregister_netdev(vsi->netdev);
16251 free_netdev(vsi->netdev);
16252
16253 goto unmap;
16254 }
16255
16256 /* Client close must be called explicitly here because the timer
16257 * has been stopped.
16258 */
16259 i40e_notify_client_of_netdev_close(pf, false);
16260
16261 i40e_fdir_teardown(pf);
16262
16263 /* If there is a switch structure or any orphans, remove them.
16264 * This will leave only the PF's VSI remaining.
16265 */
16266 i40e_pf_for_each_veb(pf, i, veb)
16267 if (veb->uplink_seid == pf->mac_seid ||
16268 veb->uplink_seid == 0)
16269 i40e_switch_branch_release(veb);
16270
16271 /* Now we can shutdown the PF's VSIs, just before we kill
16272 * adminq and hmc.
16273 */
16274 i40e_pf_for_each_vsi(pf, i, vsi) {
16275 i40e_vsi_close(vsi);
16276 i40e_vsi_release(vsi);
16277 pf->vsi[i] = NULL;
16278 }
16279
16280 i40e_cloud_filter_exit(pf);
16281
16282 /* remove attached clients */
16283 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {
16284 ret_code = i40e_lan_del_device(pf);
16285 if (ret_code)
16286 dev_warn(&pdev->dev, "Failed to delete client device: %d\n",
16287 ret_code);
16288 }
16289
16290 /* shutdown and destroy the HMC */
16291 if (hw->hmc.hmc_obj) {
16292 ret_code = i40e_shutdown_lan_hmc(hw);
16293 if (ret_code)
16294 dev_warn(&pdev->dev,
16295 "Failed to destroy the HMC resources: %d\n",
16296 ret_code);
16297 }
16298
16299 unmap:
16300 /* Free MSI/legacy interrupt 0 when in recovery mode. */
16301 if (test_bit(__I40E_RECOVERY_MODE, pf->state) &&
16302 !test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
16303 free_irq(pf->pdev->irq, pf);
16304
16305 /* shutdown the adminq */
16306 i40e_shutdown_adminq(hw);
16307
16308 /* destroy the locks only once, here */
16309 mutex_destroy(&hw->aq.arq_mutex);
16310 mutex_destroy(&hw->aq.asq_mutex);
16311
16312 /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */
16313 rtnl_lock();
16314 i40e_clear_interrupt_scheme(pf);
16315 i40e_pf_for_each_vsi(pf, i, vsi) {
16316 if (!test_bit(__I40E_RECOVERY_MODE, pf->state))
16317 i40e_vsi_clear_rings(vsi);
16318
16319 i40e_vsi_clear(vsi);
16320 pf->vsi[i] = NULL;
16321 }
16322 rtnl_unlock();
16323
16324 i40e_pf_for_each_veb(pf, i, veb) {
16325 kfree(veb);
16326 pf->veb[i] = NULL;
16327 }
16328
16329 kfree(pf->qp_pile);
16330 kfree(pf->vsi);
16331
16332 iounmap(hw->hw_addr);
16333 i40e_free_pf(pf);
16334 pci_release_mem_regions(pdev);
16335
16336 pci_disable_device(pdev);
16337 }
16338
16339 /**
16340 * i40e_enable_mc_magic_wake - enable multicast magic packet wake up
16341 * using the mac_address_write admin q function
16342 * @pf: pointer to i40e_pf struct
16343 **/
i40e_enable_mc_magic_wake(struct i40e_pf * pf)16344 static void i40e_enable_mc_magic_wake(struct i40e_pf *pf)
16345 {
16346 struct i40e_vsi *main_vsi = i40e_pf_get_main_vsi(pf);
16347 struct i40e_hw *hw = &pf->hw;
16348 u8 mac_addr[6];
16349 u16 flags = 0;
16350 int ret;
16351
16352 /* Get current MAC address in case it's an LAA */
16353 if (main_vsi && main_vsi->netdev) {
16354 ether_addr_copy(mac_addr, main_vsi->netdev->dev_addr);
16355 } else {
16356 dev_err(&pf->pdev->dev,
16357 "Failed to retrieve MAC address; using default\n");
16358 ether_addr_copy(mac_addr, hw->mac.addr);
16359 }
16360
16361 /* The FW expects the mac address write cmd to first be called with
16362 * one of these flags before calling it again with the multicast
16363 * enable flags.
16364 */
16365 flags = I40E_AQC_WRITE_TYPE_LAA_WOL;
16366
16367 if (hw->func_caps.flex10_enable && hw->partition_id != 1)
16368 flags = I40E_AQC_WRITE_TYPE_LAA_ONLY;
16369
16370 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
16371 if (ret) {
16372 dev_err(&pf->pdev->dev,
16373 "Failed to update MAC address registers; cannot enable Multicast Magic packet wake up");
16374 return;
16375 }
16376
16377 flags = I40E_AQC_MC_MAG_EN
16378 | I40E_AQC_WOL_PRESERVE_ON_PFR
16379 | I40E_AQC_WRITE_TYPE_UPDATE_MC_MAG;
16380 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
16381 if (ret)
16382 dev_err(&pf->pdev->dev,
16383 "Failed to enable Multicast Magic Packet wake up\n");
16384 }
16385
16386 /**
16387 * i40e_io_suspend - suspend all IO operations
16388 * @pf: pointer to i40e_pf struct
16389 *
16390 **/
i40e_io_suspend(struct i40e_pf * pf)16391 static int i40e_io_suspend(struct i40e_pf *pf)
16392 {
16393 struct i40e_hw *hw = &pf->hw;
16394
16395 set_bit(__I40E_DOWN, pf->state);
16396
16397 /* Ensure service task will not be running */
16398 del_timer_sync(&pf->service_timer);
16399 cancel_work_sync(&pf->service_task);
16400
16401 /* Client close must be called explicitly here because the timer
16402 * has been stopped.
16403 */
16404 i40e_notify_client_of_netdev_close(pf, false);
16405
16406 if (test_bit(I40E_HW_CAP_WOL_MC_MAGIC_PKT_WAKE, pf->hw.caps) &&
16407 pf->wol_en)
16408 i40e_enable_mc_magic_wake(pf);
16409
16410 /* Since we're going to destroy queues during the
16411 * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this
16412 * whole section
16413 */
16414 rtnl_lock();
16415
16416 i40e_prep_for_reset(pf);
16417
16418 wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
16419 wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
16420
16421 /* Clear the interrupt scheme and release our IRQs so that the system
16422 * can safely hibernate even when there are a large number of CPUs.
16423 * Otherwise hibernation might fail when mapping all the vectors back
16424 * to CPU0.
16425 */
16426 i40e_clear_interrupt_scheme(pf);
16427
16428 rtnl_unlock();
16429
16430 return 0;
16431 }
16432
16433 /**
16434 * i40e_io_resume - resume IO operations
16435 * @pf: pointer to i40e_pf struct
16436 *
16437 **/
i40e_io_resume(struct i40e_pf * pf)16438 static int i40e_io_resume(struct i40e_pf *pf)
16439 {
16440 struct device *dev = &pf->pdev->dev;
16441 int err;
16442
16443 /* We need to hold the RTNL lock prior to restoring interrupt schemes,
16444 * since we're going to be restoring queues
16445 */
16446 rtnl_lock();
16447
16448 /* We cleared the interrupt scheme when we suspended, so we need to
16449 * restore it now to resume device functionality.
16450 */
16451 err = i40e_restore_interrupt_scheme(pf);
16452 if (err) {
16453 dev_err(dev, "Cannot restore interrupt scheme: %d\n",
16454 err);
16455 }
16456
16457 clear_bit(__I40E_DOWN, pf->state);
16458 i40e_reset_and_rebuild(pf, false, true);
16459
16460 rtnl_unlock();
16461
16462 /* Clear suspended state last after everything is recovered */
16463 clear_bit(__I40E_SUSPENDED, pf->state);
16464
16465 /* Restart the service task */
16466 mod_timer(&pf->service_timer,
16467 round_jiffies(jiffies + pf->service_timer_period));
16468
16469 return 0;
16470 }
16471
16472 /**
16473 * i40e_pci_error_detected - warning that something funky happened in PCI land
16474 * @pdev: PCI device information struct
16475 * @error: the type of PCI error
16476 *
16477 * Called to warn that something happened and the error handling steps
16478 * are in progress. Allows the driver to quiesce things, be ready for
16479 * remediation.
16480 **/
i40e_pci_error_detected(struct pci_dev * pdev,pci_channel_state_t error)16481 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev,
16482 pci_channel_state_t error)
16483 {
16484 struct i40e_pf *pf = pci_get_drvdata(pdev);
16485
16486 dev_info(&pdev->dev, "%s: error %d\n", __func__, error);
16487
16488 if (!pf) {
16489 dev_info(&pdev->dev,
16490 "Cannot recover - error happened during device probe\n");
16491 return PCI_ERS_RESULT_DISCONNECT;
16492 }
16493
16494 /* shutdown all operations */
16495 if (!test_bit(__I40E_SUSPENDED, pf->state))
16496 i40e_io_suspend(pf);
16497
16498 /* Request a slot reset */
16499 return PCI_ERS_RESULT_NEED_RESET;
16500 }
16501
16502 /**
16503 * i40e_pci_error_slot_reset - a PCI slot reset just happened
16504 * @pdev: PCI device information struct
16505 *
16506 * Called to find if the driver can work with the device now that
16507 * the pci slot has been reset. If a basic connection seems good
16508 * (registers are readable and have sane content) then return a
16509 * happy little PCI_ERS_RESULT_xxx.
16510 **/
i40e_pci_error_slot_reset(struct pci_dev * pdev)16511 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev)
16512 {
16513 struct i40e_pf *pf = pci_get_drvdata(pdev);
16514 pci_ers_result_t result;
16515 u32 reg;
16516
16517 dev_dbg(&pdev->dev, "%s\n", __func__);
16518 /* enable I/O and memory of the device */
16519 if (pci_enable_device(pdev)) {
16520 dev_info(&pdev->dev,
16521 "Cannot re-enable PCI device after reset.\n");
16522 result = PCI_ERS_RESULT_DISCONNECT;
16523 } else {
16524 pci_set_master(pdev);
16525 pci_restore_state(pdev);
16526 pci_save_state(pdev);
16527 pci_wake_from_d3(pdev, false);
16528
16529 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG);
16530 if (reg == 0)
16531 result = PCI_ERS_RESULT_RECOVERED;
16532 else
16533 result = PCI_ERS_RESULT_DISCONNECT;
16534 }
16535
16536 return result;
16537 }
16538
16539 /**
16540 * i40e_pci_error_reset_prepare - prepare device driver for pci reset
16541 * @pdev: PCI device information struct
16542 */
i40e_pci_error_reset_prepare(struct pci_dev * pdev)16543 static void i40e_pci_error_reset_prepare(struct pci_dev *pdev)
16544 {
16545 struct i40e_pf *pf = pci_get_drvdata(pdev);
16546
16547 i40e_prep_for_reset(pf);
16548 }
16549
16550 /**
16551 * i40e_pci_error_reset_done - pci reset done, device driver reset can begin
16552 * @pdev: PCI device information struct
16553 */
i40e_pci_error_reset_done(struct pci_dev * pdev)16554 static void i40e_pci_error_reset_done(struct pci_dev *pdev)
16555 {
16556 struct i40e_pf *pf = pci_get_drvdata(pdev);
16557
16558 if (test_bit(__I40E_IN_REMOVE, pf->state))
16559 return;
16560
16561 i40e_reset_and_rebuild(pf, false, false);
16562 #ifdef CONFIG_PCI_IOV
16563 i40e_restore_all_vfs_msi_state(pdev);
16564 #endif /* CONFIG_PCI_IOV */
16565 }
16566
16567 /**
16568 * i40e_pci_error_resume - restart operations after PCI error recovery
16569 * @pdev: PCI device information struct
16570 *
16571 * Called to allow the driver to bring things back up after PCI error
16572 * and/or reset recovery has finished.
16573 **/
i40e_pci_error_resume(struct pci_dev * pdev)16574 static void i40e_pci_error_resume(struct pci_dev *pdev)
16575 {
16576 struct i40e_pf *pf = pci_get_drvdata(pdev);
16577
16578 dev_dbg(&pdev->dev, "%s\n", __func__);
16579 if (test_bit(__I40E_SUSPENDED, pf->state))
16580 return;
16581
16582 i40e_io_resume(pf);
16583 }
16584
16585 /**
16586 * i40e_shutdown - PCI callback for shutting down
16587 * @pdev: PCI device information struct
16588 **/
i40e_shutdown(struct pci_dev * pdev)16589 static void i40e_shutdown(struct pci_dev *pdev)
16590 {
16591 struct i40e_pf *pf = pci_get_drvdata(pdev);
16592 struct i40e_hw *hw = &pf->hw;
16593
16594 set_bit(__I40E_SUSPENDED, pf->state);
16595 set_bit(__I40E_DOWN, pf->state);
16596
16597 del_timer_sync(&pf->service_timer);
16598 cancel_work_sync(&pf->service_task);
16599 i40e_cloud_filter_exit(pf);
16600 i40e_fdir_teardown(pf);
16601
16602 /* Client close must be called explicitly here because the timer
16603 * has been stopped.
16604 */
16605 i40e_notify_client_of_netdev_close(pf, false);
16606
16607 if (test_bit(I40E_HW_CAP_WOL_MC_MAGIC_PKT_WAKE, pf->hw.caps) &&
16608 pf->wol_en)
16609 i40e_enable_mc_magic_wake(pf);
16610
16611 i40e_prep_for_reset(pf);
16612
16613 wr32(hw, I40E_PFPM_APM,
16614 (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
16615 wr32(hw, I40E_PFPM_WUFC,
16616 (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
16617
16618 /* Free MSI/legacy interrupt 0 when in recovery mode. */
16619 if (test_bit(__I40E_RECOVERY_MODE, pf->state) &&
16620 !test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
16621 free_irq(pf->pdev->irq, pf);
16622
16623 /* Since we're going to destroy queues during the
16624 * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this
16625 * whole section
16626 */
16627 rtnl_lock();
16628 i40e_clear_interrupt_scheme(pf);
16629 rtnl_unlock();
16630
16631 if (system_state == SYSTEM_POWER_OFF) {
16632 pci_wake_from_d3(pdev, pf->wol_en);
16633 pci_set_power_state(pdev, PCI_D3hot);
16634 }
16635 }
16636
16637 /**
16638 * i40e_suspend - PM callback for moving to D3
16639 * @dev: generic device information structure
16640 **/
i40e_suspend(struct device * dev)16641 static int i40e_suspend(struct device *dev)
16642 {
16643 struct i40e_pf *pf = dev_get_drvdata(dev);
16644
16645 /* If we're already suspended, then there is nothing to do */
16646 if (test_and_set_bit(__I40E_SUSPENDED, pf->state))
16647 return 0;
16648 return i40e_io_suspend(pf);
16649 }
16650
16651 /**
16652 * i40e_resume - PM callback for waking up from D3
16653 * @dev: generic device information structure
16654 **/
i40e_resume(struct device * dev)16655 static int i40e_resume(struct device *dev)
16656 {
16657 struct i40e_pf *pf = dev_get_drvdata(dev);
16658
16659 /* If we're not suspended, then there is nothing to do */
16660 if (!test_bit(__I40E_SUSPENDED, pf->state))
16661 return 0;
16662 return i40e_io_resume(pf);
16663 }
16664
16665 static const struct pci_error_handlers i40e_err_handler = {
16666 .error_detected = i40e_pci_error_detected,
16667 .slot_reset = i40e_pci_error_slot_reset,
16668 .reset_prepare = i40e_pci_error_reset_prepare,
16669 .reset_done = i40e_pci_error_reset_done,
16670 .resume = i40e_pci_error_resume,
16671 };
16672
16673 static DEFINE_SIMPLE_DEV_PM_OPS(i40e_pm_ops, i40e_suspend, i40e_resume);
16674
16675 static struct pci_driver i40e_driver = {
16676 .name = i40e_driver_name,
16677 .id_table = i40e_pci_tbl,
16678 .probe = i40e_probe,
16679 .remove = i40e_remove,
16680 .driver.pm = pm_sleep_ptr(&i40e_pm_ops),
16681 .shutdown = i40e_shutdown,
16682 .err_handler = &i40e_err_handler,
16683 .sriov_configure = i40e_pci_sriov_configure,
16684 };
16685
16686 /**
16687 * i40e_init_module - Driver registration routine
16688 *
16689 * i40e_init_module is the first routine called when the driver is
16690 * loaded. All it does is register with the PCI subsystem.
16691 **/
i40e_init_module(void)16692 static int __init i40e_init_module(void)
16693 {
16694 int err;
16695
16696 pr_info("%s: %s\n", i40e_driver_name, i40e_driver_string);
16697 pr_info("%s: %s\n", i40e_driver_name, i40e_copyright);
16698
16699 /* There is no need to throttle the number of active tasks because
16700 * each device limits its own task using a state bit for scheduling
16701 * the service task, and the device tasks do not interfere with each
16702 * other, so we don't set a max task limit. We must set WQ_MEM_RECLAIM
16703 * since we need to be able to guarantee forward progress even under
16704 * memory pressure.
16705 */
16706 i40e_wq = alloc_workqueue("%s", 0, 0, i40e_driver_name);
16707 if (!i40e_wq) {
16708 pr_err("%s: Failed to create workqueue\n", i40e_driver_name);
16709 return -ENOMEM;
16710 }
16711
16712 i40e_dbg_init();
16713 err = pci_register_driver(&i40e_driver);
16714 if (err) {
16715 destroy_workqueue(i40e_wq);
16716 i40e_dbg_exit();
16717 return err;
16718 }
16719
16720 return 0;
16721 }
16722 module_init(i40e_init_module);
16723
16724 /**
16725 * i40e_exit_module - Driver exit cleanup routine
16726 *
16727 * i40e_exit_module is called just before the driver is removed
16728 * from memory.
16729 **/
i40e_exit_module(void)16730 static void __exit i40e_exit_module(void)
16731 {
16732 pci_unregister_driver(&i40e_driver);
16733 destroy_workqueue(i40e_wq);
16734 ida_destroy(&i40e_client_ida);
16735 i40e_dbg_exit();
16736 }
16737 module_exit(i40e_exit_module);
16738