xref: /freebsd/sys/dev/ice/if_ice_iflib.c (revision e17f5b1d)
1 /* SPDX-License-Identifier: BSD-3-Clause */
2 /*  Copyright (c) 2020, Intel Corporation
3  *  All rights reserved.
4  *
5  *  Redistribution and use in source and binary forms, with or without
6  *  modification, are permitted provided that the following conditions are met:
7  *
8  *   1. Redistributions of source code must retain the above copyright notice,
9  *      this list of conditions and the following disclaimer.
10  *
11  *   2. Redistributions in binary form must reproduce the above copyright
12  *      notice, this list of conditions and the following disclaimer in the
13  *      documentation and/or other materials provided with the distribution.
14  *
15  *   3. Neither the name of the Intel Corporation nor the names of its
16  *      contributors may be used to endorse or promote products derived from
17  *      this software without specific prior written permission.
18  *
19  *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20  *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21  *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22  *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
23  *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24  *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25  *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26  *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27  *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28  *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29  *  POSSIBILITY OF SUCH DAMAGE.
30  */
31 /*$FreeBSD$*/
32 
33 /**
34  * @file if_ice_iflib.c
35  * @brief iflib driver implementation
36  *
37  * Contains the main entry point for the iflib driver implementation. It
38  * implements the various ifdi driver methods, and sets up the module and
39  * driver values to load an iflib driver.
40  */
41 
42 #include "ice_iflib.h"
43 #include "ice_drv_info.h"
44 #include "ice_switch.h"
45 #include "ice_sched.h"
46 
47 #include <sys/module.h>
48 #include <sys/sockio.h>
49 #include <sys/smp.h>
50 #include <dev/pci/pcivar.h>
51 #include <dev/pci/pcireg.h>
52 
53 /*
54  * Device method prototypes
55  */
56 
57 static void *ice_register(device_t);
58 static int  ice_if_attach_pre(if_ctx_t);
59 static int  ice_attach_pre_recovery_mode(struct ice_softc *sc);
60 static int  ice_if_attach_post(if_ctx_t);
61 static void ice_attach_post_recovery_mode(struct ice_softc *sc);
62 static int  ice_if_detach(if_ctx_t);
63 static int  ice_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int ntxqs, int ntxqsets);
64 static int  ice_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nqs, int nqsets);
65 static int ice_if_msix_intr_assign(if_ctx_t ctx, int msix);
66 static void ice_if_queues_free(if_ctx_t ctx);
67 static int ice_if_mtu_set(if_ctx_t ctx, uint32_t mtu);
68 static void ice_if_intr_enable(if_ctx_t ctx);
69 static void ice_if_intr_disable(if_ctx_t ctx);
70 static int ice_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid);
71 static int ice_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid);
72 static int ice_if_promisc_set(if_ctx_t ctx, int flags);
73 static void ice_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr);
74 static int ice_if_media_change(if_ctx_t ctx);
75 static void ice_if_init(if_ctx_t ctx);
76 static void ice_if_timer(if_ctx_t ctx, uint16_t qid);
77 static void ice_if_update_admin_status(if_ctx_t ctx);
78 static void ice_if_multi_set(if_ctx_t ctx);
79 static void ice_if_vlan_register(if_ctx_t ctx, u16 vtag);
80 static void ice_if_vlan_unregister(if_ctx_t ctx, u16 vtag);
81 static void ice_if_stop(if_ctx_t ctx);
82 static uint64_t ice_if_get_counter(if_ctx_t ctx, ift_counter counter);
83 static int ice_if_priv_ioctl(if_ctx_t ctx, u_long command, caddr_t data);
84 static int ice_if_i2c_req(if_ctx_t ctx, struct ifi2creq *req);
85 
86 static int ice_msix_que(void *arg);
87 static int ice_msix_admin(void *arg);
88 
89 /*
90  * Helper function prototypes
91  */
92 static int ice_pci_mapping(struct ice_softc *sc);
93 static void ice_free_pci_mapping(struct ice_softc *sc);
94 static void ice_update_link_status(struct ice_softc *sc, bool update_media);
95 static void ice_init_device_features(struct ice_softc *sc);
96 static void ice_init_tx_tracking(struct ice_vsi *vsi);
97 static void ice_handle_reset_event(struct ice_softc *sc);
98 static void ice_handle_pf_reset_request(struct ice_softc *sc);
99 static void ice_prepare_for_reset(struct ice_softc *sc);
100 static int ice_rebuild_pf_vsi_qmap(struct ice_softc *sc);
101 static void ice_rebuild(struct ice_softc *sc);
102 static void ice_rebuild_recovery_mode(struct ice_softc *sc);
103 static void ice_free_irqvs(struct ice_softc *sc);
104 static void ice_update_rx_mbuf_sz(struct ice_softc *sc);
105 static void ice_poll_for_media_avail(struct ice_softc *sc);
106 static void ice_setup_scctx(struct ice_softc *sc);
107 static int ice_allocate_msix(struct ice_softc *sc);
108 static void ice_admin_timer(void *arg);
109 static void ice_transition_recovery_mode(struct ice_softc *sc);
110 static void ice_transition_safe_mode(struct ice_softc *sc);
111 
112 /*
113  * Device Interface Declaration
114  */
115 
116 /**
117  * @var ice_methods
118  * @brief ice driver method entry points
119  *
120  * List of device methods implementing the generic device interface used by
121  * the device stack to interact with the ice driver. Since this is an iflib
122  * driver, most of the methods point to the generic iflib implementation.
123  */
124 static device_method_t ice_methods[] = {
125 	/* Device interface */
126 	DEVMETHOD(device_register, ice_register),
127 	DEVMETHOD(device_probe,    iflib_device_probe_vendor),
128 	DEVMETHOD(device_attach,   iflib_device_attach),
129 	DEVMETHOD(device_detach,   iflib_device_detach),
130 	DEVMETHOD(device_shutdown, iflib_device_shutdown),
131 	DEVMETHOD(device_suspend,  iflib_device_suspend),
132 	DEVMETHOD(device_resume,   iflib_device_resume),
133 	DEVMETHOD_END
134 };
135 
136 /**
137  * @var ice_iflib_methods
138  * @brief iflib method entry points
139  *
140  * List of device methods used by the iflib stack to interact with this
141  * driver. These are the real main entry points used to interact with this
142  * driver.
143  */
144 static device_method_t ice_iflib_methods[] = {
145 	DEVMETHOD(ifdi_attach_pre, ice_if_attach_pre),
146 	DEVMETHOD(ifdi_attach_post, ice_if_attach_post),
147 	DEVMETHOD(ifdi_detach, ice_if_detach),
148 	DEVMETHOD(ifdi_tx_queues_alloc, ice_if_tx_queues_alloc),
149 	DEVMETHOD(ifdi_rx_queues_alloc, ice_if_rx_queues_alloc),
150 	DEVMETHOD(ifdi_msix_intr_assign, ice_if_msix_intr_assign),
151 	DEVMETHOD(ifdi_queues_free, ice_if_queues_free),
152 	DEVMETHOD(ifdi_mtu_set, ice_if_mtu_set),
153 	DEVMETHOD(ifdi_intr_enable, ice_if_intr_enable),
154 	DEVMETHOD(ifdi_intr_disable, ice_if_intr_disable),
155 	DEVMETHOD(ifdi_rx_queue_intr_enable, ice_if_rx_queue_intr_enable),
156 	DEVMETHOD(ifdi_tx_queue_intr_enable, ice_if_tx_queue_intr_enable),
157 	DEVMETHOD(ifdi_promisc_set, ice_if_promisc_set),
158 	DEVMETHOD(ifdi_media_status, ice_if_media_status),
159 	DEVMETHOD(ifdi_media_change, ice_if_media_change),
160 	DEVMETHOD(ifdi_init, ice_if_init),
161 	DEVMETHOD(ifdi_stop, ice_if_stop),
162 	DEVMETHOD(ifdi_timer, ice_if_timer),
163 	DEVMETHOD(ifdi_update_admin_status, ice_if_update_admin_status),
164 	DEVMETHOD(ifdi_multi_set, ice_if_multi_set),
165 	DEVMETHOD(ifdi_vlan_register, ice_if_vlan_register),
166 	DEVMETHOD(ifdi_vlan_unregister, ice_if_vlan_unregister),
167 	DEVMETHOD(ifdi_get_counter, ice_if_get_counter),
168 	DEVMETHOD(ifdi_priv_ioctl, ice_if_priv_ioctl),
169 	DEVMETHOD(ifdi_i2c_req, ice_if_i2c_req),
170 	DEVMETHOD_END
171 };
172 
173 /**
174  * @var ice_driver
175  * @brief driver structure for the generic device stack
176  *
177  * driver_t definition used to setup the generic device methods.
178  */
179 static driver_t ice_driver = {
180 	.name = "ice",
181 	.methods = ice_methods,
182 	.size = sizeof(struct ice_softc),
183 };
184 
185 /**
186  * @var ice_iflib_driver
187  * @brief driver structure for the iflib stack
188  *
189  * driver_t definition used to setup the iflib device methods.
190  */
191 static driver_t ice_iflib_driver = {
192 	.name = "ice",
193 	.methods = ice_iflib_methods,
194 	.size = sizeof(struct ice_softc),
195 };
196 
197 extern struct if_txrx ice_txrx;
198 extern struct if_txrx ice_recovery_txrx;
199 
200 /**
201  * @var ice_sctx
202  * @brief ice driver shared context
203  *
204  * Structure defining shared values (context) that is used by all instances of
205  * the device. Primarily used to setup details about how the iflib stack
206  * should treat this driver. Also defines the default, minimum, and maximum
207  * number of descriptors in each ring.
208  */
209 static struct if_shared_ctx ice_sctx = {
210 	.isc_magic = IFLIB_MAGIC,
211 	.isc_q_align = PAGE_SIZE,
212 
213 	.isc_tx_maxsize = ICE_MAX_FRAME_SIZE,
214 	/* We could technically set this as high as ICE_MAX_DMA_SEG_SIZE, but
215 	 * that doesn't make sense since that would be larger than the maximum
216 	 * size of a single packet.
217 	 */
218 	.isc_tx_maxsegsize = ICE_MAX_FRAME_SIZE,
219 
220 	/* XXX: This is only used by iflib to ensure that
221 	 * scctx->isc_tx_tso_size_max + the VLAN header is a valid size.
222 	 */
223 	.isc_tso_maxsize = ICE_TSO_SIZE + sizeof(struct ether_vlan_header),
224 	/* XXX: This is used by iflib to set the number of segments in the TSO
225 	 * DMA tag. However, scctx->isc_tx_tso_segsize_max is used to set the
226 	 * related ifnet parameter.
227 	 */
228 	.isc_tso_maxsegsize = ICE_MAX_DMA_SEG_SIZE,
229 
230 	.isc_rx_maxsize = ICE_MAX_FRAME_SIZE,
231 	.isc_rx_nsegments = ICE_MAX_RX_SEGS,
232 	.isc_rx_maxsegsize = ICE_MAX_FRAME_SIZE,
233 
234 	.isc_nfl = 1,
235 	.isc_ntxqs = 1,
236 	.isc_nrxqs = 1,
237 
238 	.isc_admin_intrcnt = 1,
239 	.isc_vendor_info = ice_vendor_info_array,
240 	.isc_driver_version = __DECONST(char *, ice_driver_version),
241 	.isc_driver = &ice_iflib_driver,
242 
243 	/*
244 	 * IFLIB_NEED_SCRATCH ensures that mbufs have scratch space available
245 	 * for hardware checksum offload
246 	 *
247 	 * IFLIB_TSO_INIT_IP ensures that the TSO packets have zeroed out the
248 	 * IP sum field, required by our hardware to calculate valid TSO
249 	 * checksums.
250 	 *
251 	 * IFLIB_ADMIN_ALWAYS_RUN ensures that the administrative task runs
252 	 * even when the interface is down.
253 	 *
254 	 * IFLIB_SKIP_MSIX allows the driver to handle allocating MSI-X
255 	 * vectors manually instead of relying on iflib code to do this.
256 	 */
257 	.isc_flags = IFLIB_NEED_SCRATCH | IFLIB_TSO_INIT_IP |
258 		IFLIB_ADMIN_ALWAYS_RUN | IFLIB_SKIP_MSIX,
259 
260 	.isc_nrxd_min = {ICE_MIN_DESC_COUNT},
261 	.isc_ntxd_min = {ICE_MIN_DESC_COUNT},
262 	.isc_nrxd_max = {ICE_IFLIB_MAX_DESC_COUNT},
263 	.isc_ntxd_max = {ICE_IFLIB_MAX_DESC_COUNT},
264 	.isc_nrxd_default = {ICE_DEFAULT_DESC_COUNT},
265 	.isc_ntxd_default = {ICE_DEFAULT_DESC_COUNT},
266 };
267 
268 /**
269  * @var ice_devclass
270  * @brief ice driver device class
271  *
272  * device class used to setup the ice driver module kobject class.
273  */
274 devclass_t ice_devclass;
275 DRIVER_MODULE(ice, pci, ice_driver, ice_devclass, ice_module_event_handler, 0);
276 
277 MODULE_VERSION(ice, 1);
278 MODULE_DEPEND(ice, pci, 1, 1, 1);
279 MODULE_DEPEND(ice, ether, 1, 1, 1);
280 MODULE_DEPEND(ice, iflib, 1, 1, 1);
281 MODULE_DEPEND(ice, firmware, 1, 1, 1);
282 
283 IFLIB_PNP_INFO(pci, ice, ice_vendor_info_array);
284 
285 /* Static driver-wide sysctls */
286 #include "ice_iflib_sysctls.h"
287 
288 /**
289  * ice_pci_mapping - Map PCI BAR memory
290  * @sc: device private softc
291  *
292  * Map PCI BAR 0 for device operation.
293  */
294 static int
295 ice_pci_mapping(struct ice_softc *sc)
296 {
297 	int rc;
298 
299 	/* Map BAR0 */
300 	rc = ice_map_bar(sc->dev, &sc->bar0, 0);
301 	if (rc)
302 		return rc;
303 
304 	return 0;
305 }
306 
307 /**
308  * ice_free_pci_mapping - Release PCI BAR memory
309  * @sc: device private softc
310  *
311  * Release PCI BARs which were previously mapped by ice_pci_mapping().
312  */
313 static void
314 ice_free_pci_mapping(struct ice_softc *sc)
315 {
316 	/* Free BAR0 */
317 	ice_free_bar(sc->dev, &sc->bar0);
318 }
319 
320 /*
321  * Device methods
322  */
323 
324 /**
325  * ice_register - register device method callback
326  * @dev: the device being registered
327  *
328  * Returns a pointer to the shared context structure, which is used by iflib.
329  */
330 static void *
331 ice_register(device_t dev __unused)
332 {
333 	return &ice_sctx;
334 } /* ice_register */
335 
336 /**
337  * ice_setup_scctx - Setup the iflib softc context structure
338  * @sc: the device private structure
339  *
340  * Setup the parameters in if_softc_ctx_t structure used by the iflib stack
341  * when loading.
342  */
343 static void
344 ice_setup_scctx(struct ice_softc *sc)
345 {
346 	if_softc_ctx_t scctx = sc->scctx;
347 	struct ice_hw *hw = &sc->hw;
348 	bool safe_mode, recovery_mode;
349 
350 	safe_mode = ice_is_bit_set(sc->feat_en, ICE_FEATURE_SAFE_MODE);
351 	recovery_mode = ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE);
352 
353 	/*
354 	 * If the driver loads in Safe mode or Recovery mode, limit iflib to
355 	 * a single queue pair.
356 	 */
357 	if (safe_mode || recovery_mode) {
358 		scctx->isc_ntxqsets = scctx->isc_nrxqsets = 1;
359 		scctx->isc_ntxqsets_max = 1;
360 		scctx->isc_nrxqsets_max = 1;
361 	} else {
362 		/*
363 		 * iflib initially sets the isc_ntxqsets and isc_nrxqsets to
364 		 * the values of the override sysctls. Cache these initial
365 		 * values so that the driver can be aware of what the iflib
366 		 * sysctl value is when setting up MSI-X vectors.
367 		 */
368 		sc->ifc_sysctl_ntxqs = scctx->isc_ntxqsets;
369 		sc->ifc_sysctl_nrxqs = scctx->isc_nrxqsets;
370 
371 		if (scctx->isc_ntxqsets == 0)
372 			scctx->isc_ntxqsets = hw->func_caps.common_cap.rss_table_size;
373 		if (scctx->isc_nrxqsets == 0)
374 			scctx->isc_nrxqsets = hw->func_caps.common_cap.rss_table_size;
375 
376 		scctx->isc_ntxqsets_max = hw->func_caps.common_cap.num_txq;
377 		scctx->isc_nrxqsets_max = hw->func_caps.common_cap.num_rxq;
378 
379 		/*
380 		 * Sanity check that the iflib sysctl values are within the
381 		 * maximum supported range.
382 		 */
383 		if (sc->ifc_sysctl_ntxqs > scctx->isc_ntxqsets_max)
384 			sc->ifc_sysctl_ntxqs = scctx->isc_ntxqsets_max;
385 		if (sc->ifc_sysctl_nrxqs > scctx->isc_nrxqsets_max)
386 			sc->ifc_sysctl_nrxqs = scctx->isc_nrxqsets_max;
387 	}
388 
389 	scctx->isc_txqsizes[0] = roundup2(scctx->isc_ntxd[0]
390 	    * sizeof(struct ice_tx_desc), DBA_ALIGN);
391 	scctx->isc_rxqsizes[0] = roundup2(scctx->isc_nrxd[0]
392 	    * sizeof(union ice_32b_rx_flex_desc), DBA_ALIGN);
393 
394 	scctx->isc_tx_nsegments = ICE_MAX_TX_SEGS;
395 	scctx->isc_tx_tso_segments_max = ICE_MAX_TSO_SEGS;
396 	scctx->isc_tx_tso_size_max = ICE_TSO_SIZE;
397 	scctx->isc_tx_tso_segsize_max = ICE_MAX_DMA_SEG_SIZE;
398 
399 	scctx->isc_msix_bar = PCIR_BAR(ICE_MSIX_BAR);
400 	scctx->isc_rss_table_size = hw->func_caps.common_cap.rss_table_size;
401 
402 	/*
403 	 * If the driver loads in recovery mode, disable Tx/Rx functionality
404 	 */
405 	if (recovery_mode)
406 		scctx->isc_txrx = &ice_recovery_txrx;
407 	else
408 		scctx->isc_txrx = &ice_txrx;
409 
410 	/*
411 	 * If the driver loads in Safe mode or Recovery mode, disable
412 	 * advanced features including hardware offloads.
413 	 */
414 	if (safe_mode || recovery_mode) {
415 		scctx->isc_capenable = ICE_SAFE_CAPS;
416 		scctx->isc_tx_csum_flags = 0;
417 	} else {
418 		scctx->isc_capenable = ICE_FULL_CAPS;
419 		scctx->isc_tx_csum_flags = ICE_CSUM_OFFLOAD;
420 	}
421 
422 	scctx->isc_capabilities = scctx->isc_capenable;
423 } /* ice_setup_scctx */
424 
425 /**
426  * ice_if_attach_pre - Early device attach logic
427  * @ctx: the iflib context structure
428  *
429  * Called by iflib during the attach process. Earliest main driver entry
430  * point which performs necessary hardware and driver initialization. Called
431  * before the Tx and Rx queues are allocated.
432  */
433 static int
434 ice_if_attach_pre(if_ctx_t ctx)
435 {
436 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
437 	enum ice_fw_modes fw_mode;
438 	enum ice_status status;
439 	if_softc_ctx_t scctx;
440 	struct ice_hw *hw;
441 	device_t dev;
442 	int err;
443 
444 	device_printf(iflib_get_dev(ctx), "Loading the iflib ice driver\n");
445 
446 	sc->ctx = ctx;
447 	sc->media = iflib_get_media(ctx);
448 	sc->sctx = iflib_get_sctx(ctx);
449 	sc->iflib_ctx_lock = iflib_ctx_lock_get(ctx);
450 
451 	dev = sc->dev = iflib_get_dev(ctx);
452 	scctx = sc->scctx = iflib_get_softc_ctx(ctx);
453 
454 	hw = &sc->hw;
455 	hw->back = sc;
456 
457 	snprintf(sc->admin_mtx_name, sizeof(sc->admin_mtx_name),
458 		 "%s:admin", device_get_nameunit(dev));
459 	mtx_init(&sc->admin_mtx, sc->admin_mtx_name, NULL, MTX_DEF);
460 	callout_init_mtx(&sc->admin_timer, &sc->admin_mtx, 0);
461 
462 	ASSERT_CTX_LOCKED(sc);
463 
464 	if (ice_pci_mapping(sc)) {
465 		err = (ENXIO);
466 		goto destroy_admin_timer;
467 	}
468 
469 	/* Save off the PCI information */
470 	ice_save_pci_info(hw, dev);
471 
472 	/* create tunables as early as possible */
473 	ice_add_device_tunables(sc);
474 
475 	/* Setup ControlQ lengths */
476 	ice_set_ctrlq_len(hw);
477 
478 	fw_mode = ice_get_fw_mode(hw);
479 	if (fw_mode == ICE_FW_MODE_REC) {
480 		device_printf(dev, "Firmware recovery mode detected. Limiting functionality. Refer to Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n");
481 
482 		err = ice_attach_pre_recovery_mode(sc);
483 		if (err)
484 			goto free_pci_mapping;
485 
486 		return (0);
487 	}
488 
489 	/* Initialize the hw data structure */
490 	status = ice_init_hw(hw);
491 	if (status) {
492 		if (status == ICE_ERR_FW_API_VER) {
493 			/* Enter recovery mode, so that the driver remains
494 			 * loaded. This way, if the system administrator
495 			 * cannot update the driver, they may still attempt to
496 			 * downgrade the NVM.
497 			 */
498 			err = ice_attach_pre_recovery_mode(sc);
499 			if (err)
500 				goto free_pci_mapping;
501 
502 			return (0);
503 		} else {
504 			err = EIO;
505 			device_printf(dev, "Unable to initialize hw, err %s aq_err %s\n",
506 				      ice_status_str(status),
507 				      ice_aq_str(hw->adminq.sq_last_status));
508 		}
509 		goto free_pci_mapping;
510 	}
511 
512 	/* Notify firmware of the device driver version */
513 	err = ice_send_version(sc);
514 	if (err)
515 		goto deinit_hw;
516 
517 	ice_load_pkg_file(sc);
518 
519 	err = ice_init_link_events(sc);
520 	if (err) {
521 		device_printf(dev, "ice_init_link_events failed: %s\n",
522 			      ice_err_str(err));
523 		goto deinit_hw;
524 	}
525 
526 	ice_print_nvm_version(sc);
527 
528 	ice_init_device_features(sc);
529 
530 	/* Setup the MAC address */
531 	iflib_set_mac(ctx, hw->port_info->mac.lan_addr);
532 
533 	/* Setup the iflib softc context structure */
534 	ice_setup_scctx(sc);
535 
536 	/* Initialize the Tx queue manager */
537 	err = ice_resmgr_init(&sc->tx_qmgr, hw->func_caps.common_cap.num_txq);
538 	if (err) {
539 		device_printf(dev, "Unable to initialize Tx queue manager: %s\n",
540 			      ice_err_str(err));
541 		goto deinit_hw;
542 	}
543 
544 	/* Initialize the Rx queue manager */
545 	err = ice_resmgr_init(&sc->rx_qmgr, hw->func_caps.common_cap.num_rxq);
546 	if (err) {
547 		device_printf(dev, "Unable to initialize Rx queue manager: %s\n",
548 			      ice_err_str(err));
549 		goto free_tx_qmgr;
550 	}
551 
552 	/* Initialize the interrupt resource manager */
553 	err = ice_alloc_intr_tracking(sc);
554 	if (err)
555 		/* Errors are already printed */
556 		goto free_rx_qmgr;
557 
558 	/* Determine maximum number of VSIs we'll prepare for */
559 	sc->num_available_vsi = min(ICE_MAX_VSI_AVAILABLE,
560 				    hw->func_caps.guar_num_vsi);
561 
562 	if (!sc->num_available_vsi) {
563 		err = EIO;
564 		device_printf(dev, "No VSIs allocated to host\n");
565 		goto free_intr_tracking;
566 	}
567 
568 	/* Allocate storage for the VSI pointers */
569 	sc->all_vsi = (struct ice_vsi **)
570 		malloc(sizeof(struct ice_vsi *) * sc->num_available_vsi,
571 		       M_ICE, M_WAITOK | M_ZERO);
572 	if (!sc->all_vsi) {
573 		err = ENOMEM;
574 		device_printf(dev, "Unable to allocate VSI array\n");
575 		goto free_intr_tracking;
576 	}
577 
578 	/*
579 	 * Prepare the statically allocated primary PF VSI in the softc
580 	 * structure. Other VSIs will be dynamically allocated as needed.
581 	 */
582 	ice_setup_pf_vsi(sc);
583 
584 	err = ice_alloc_vsi_qmap(&sc->pf_vsi, scctx->isc_ntxqsets_max,
585 	    scctx->isc_nrxqsets_max);
586 	if (err) {
587 		device_printf(dev, "Unable to allocate VSI Queue maps\n");
588 		goto free_main_vsi;
589 	}
590 
591 	/* Allocate MSI-X vectors (due to isc_flags IFLIB_SKIP_MSIX) */
592 	err = ice_allocate_msix(sc);
593 	if (err)
594 		goto free_main_vsi;
595 
596 	return 0;
597 
598 free_main_vsi:
599 	/* ice_release_vsi will free the queue maps if they were allocated */
600 	ice_release_vsi(&sc->pf_vsi);
601 	free(sc->all_vsi, M_ICE);
602 	sc->all_vsi = NULL;
603 free_intr_tracking:
604 	ice_free_intr_tracking(sc);
605 free_rx_qmgr:
606 	ice_resmgr_destroy(&sc->rx_qmgr);
607 free_tx_qmgr:
608 	ice_resmgr_destroy(&sc->tx_qmgr);
609 deinit_hw:
610 	ice_deinit_hw(hw);
611 free_pci_mapping:
612 	ice_free_pci_mapping(sc);
613 destroy_admin_timer:
614 	mtx_lock(&sc->admin_mtx);
615 	callout_stop(&sc->admin_timer);
616 	mtx_unlock(&sc->admin_mtx);
617 	mtx_destroy(&sc->admin_mtx);
618 	return err;
619 } /* ice_if_attach_pre */
620 
621 /**
622  * ice_attach_pre_recovery_mode - Limited driver attach_pre for FW recovery
623  * @sc: the device private softc
624  *
625  * Loads the device driver in limited Firmware Recovery mode, intended to
626  * allow users to update the firmware to attempt to recover the device.
627  *
628  * @remark We may enter recovery mode in case either (a) the firmware is
629  * detected to be in an invalid state and must be re-programmed, or (b) the
630  * driver detects that the loaded firmware has a non-compatible API version
631  * that the driver cannot operate with.
632  */
633 static int
634 ice_attach_pre_recovery_mode(struct ice_softc *sc)
635 {
636 	ice_set_state(&sc->state, ICE_STATE_RECOVERY_MODE);
637 
638 	/* Setup the iflib softc context */
639 	ice_setup_scctx(sc);
640 
641 	/* Setup the PF VSI back pointer */
642 	sc->pf_vsi.sc = sc;
643 
644 	/*
645 	 * We still need to allocate MSI-X vectors since we need one vector to
646 	 * run the administrative admin interrupt
647 	 */
648 	return ice_allocate_msix(sc);
649 }
650 
651 /**
652  * ice_update_link_status - notify OS of link state change
653  * @sc: device private softc structure
654  * @update_media: true if we should update media even if link didn't change
655  *
656  * Called to notify iflib core of link status changes. Should be called once
657  * during attach_post, and whenever link status changes during runtime.
658  *
659  * This call only updates the currently supported media types if the link
660  * status changed, or if update_media is set to true.
661  */
662 static void
663 ice_update_link_status(struct ice_softc *sc, bool update_media)
664 {
665 	struct ice_hw *hw = &sc->hw;
666 
667 	/* Never report link up when in recovery mode */
668 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
669 		return;
670 
671 	/* Report link status to iflib only once each time it changes */
672 	if (!ice_testandset_state(&sc->state, ICE_STATE_LINK_STATUS_REPORTED)) {
673 		if (sc->link_up) { /* link is up */
674 			uint64_t baudrate = ice_aq_speed_to_rate(sc->hw.port_info);
675 
676 			iflib_link_state_change(sc->ctx, LINK_STATE_UP, baudrate);
677 
678 			ice_link_up_msg(sc);
679 
680 			update_media = true;
681 		} else { /* link is down */
682 			iflib_link_state_change(sc->ctx, LINK_STATE_DOWN, 0);
683 
684 			update_media = true;
685 		}
686 	}
687 
688 	/* Update the supported media types */
689 	if (update_media) {
690 		enum ice_status status = ice_add_media_types(sc, sc->media);
691 		if (status)
692 			device_printf(sc->dev, "Error adding device media types: %s aq_err %s\n",
693 				      ice_status_str(status),
694 				      ice_aq_str(hw->adminq.sq_last_status));
695 	}
696 
697 	/* TODO: notify VFs of link state change */
698 }
699 
700 /**
701  * ice_if_attach_post - Late device attach logic
702  * @ctx: the iflib context structure
703  *
704  * Called by iflib to finish up attaching the device. Performs any attach
705  * logic which must wait until after the Tx and Rx queues have been
706  * allocated.
707  */
708 static int
709 ice_if_attach_post(if_ctx_t ctx)
710 {
711 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
712 	if_t ifp = iflib_get_ifp(ctx);
713 	int err;
714 
715 	ASSERT_CTX_LOCKED(sc);
716 
717 	/* We don't yet support loading if MSI-X is not supported */
718 	if (sc->scctx->isc_intr != IFLIB_INTR_MSIX) {
719 		device_printf(sc->dev, "The ice driver does not support loading without MSI-X\n");
720 		return (ENOTSUP);
721 	}
722 
723 	/* The ifnet structure hasn't yet been initialized when the attach_pre
724 	 * handler is called, so wait until attach_post to setup the
725 	 * isc_max_frame_size.
726 	 */
727 
728 	sc->ifp = ifp;
729 	sc->scctx->isc_max_frame_size = ifp->if_mtu +
730 		ETHER_HDR_LEN + ETHER_CRC_LEN + ETHER_VLAN_ENCAP_LEN;
731 
732 	/*
733 	 * If we are in recovery mode, only perform a limited subset of
734 	 * initialization to support NVM recovery.
735 	 */
736 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) {
737 		ice_attach_post_recovery_mode(sc);
738 		return (0);
739 	}
740 
741 	sc->pf_vsi.max_frame_size = sc->scctx->isc_max_frame_size;
742 
743 	err = ice_initialize_vsi(&sc->pf_vsi);
744 	if (err) {
745 		device_printf(sc->dev, "Unable to initialize Main VSI: %s\n",
746 			      ice_err_str(err));
747 		return err;
748 	}
749 
750 	/* Configure the main PF VSI for RSS */
751 	err = ice_config_rss(&sc->pf_vsi);
752 	if (err) {
753 		device_printf(sc->dev,
754 			      "Unable to configure RSS for the main VSI, err %s\n",
755 			      ice_err_str(err));
756 		return err;
757 	}
758 
759 	/* Configure switch to drop transmitted LLDP and PAUSE frames */
760 	err = ice_cfg_pf_ethertype_filters(sc);
761 	if (err)
762 		return err;
763 
764 	ice_get_and_print_bus_info(sc);
765 
766 	ice_set_link_management_mode(sc);
767 
768 	ice_init_saved_phy_cfg(sc);
769 
770 	ice_add_device_sysctls(sc);
771 
772 	/* Get DCBX/LLDP state and start DCBX agent */
773 	ice_init_dcb_setup(sc);
774 
775 	/* Setup link configuration parameters */
776 	ice_init_link_configuration(sc);
777 	ice_update_link_status(sc, true);
778 
779 	/* Configure interrupt causes for the administrative interrupt */
780 	ice_configure_misc_interrupts(sc);
781 
782 	/* Enable ITR 0 right away, so that we can handle admin interrupts */
783 	ice_enable_intr(&sc->hw, sc->irqvs[0].me);
784 
785 	/* Start the admin timer */
786 	mtx_lock(&sc->admin_mtx);
787 	callout_reset(&sc->admin_timer, hz/2, ice_admin_timer, sc);
788 	mtx_unlock(&sc->admin_mtx);
789 
790 	return 0;
791 } /* ice_if_attach_post */
792 
793 /**
794  * ice_attach_post_recovery_mode - Limited driver attach_post for FW recovery
795  * @sc: the device private softc
796  *
797  * Performs minimal work to prepare the driver to recover an NVM in case the
798  * firmware is in recovery mode.
799  */
800 static void
801 ice_attach_post_recovery_mode(struct ice_softc *sc)
802 {
803 	/* Configure interrupt causes for the administrative interrupt */
804 	ice_configure_misc_interrupts(sc);
805 
806 	/* Enable ITR 0 right away, so that we can handle admin interrupts */
807 	ice_enable_intr(&sc->hw, sc->irqvs[0].me);
808 
809 	/* Start the admin timer */
810 	mtx_lock(&sc->admin_mtx);
811 	callout_reset(&sc->admin_timer, hz/2, ice_admin_timer, sc);
812 	mtx_unlock(&sc->admin_mtx);
813 }
814 
815 /**
816  * ice_free_irqvs - Free IRQ vector memory
817  * @sc: the device private softc structure
818  *
819  * Free IRQ vector memory allocated during ice_if_msix_intr_assign.
820  */
821 static void
822 ice_free_irqvs(struct ice_softc *sc)
823 {
824 	struct ice_vsi *vsi = &sc->pf_vsi;
825 	if_ctx_t ctx = sc->ctx;
826 	int i;
827 
828 	/* If the irqvs array is NULL, then there are no vectors to free */
829 	if (sc->irqvs == NULL)
830 		return;
831 
832 	/* Free the IRQ vectors */
833 	for (i = 0; i < sc->num_irq_vectors; i++)
834 		iflib_irq_free(ctx, &sc->irqvs[i].irq);
835 
836 	/* Clear the irqv pointers */
837 	for (i = 0; i < vsi->num_rx_queues; i++)
838 		vsi->rx_queues[i].irqv = NULL;
839 
840 	for (i = 0; i < vsi->num_tx_queues; i++)
841 		vsi->tx_queues[i].irqv = NULL;
842 
843 	/* Release the vector array memory */
844 	free(sc->irqvs, M_ICE);
845 	sc->irqvs = NULL;
846 	sc->num_irq_vectors = 0;
847 }
848 
849 /**
850  * ice_if_detach - Device driver detach logic
851  * @ctx: iflib context structure
852  *
853  * Perform device shutdown logic to detach the device driver.
854  *
855  * Note that there is no guarantee of the ordering of ice_if_queues_free() and
856  * ice_if_detach(). It is possible for the functions to be called in either
857  * order, and they must not assume to have a strict ordering.
858  */
859 static int
860 ice_if_detach(if_ctx_t ctx)
861 {
862 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
863 	struct ice_vsi *vsi = &sc->pf_vsi;
864 	int i;
865 
866 	ASSERT_CTX_LOCKED(sc);
867 
868 	/* Indicate that we're detaching */
869 	ice_set_state(&sc->state, ICE_STATE_DETACHING);
870 
871 	/* Stop the admin timer */
872 	mtx_lock(&sc->admin_mtx);
873 	callout_stop(&sc->admin_timer);
874 	mtx_unlock(&sc->admin_mtx);
875 	mtx_destroy(&sc->admin_mtx);
876 
877 	/* Free allocated media types */
878 	ifmedia_removeall(sc->media);
879 
880 	/* Free the Tx and Rx sysctl contexts, and assign NULL to the node
881 	 * pointers. Note, the calls here and those in ice_if_queues_free()
882 	 * are *BOTH* necessary, as we cannot guarantee which path will be
883 	 * run first
884 	 */
885 	ice_vsi_del_txqs_ctx(vsi);
886 	ice_vsi_del_rxqs_ctx(vsi);
887 
888 	/* Release MSI-X resources */
889 	ice_free_irqvs(sc);
890 
891 	for (i = 0; i < sc->num_available_vsi; i++) {
892 		if (sc->all_vsi[i])
893 			ice_release_vsi(sc->all_vsi[i]);
894 	}
895 
896 	if (sc->all_vsi) {
897 		free(sc->all_vsi, M_ICE);
898 		sc->all_vsi = NULL;
899 	}
900 
901 	/* Release MSI-X memory */
902 	pci_release_msi(sc->dev);
903 
904 	if (sc->msix_table != NULL) {
905 		bus_release_resource(sc->dev, SYS_RES_MEMORY,
906 				     rman_get_rid(sc->msix_table),
907 				     sc->msix_table);
908 		sc->msix_table = NULL;
909 	}
910 
911 	ice_free_intr_tracking(sc);
912 
913 	/* Destroy the queue managers */
914 	ice_resmgr_destroy(&sc->tx_qmgr);
915 	ice_resmgr_destroy(&sc->rx_qmgr);
916 
917 	if (!ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
918 		ice_deinit_hw(&sc->hw);
919 
920 	ice_free_pci_mapping(sc);
921 
922 	return 0;
923 } /* ice_if_detach */
924 
925 /**
926  * ice_if_tx_queues_alloc - Allocate Tx queue memory
927  * @ctx: iflib context structure
928  * @vaddrs: virtual addresses for the queue memory
929  * @paddrs: physical addresses for the queue memory
930  * @ntxqs: the number of Tx queues per set (should always be 1)
931  * @ntxqsets: the number of Tx queue sets to allocate
932  *
933  * Called by iflib to allocate Tx queues for the device. Allocates driver
934  * memory to track each queue, the status arrays used for descriptor
935  * status reporting, and Tx queue sysctls.
936  */
937 static int
938 ice_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs,
939 		       int __invariant_only ntxqs, int ntxqsets)
940 {
941 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
942 	struct ice_vsi *vsi = &sc->pf_vsi;
943 	struct ice_tx_queue *txq;
944 	int err, i, j;
945 
946 	MPASS(ntxqs == 1);
947 	MPASS(sc->scctx->isc_ntxd[0] <= ICE_MAX_DESC_COUNT);
948 	ASSERT_CTX_LOCKED(sc);
949 
950 	/* Do not bother allocating queues if we're in recovery mode */
951 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
952 		return (0);
953 
954 	/* Allocate queue structure memory */
955 	if (!(vsi->tx_queues =
956 	      (struct ice_tx_queue *) malloc(sizeof(struct ice_tx_queue) * ntxqsets, M_ICE, M_WAITOK | M_ZERO))) {
957 		device_printf(sc->dev, "Unable to allocate Tx queue memory\n");
958 		return (ENOMEM);
959 	}
960 
961 	/* Allocate report status arrays */
962 	for (i = 0, txq = vsi->tx_queues; i < ntxqsets; i++, txq++) {
963 		if (!(txq->tx_rsq =
964 		      (uint16_t *) malloc(sizeof(uint16_t) * sc->scctx->isc_ntxd[0], M_ICE, M_WAITOK))) {
965 			device_printf(sc->dev, "Unable to allocate tx_rsq memory\n");
966 			err = ENOMEM;
967 			goto free_tx_queues;
968 		}
969 		/* Initialize report status array */
970 		for (j = 0; j < sc->scctx->isc_ntxd[0]; j++)
971 			txq->tx_rsq[j] = QIDX_INVALID;
972 	}
973 
974 	/* Assign queues from PF space to the main VSI */
975 	err = ice_resmgr_assign_contiguous(&sc->tx_qmgr, vsi->tx_qmap, ntxqsets);
976 	if (err) {
977 		device_printf(sc->dev, "Unable to assign PF queues: %s\n",
978 			      ice_err_str(err));
979 		goto free_tx_queues;
980 	}
981 	vsi->qmap_type = ICE_RESMGR_ALLOC_CONTIGUOUS;
982 
983 	/* Add Tx queue sysctls context */
984 	ice_vsi_add_txqs_ctx(vsi);
985 
986 	for (i = 0, txq = vsi->tx_queues; i < ntxqsets; i++, txq++) {
987 		txq->me = i;
988 		txq->vsi = vsi;
989 
990 		/* store the queue size for easier access */
991 		txq->desc_count = sc->scctx->isc_ntxd[0];
992 
993 		/* get the virtual and physical address of the hardware queues */
994 		txq->tail = QTX_COMM_DBELL(vsi->tx_qmap[i]);
995 		txq->tx_base = (struct ice_tx_desc *)vaddrs[i];
996 		txq->tx_paddr = paddrs[i];
997 
998 		ice_add_txq_sysctls(txq);
999 	}
1000 
1001 	vsi->num_tx_queues = ntxqsets;
1002 
1003 	return (0);
1004 
1005 free_tx_queues:
1006 	for (i = 0, txq = vsi->tx_queues; i < ntxqsets; i++, txq++) {
1007 		if (txq->tx_rsq != NULL) {
1008 			free(txq->tx_rsq, M_ICE);
1009 			txq->tx_rsq = NULL;
1010 		}
1011 	}
1012 	free(vsi->tx_queues, M_ICE);
1013 	vsi->tx_queues = NULL;
1014 	return err;
1015 }
1016 
1017 /**
1018  * ice_if_rx_queues_alloc - Allocate Rx queue memory
1019  * @ctx: iflib context structure
1020  * @vaddrs: virtual addresses for the queue memory
1021  * @paddrs: physical addresses for the queue memory
1022  * @nrxqs: number of Rx queues per set (should always be 1)
1023  * @nrxqsets: number of Rx queue sets to allocate
1024  *
1025  * Called by iflib to allocate Rx queues for the device. Allocates driver
1026  * memory to track each queue, as well as sets up the Rx queue sysctls.
1027  */
1028 static int
1029 ice_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs,
1030 		       int __invariant_only nrxqs, int nrxqsets)
1031 {
1032 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1033 	struct ice_vsi *vsi = &sc->pf_vsi;
1034 	struct ice_rx_queue *rxq;
1035 	int err, i;
1036 
1037 	MPASS(nrxqs == 1);
1038 	MPASS(sc->scctx->isc_nrxd[0] <= ICE_MAX_DESC_COUNT);
1039 	ASSERT_CTX_LOCKED(sc);
1040 
1041 	/* Do not bother allocating queues if we're in recovery mode */
1042 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1043 		return (0);
1044 
1045 	/* Allocate queue structure memory */
1046 	if (!(vsi->rx_queues =
1047 	      (struct ice_rx_queue *) malloc(sizeof(struct ice_rx_queue) * nrxqsets, M_ICE, M_WAITOK | M_ZERO))) {
1048 		device_printf(sc->dev, "Unable to allocate Rx queue memory\n");
1049 		return (ENOMEM);
1050 	}
1051 
1052 	/* Assign queues from PF space to the main VSI */
1053 	err = ice_resmgr_assign_contiguous(&sc->rx_qmgr, vsi->rx_qmap, nrxqsets);
1054 	if (err) {
1055 		device_printf(sc->dev, "Unable to assign PF queues: %s\n",
1056 			      ice_err_str(err));
1057 		goto free_rx_queues;
1058 	}
1059 	vsi->qmap_type = ICE_RESMGR_ALLOC_CONTIGUOUS;
1060 
1061 	/* Add Rx queue sysctls context */
1062 	ice_vsi_add_rxqs_ctx(vsi);
1063 
1064 	for (i = 0, rxq = vsi->rx_queues; i < nrxqsets; i++, rxq++) {
1065 		rxq->me = i;
1066 		rxq->vsi = vsi;
1067 
1068 		/* store the queue size for easier access */
1069 		rxq->desc_count = sc->scctx->isc_nrxd[0];
1070 
1071 		/* get the virtual and physical address of the hardware queues */
1072 		rxq->tail = QRX_TAIL(vsi->rx_qmap[i]);
1073 		rxq->rx_base = (union ice_32b_rx_flex_desc *)vaddrs[i];
1074 		rxq->rx_paddr = paddrs[i];
1075 
1076 		ice_add_rxq_sysctls(rxq);
1077 	}
1078 
1079 	vsi->num_rx_queues = nrxqsets;
1080 
1081 	return (0);
1082 
1083 free_rx_queues:
1084 	free(vsi->rx_queues, M_ICE);
1085 	vsi->rx_queues = NULL;
1086 	return err;
1087 }
1088 
1089 /**
1090  * ice_if_queues_free - Free queue memory
1091  * @ctx: the iflib context structure
1092  *
1093  * Free queue memory allocated by ice_if_tx_queues_alloc() and
1094  * ice_if_rx_queues_alloc().
1095  *
1096  * There is no guarantee that ice_if_queues_free() and ice_if_detach() will be
1097  * called in the same order. It's possible for ice_if_queues_free() to be
1098  * called prior to ice_if_detach(), and vice versa.
1099  *
1100  * For this reason, the main VSI is a static member of the ice_softc, which is
1101  * not free'd until after iflib finishes calling both of these functions.
1102  *
1103  * Thus, care must be taken in how we manage the memory being freed by this
1104  * function, and in what tasks it can and must perform.
1105  */
1106 static void
1107 ice_if_queues_free(if_ctx_t ctx)
1108 {
1109 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1110 	struct ice_vsi *vsi = &sc->pf_vsi;
1111 	struct ice_tx_queue *txq;
1112 	int i;
1113 
1114 	/* Free the Tx and Rx sysctl contexts, and assign NULL to the node
1115 	 * pointers. Note, the calls here and those in ice_if_detach()
1116 	 * are *BOTH* necessary, as we cannot guarantee which path will be
1117 	 * run first
1118 	 */
1119 	ice_vsi_del_txqs_ctx(vsi);
1120 	ice_vsi_del_rxqs_ctx(vsi);
1121 
1122 	/* Release MSI-X IRQ vectors, if not yet released in ice_if_detach */
1123 	ice_free_irqvs(sc);
1124 
1125 	if (vsi->tx_queues != NULL) {
1126 		/* free the tx_rsq arrays */
1127 		for (i = 0, txq = vsi->tx_queues; i < vsi->num_tx_queues; i++, txq++) {
1128 			if (txq->tx_rsq != NULL) {
1129 				free(txq->tx_rsq, M_ICE);
1130 				txq->tx_rsq = NULL;
1131 			}
1132 		}
1133 		free(vsi->tx_queues, M_ICE);
1134 		vsi->tx_queues = NULL;
1135 		vsi->num_tx_queues = 0;
1136 	}
1137 	if (vsi->rx_queues != NULL) {
1138 		free(vsi->rx_queues, M_ICE);
1139 		vsi->rx_queues = NULL;
1140 		vsi->num_rx_queues = 0;
1141 	}
1142 }
1143 
1144 /**
1145  * ice_msix_que - Fast interrupt handler for MSI-X receive queues
1146  * @arg: The Rx queue memory
1147  *
1148  * Interrupt filter function for iflib MSI-X interrupts. Called by iflib when
1149  * an MSI-X interrupt for a given queue is triggered. Currently this just asks
1150  * iflib to schedule the main Rx thread.
1151  */
1152 static int
1153 ice_msix_que(void *arg)
1154 {
1155 	struct ice_rx_queue __unused *rxq = (struct ice_rx_queue *)arg;
1156 
1157 	/* TODO: dynamic ITR algorithm?? */
1158 
1159 	return (FILTER_SCHEDULE_THREAD);
1160 }
1161 
1162 /**
1163  * ice_msix_admin - Fast interrupt handler for MSI-X admin interrupt
1164  * @arg: pointer to device softc memory
1165  *
1166  * Called by iflib when an administrative interrupt occurs. Should perform any
1167  * fast logic for handling the interrupt cause, and then indicate whether the
1168  * admin task needs to be queued.
1169  */
1170 static int
1171 ice_msix_admin(void *arg)
1172 {
1173 	struct ice_softc *sc = (struct ice_softc *)arg;
1174 	struct ice_hw *hw = &sc->hw;
1175 	device_t dev = sc->dev;
1176 	u32 oicr;
1177 
1178 	/* There is no safe way to modify the enabled miscellaneous causes of
1179 	 * the OICR vector at runtime, as doing so would be prone to race
1180 	 * conditions. Reading PFINT_OICR will unmask the associated interrupt
1181 	 * causes and allow future interrupts to occur. The admin interrupt
1182 	 * vector will not be re-enabled until after we exit this function,
1183 	 * but any delayed tasks must be resilient against possible "late
1184 	 * arrival" interrupts that occur while we're already handling the
1185 	 * task. This is done by using state bits and serializing these
1186 	 * delayed tasks via the admin status task function.
1187 	 */
1188 	oicr = rd32(hw, PFINT_OICR);
1189 
1190 	/* Processing multiple controlq interrupts on a single vector does not
1191 	 * provide an indication of which controlq triggered the interrupt.
1192 	 * We might try reading the INTEVENT bit of the respective PFINT_*_CTL
1193 	 * registers. However, the INTEVENT bit is not guaranteed to be set as
1194 	 * it gets automatically cleared when the hardware acknowledges the
1195 	 * interrupt.
1196 	 *
1197 	 * This means we don't really have a good indication of whether or
1198 	 * which controlq triggered this interrupt. We'll just notify the
1199 	 * admin task that it should check all the controlqs.
1200 	 */
1201 	ice_set_state(&sc->state, ICE_STATE_CONTROLQ_EVENT_PENDING);
1202 
1203 	if (oicr & PFINT_OICR_VFLR_M) {
1204 		ice_set_state(&sc->state, ICE_STATE_VFLR_PENDING);
1205 	}
1206 
1207 	if (oicr & PFINT_OICR_MAL_DETECT_M) {
1208 		ice_set_state(&sc->state, ICE_STATE_MDD_PENDING);
1209 	}
1210 
1211 	if (oicr & PFINT_OICR_GRST_M) {
1212 		u32 reset;
1213 
1214 		reset = (rd32(hw, GLGEN_RSTAT) & GLGEN_RSTAT_RESET_TYPE_M) >>
1215 			GLGEN_RSTAT_RESET_TYPE_S;
1216 
1217 		if (reset == ICE_RESET_CORER)
1218 			sc->soft_stats.corer_count++;
1219 		else if (reset == ICE_RESET_GLOBR)
1220 			sc->soft_stats.globr_count++;
1221 		else
1222 			sc->soft_stats.empr_count++;
1223 
1224 		/* There are a couple of bits at play for handling resets.
1225 		 * First, the ICE_STATE_RESET_OICR_RECV bit is used to
1226 		 * indicate that the driver has received an OICR with a reset
1227 		 * bit active, indicating that a CORER/GLOBR/EMPR is about to
1228 		 * happen. Second, we set hw->reset_ongoing to indicate that
1229 		 * the hardware is in reset. We will set this back to false as
1230 		 * soon as the driver has determined that the hardware is out
1231 		 * of reset.
1232 		 *
1233 		 * If the driver wishes to trigger a reqest, it can set one of
1234 		 * the ICE_STATE_RESET_*_REQ bits, which will trigger the
1235 		 * correct type of reset.
1236 		 */
1237 		if (!ice_testandset_state(&sc->state, ICE_STATE_RESET_OICR_RECV))
1238 			hw->reset_ongoing = true;
1239 	}
1240 
1241 	if (oicr & PFINT_OICR_ECC_ERR_M) {
1242 		device_printf(dev, "ECC Error detected!\n");
1243 		ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ);
1244 	}
1245 
1246 	if (oicr & PFINT_OICR_PE_CRITERR_M) {
1247 		device_printf(dev, "Critical Protocol Engine Error detected!\n");
1248 		ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ);
1249 	}
1250 
1251 	if (oicr & PFINT_OICR_PCI_EXCEPTION_M) {
1252 		device_printf(dev, "PCI Exception detected!\n");
1253 		ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ);
1254 	}
1255 
1256 	if (oicr & PFINT_OICR_HMC_ERR_M) {
1257 		/* Log the HMC errors, but don't disable the interrupt cause */
1258 		ice_log_hmc_error(hw, dev);
1259 	}
1260 
1261 	return (FILTER_SCHEDULE_THREAD);
1262 }
1263 
1264 /**
1265  * ice_allocate_msix - Allocate MSI-X vectors for the interface
1266  * @sc: the device private softc
1267  *
1268  * Map the MSI-X bar, and then request MSI-X vectors in a two-stage process.
1269  *
1270  * First, determine a suitable total number of vectors based on the number
1271  * of CPUs, RSS buckets, the administrative vector, and other demands such as
1272  * RDMA.
1273  *
1274  * Request the desired amount of vectors, and see how many we obtain. If we
1275  * don't obtain as many as desired, reduce the demands by lowering the number
1276  * of requested queues or reducing the demand from other features such as
1277  * RDMA.
1278  *
1279  * @remark This function is required because the driver sets the
1280  * IFLIB_SKIP_MSIX flag indicating that the driver will manage MSI-X vectors
1281  * manually.
1282  *
1283  * @remark This driver will only use MSI-X vectors. If this is not possible,
1284  * neither MSI or legacy interrupts will be tried.
1285  *
1286  * @post on success this function must set the following scctx parameters:
1287  * isc_vectors, isc_nrxqsets, isc_ntxqsets, and isc_intr.
1288  *
1289  * @returns zero on success or an error code on failure.
1290  */
1291 static int
1292 ice_allocate_msix(struct ice_softc *sc)
1293 {
1294 	bool iflib_override_queue_count = false;
1295 	if_softc_ctx_t scctx = sc->scctx;
1296 	device_t dev = sc->dev;
1297 	cpuset_t cpus;
1298 	int bar, queues, vectors, requested;
1299 	int err = 0;
1300 
1301 	/* Allocate the MSI-X bar */
1302 	bar = scctx->isc_msix_bar;
1303 	sc->msix_table = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &bar, RF_ACTIVE);
1304 	if (!sc->msix_table) {
1305 		device_printf(dev, "Unable to map MSI-X table\n");
1306 		return (ENOMEM);
1307 	}
1308 
1309 	/* Check if the iflib queue count sysctls have been set */
1310 	if (sc->ifc_sysctl_ntxqs || sc->ifc_sysctl_nrxqs)
1311 		iflib_override_queue_count = true;
1312 
1313 	err = bus_get_cpus(dev, INTR_CPUS, sizeof(cpus), &cpus);
1314 	if (err) {
1315 		device_printf(dev, "%s: Unable to fetch the CPU list: %s\n",
1316 			      __func__, ice_err_str(err));
1317 		CPU_COPY(&all_cpus, &cpus);
1318 	}
1319 
1320 	/* Attempt to mimic behavior of iflib_msix_init */
1321 	if (iflib_override_queue_count) {
1322 		/*
1323 		 * If the override sysctls have been set, limit the queues to
1324 		 * the number of logical CPUs.
1325 		 */
1326 		queues = mp_ncpus;
1327 	} else {
1328 		/*
1329 		 * Otherwise, limit the queue count to the CPUs associated
1330 		 * with the NUMA node the device is associated with.
1331 		 */
1332 		queues = CPU_COUNT(&cpus);
1333 	}
1334 
1335 	/* Clamp to the number of RSS buckets */
1336 	queues = imin(queues, rss_getnumbuckets());
1337 
1338 	/*
1339 	 * Clamp the number of queue pairs to the minimum of the requested Tx
1340 	 * and Rx queues.
1341 	 */
1342 	queues = imin(queues, sc->ifc_sysctl_ntxqs ?: scctx->isc_ntxqsets);
1343 	queues = imin(queues, sc->ifc_sysctl_nrxqs ?: scctx->isc_nrxqsets);
1344 
1345 	/*
1346 	 * Determine the number of vectors to request. Note that we also need
1347 	 * to allocate one vector for administrative tasks.
1348 	 */
1349 	requested = queues + 1;
1350 
1351 	vectors = requested;
1352 
1353 	err = pci_alloc_msix(dev, &vectors);
1354 	if (err) {
1355 		device_printf(dev, "Failed to allocate %d MSI-X vectors, err %s\n",
1356 			      vectors, ice_err_str(err));
1357 		goto err_free_msix_table;
1358 	}
1359 
1360 	/* If we don't receive enough vectors, reduce demands */
1361 	if (vectors < requested) {
1362 		int diff = requested - vectors;
1363 
1364 		device_printf(dev, "Requested %d MSI-X vectors, but got only %d\n",
1365 			      requested, vectors);
1366 
1367 		/*
1368 		 * If we still have a difference, we need to reduce the number
1369 		 * of queue pairs.
1370 		 *
1371 		 * However, we still need at least one vector for the admin
1372 		 * interrupt and one queue pair.
1373 		 */
1374 		if (queues <= diff) {
1375 			device_printf(dev, "Unable to allocate sufficient MSI-X vectors\n");
1376 			err = (ERANGE);
1377 			goto err_pci_release_msi;
1378 		}
1379 
1380 		queues -= diff;
1381 	}
1382 
1383 	device_printf(dev, "Using %d Tx and Rx queues\n", queues);
1384 	device_printf(dev, "Using MSI-X interrupts with %d vectors\n",
1385 		      vectors);
1386 
1387 	scctx->isc_vectors = vectors;
1388 	scctx->isc_nrxqsets = queues;
1389 	scctx->isc_ntxqsets = queues;
1390 	scctx->isc_intr = IFLIB_INTR_MSIX;
1391 
1392 	/* Interrupt allocation tracking isn't required in recovery mode,
1393 	 * since neither RDMA nor VFs are enabled.
1394 	 */
1395 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1396 		return (0);
1397 
1398 	/* Keep track of which interrupt indices are being used for what */
1399 	sc->lan_vectors = vectors;
1400 	err = ice_resmgr_assign_contiguous(&sc->imgr, sc->pf_imap, sc->lan_vectors);
1401 	if (err) {
1402 		device_printf(dev, "Unable to assign PF interrupt mapping: %s\n",
1403 			      ice_err_str(err));
1404 		goto err_pci_release_msi;
1405 	}
1406 
1407 	return (0);
1408 
1409 err_pci_release_msi:
1410 	pci_release_msi(dev);
1411 err_free_msix_table:
1412 	if (sc->msix_table != NULL) {
1413 		bus_release_resource(sc->dev, SYS_RES_MEMORY,
1414 				rman_get_rid(sc->msix_table),
1415 				sc->msix_table);
1416 		sc->msix_table = NULL;
1417 	}
1418 
1419 	return (err);
1420 }
1421 
1422 /**
1423  * ice_if_msix_intr_assign - Assign MSI-X interrupt vectors to queues
1424  * @ctx: the iflib context structure
1425  * @msix: the number of vectors we were assigned
1426  *
1427  * Called by iflib to assign MSI-X vectors to queues. Currently requires that
1428  * we get at least the same number of vectors as we have queues, and that we
1429  * always have the same number of Tx and Rx queues.
1430  *
1431  * Tx queues use a softirq instead of using their own hardware interrupt.
1432  */
1433 static int
1434 ice_if_msix_intr_assign(if_ctx_t ctx, int msix)
1435 {
1436 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1437 	struct ice_vsi *vsi = &sc->pf_vsi;
1438 	int err, i, vector;
1439 
1440 	ASSERT_CTX_LOCKED(sc);
1441 
1442 	if (vsi->num_rx_queues != vsi->num_tx_queues) {
1443 		device_printf(sc->dev,
1444 			      "iflib requested %d Tx queues, and %d Rx queues, but the driver isn't able to support a differing number of Tx and Rx queues\n",
1445 			      vsi->num_tx_queues, vsi->num_rx_queues);
1446 		return (EOPNOTSUPP);
1447 	}
1448 
1449 	if (msix < (vsi->num_rx_queues + 1)) {
1450 		device_printf(sc->dev,
1451 			      "Not enough MSI-X vectors to assign one vector to each queue pair\n");
1452 		return (EOPNOTSUPP);
1453 	}
1454 
1455 	/* Save the number of vectors for future use */
1456 	sc->num_irq_vectors = vsi->num_rx_queues + 1;
1457 
1458 	/* Allocate space to store the IRQ vector data */
1459 	if (!(sc->irqvs =
1460 	      (struct ice_irq_vector *) malloc(sizeof(struct ice_irq_vector) * (sc->num_irq_vectors),
1461 					       M_ICE, M_NOWAIT))) {
1462 		device_printf(sc->dev,
1463 			      "Unable to allocate irqv memory\n");
1464 		return (ENOMEM);
1465 	}
1466 
1467 	/* Administrative interrupt events will use vector 0 */
1468 	err = iflib_irq_alloc_generic(ctx, &sc->irqvs[0].irq, 1, IFLIB_INTR_ADMIN,
1469 				      ice_msix_admin, sc, 0, "admin");
1470 	if (err) {
1471 		device_printf(sc->dev,
1472 			      "Failed to register Admin queue handler: %s\n",
1473 			      ice_err_str(err));
1474 		goto free_irqvs;
1475 	}
1476 	sc->irqvs[0].me = 0;
1477 
1478 	/* Do not allocate queue interrupts when in recovery mode */
1479 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1480 		return (0);
1481 
1482 	for (i = 0, vector = 1; i < vsi->num_rx_queues; i++, vector++) {
1483 		struct ice_rx_queue *rxq = &vsi->rx_queues[i];
1484 		struct ice_tx_queue *txq = &vsi->tx_queues[i];
1485 		int rid = vector + 1;
1486 		char irq_name[16];
1487 
1488 		snprintf(irq_name, sizeof(irq_name), "rxq%d", i);
1489 		err = iflib_irq_alloc_generic(ctx, &sc->irqvs[vector].irq, rid,
1490 					      IFLIB_INTR_RX, ice_msix_que,
1491 					      rxq, rxq->me, irq_name);
1492 		if (err) {
1493 			device_printf(sc->dev,
1494 				      "Failed to allocate q int %d err: %s\n",
1495 				      i, ice_err_str(err));
1496 			vector--;
1497 			i--;
1498 			goto fail;
1499 		}
1500 		sc->irqvs[vector].me = vector;
1501 		rxq->irqv = &sc->irqvs[vector];
1502 
1503 		bzero(irq_name, sizeof(irq_name));
1504 
1505 		snprintf(irq_name, sizeof(irq_name), "txq%d", i);
1506 		iflib_softirq_alloc_generic(ctx, &sc->irqvs[vector].irq,
1507 					    IFLIB_INTR_TX, txq,
1508 					    txq->me, irq_name);
1509 		txq->irqv = &sc->irqvs[vector];
1510 	}
1511 
1512 	return (0);
1513 fail:
1514 	for (; i >= 0; i--, vector--)
1515 		iflib_irq_free(ctx, &sc->irqvs[vector].irq);
1516 	iflib_irq_free(ctx, &sc->irqvs[0].irq);
1517 free_irqvs:
1518 	free(sc->irqvs, M_ICE);
1519 	sc->irqvs = NULL;
1520 	return err;
1521 }
1522 
1523 /**
1524  * ice_if_mtu_set - Set the device MTU
1525  * @ctx: iflib context structure
1526  * @mtu: the MTU requested
1527  *
1528  * Called by iflib to configure the device's Maximum Transmission Unit (MTU).
1529  *
1530  * @pre assumes the caller holds the iflib CTX lock
1531  */
1532 static int
1533 ice_if_mtu_set(if_ctx_t ctx, uint32_t mtu)
1534 {
1535 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1536 
1537 	ASSERT_CTX_LOCKED(sc);
1538 
1539 	/* Do not support configuration when in recovery mode */
1540 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1541 		return (ENOSYS);
1542 
1543 	if (mtu < ICE_MIN_MTU || mtu > ICE_MAX_MTU)
1544 		return (EINVAL);
1545 
1546 	sc->scctx->isc_max_frame_size = mtu +
1547 		ETHER_HDR_LEN + ETHER_CRC_LEN + ETHER_VLAN_ENCAP_LEN;
1548 
1549 	sc->pf_vsi.max_frame_size = sc->scctx->isc_max_frame_size;
1550 
1551 	return (0);
1552 }
1553 
1554 /**
1555  * ice_if_intr_enable - Enable device interrupts
1556  * @ctx: iflib context structure
1557  *
1558  * Called by iflib to request enabling device interrupts.
1559  */
1560 static void
1561 ice_if_intr_enable(if_ctx_t ctx)
1562 {
1563 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1564 	struct ice_vsi *vsi = &sc->pf_vsi;
1565 	struct ice_hw *hw = &sc->hw;
1566 
1567 	ASSERT_CTX_LOCKED(sc);
1568 
1569 	/* Enable ITR 0 */
1570 	ice_enable_intr(hw, sc->irqvs[0].me);
1571 
1572 	/* Do not enable queue interrupts in recovery mode */
1573 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1574 		return;
1575 
1576 	/* Enable all queue interrupts */
1577 	for (int i = 0; i < vsi->num_rx_queues; i++)
1578 		ice_enable_intr(hw, vsi->rx_queues[i].irqv->me);
1579 }
1580 
1581 /**
1582  * ice_if_intr_disable - Disable device interrupts
1583  * @ctx: iflib context structure
1584  *
1585  * Called by iflib to request disabling device interrupts.
1586  */
1587 static void
1588 ice_if_intr_disable(if_ctx_t ctx)
1589 {
1590 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1591 	struct ice_hw *hw = &sc->hw;
1592 	unsigned int i;
1593 
1594 	ASSERT_CTX_LOCKED(sc);
1595 
1596 	/* IFDI_INTR_DISABLE may be called prior to interrupts actually being
1597 	 * assigned to queues. Instead of assuming that the interrupt
1598 	 * assignment in the rx_queues structure is valid, just disable all
1599 	 * possible interrupts
1600 	 *
1601 	 * Note that we choose not to disable ITR 0 because this handles the
1602 	 * AdminQ interrupts, and we want to keep processing these even when
1603 	 * the interface is offline.
1604 	 */
1605 	for (i = 1; i < hw->func_caps.common_cap.num_msix_vectors; i++)
1606 		ice_disable_intr(hw, i);
1607 }
1608 
1609 /**
1610  * ice_if_rx_queue_intr_enable - Enable a specific Rx queue interrupt
1611  * @ctx: iflib context structure
1612  * @rxqid: the Rx queue to enable
1613  *
1614  * Enable a specific Rx queue interrupt.
1615  *
1616  * This function is not protected by the iflib CTX lock.
1617  */
1618 static int
1619 ice_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid)
1620 {
1621 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1622 	struct ice_vsi *vsi = &sc->pf_vsi;
1623 	struct ice_hw *hw = &sc->hw;
1624 
1625 	/* Do not enable queue interrupts in recovery mode */
1626 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1627 		return (ENOSYS);
1628 
1629 	ice_enable_intr(hw, vsi->rx_queues[rxqid].irqv->me);
1630 	return (0);
1631 }
1632 
1633 /**
1634  * ice_if_tx_queue_intr_enable - Enable a specific Tx queue interrupt
1635  * @ctx: iflib context structure
1636  * @txqid: the Tx queue to enable
1637  *
1638  * Enable a specific Tx queue interrupt.
1639  *
1640  * This function is not protected by the iflib CTX lock.
1641  */
1642 static int
1643 ice_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid)
1644 {
1645 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1646 	struct ice_vsi *vsi = &sc->pf_vsi;
1647 	struct ice_hw *hw = &sc->hw;
1648 
1649 	/* Do not enable queue interrupts in recovery mode */
1650 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1651 		return (ENOSYS);
1652 
1653 	ice_enable_intr(hw, vsi->tx_queues[txqid].irqv->me);
1654 	return (0);
1655 }
1656 
1657 /**
1658  * ice_if_promisc_set - Set device promiscuous mode
1659  * @ctx: iflib context structure
1660  * @flags: promiscuous flags to configure
1661  *
1662  * Called by iflib to configure device promiscuous mode.
1663  *
1664  * @remark Calls to this function will always overwrite the previous setting
1665  */
1666 static int
1667 ice_if_promisc_set(if_ctx_t ctx, int flags)
1668 {
1669 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1670 	struct ice_hw *hw = &sc->hw;
1671 	device_t dev = sc->dev;
1672 	enum ice_status status;
1673 	bool promisc_enable = flags & IFF_PROMISC;
1674 	bool multi_enable = flags & IFF_ALLMULTI;
1675 
1676 	/* Do not support configuration when in recovery mode */
1677 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1678 		return (ENOSYS);
1679 
1680 	if (multi_enable)
1681 		return (EOPNOTSUPP);
1682 
1683 	if (promisc_enable) {
1684 		status = ice_set_vsi_promisc(hw, sc->pf_vsi.idx,
1685 					     ICE_VSI_PROMISC_MASK, 0);
1686 		if (status && status != ICE_ERR_ALREADY_EXISTS) {
1687 			device_printf(dev,
1688 				      "Failed to enable promiscuous mode for PF VSI, err %s aq_err %s\n",
1689 				      ice_status_str(status),
1690 				      ice_aq_str(hw->adminq.sq_last_status));
1691 			return (EIO);
1692 		}
1693 	} else {
1694 		status = ice_clear_vsi_promisc(hw, sc->pf_vsi.idx,
1695 					       ICE_VSI_PROMISC_MASK, 0);
1696 		if (status) {
1697 			device_printf(dev,
1698 				      "Failed to disable promiscuous mode for PF VSI, err %s aq_err %s\n",
1699 				      ice_status_str(status),
1700 				      ice_aq_str(hw->adminq.sq_last_status));
1701 			return (EIO);
1702 		}
1703 	}
1704 
1705 	return (0);
1706 }
1707 
1708 /**
1709  * ice_if_media_change - Change device media
1710  * @ctx: device ctx structure
1711  *
1712  * Called by iflib when a media change is requested. This operation is not
1713  * supported by the hardware, so we just return an error code.
1714  */
1715 static int
1716 ice_if_media_change(if_ctx_t ctx)
1717 {
1718 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1719 
1720 	device_printf(sc->dev, "Media change is not supported.\n");
1721 	return (ENODEV);
1722 }
1723 
1724 /**
1725  * ice_if_media_status - Report current device media
1726  * @ctx: iflib context structure
1727  * @ifmr: ifmedia request structure to update
1728  *
1729  * Updates the provided ifmr with current device media status, including link
1730  * status and media type.
1731  */
1732 static void
1733 ice_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr)
1734 {
1735 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1736 	struct ice_link_status *li = &sc->hw.port_info->phy.link_info;
1737 
1738 	ifmr->ifm_status = IFM_AVALID;
1739 	ifmr->ifm_active = IFM_ETHER;
1740 
1741 	/* Never report link up or media types when in recovery mode */
1742 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1743 		return;
1744 
1745 	if (!sc->link_up)
1746 		return;
1747 
1748 	ifmr->ifm_status |= IFM_ACTIVE;
1749 	ifmr->ifm_active |= IFM_FDX;
1750 
1751 	if (li->phy_type_low)
1752 		ifmr->ifm_active |= ice_get_phy_type_low(li->phy_type_low);
1753 	else if (li->phy_type_high)
1754 		ifmr->ifm_active |= ice_get_phy_type_high(li->phy_type_high);
1755 	else
1756 		ifmr->ifm_active |= IFM_UNKNOWN;
1757 
1758 	/* Report flow control status as well */
1759 	if (li->an_info & ICE_AQ_LINK_PAUSE_TX)
1760 		ifmr->ifm_active |= IFM_ETH_TXPAUSE;
1761 	if (li->an_info & ICE_AQ_LINK_PAUSE_RX)
1762 		ifmr->ifm_active |= IFM_ETH_RXPAUSE;
1763 }
1764 
1765 /**
1766  * ice_init_tx_tracking - Initialize Tx queue software tracking values
1767  * @vsi: the VSI to initialize
1768  *
1769  * Initialize Tx queue software tracking values, including the Report Status
1770  * queue, and related software tracking values.
1771  */
1772 static void
1773 ice_init_tx_tracking(struct ice_vsi *vsi)
1774 {
1775 	struct ice_tx_queue *txq;
1776 	size_t j;
1777 	int i;
1778 
1779 	for (i = 0, txq = vsi->tx_queues; i < vsi->num_tx_queues; i++, txq++) {
1780 
1781 		txq->tx_rs_cidx = txq->tx_rs_pidx = 0;
1782 
1783 		/* Initialize the last processed descriptor to be the end of
1784 		 * the ring, rather than the start, so that we avoid an
1785 		 * off-by-one error in ice_ift_txd_credits_update for the
1786 		 * first packet.
1787 		 */
1788 		txq->tx_cidx_processed = txq->desc_count - 1;
1789 
1790 		for (j = 0; j < txq->desc_count; j++)
1791 			txq->tx_rsq[j] = QIDX_INVALID;
1792 	}
1793 }
1794 
1795 /**
1796  * ice_update_rx_mbuf_sz - Update the Rx buffer size for all queues
1797  * @sc: the device softc
1798  *
1799  * Called to update the Rx queue mbuf_sz parameter for configuring the receive
1800  * buffer sizes when programming hardware.
1801  */
1802 static void
1803 ice_update_rx_mbuf_sz(struct ice_softc *sc)
1804 {
1805 	uint32_t mbuf_sz = iflib_get_rx_mbuf_sz(sc->ctx);
1806 	struct ice_vsi *vsi = &sc->pf_vsi;
1807 
1808 	MPASS(mbuf_sz <= UINT16_MAX);
1809 	vsi->mbuf_sz = mbuf_sz;
1810 }
1811 
1812 /**
1813  * ice_if_init - Initialize the device
1814  * @ctx: iflib ctx structure
1815  *
1816  * Called by iflib to bring the device up, i.e. ifconfig ice0 up. Initializes
1817  * device filters and prepares the Tx and Rx engines.
1818  *
1819  * @pre assumes the caller holds the iflib CTX lock
1820  */
1821 static void
1822 ice_if_init(if_ctx_t ctx)
1823 {
1824 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1825 	device_t dev = sc->dev;
1826 	int err;
1827 
1828 	ASSERT_CTX_LOCKED(sc);
1829 
1830 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1831 		return;
1832 
1833 	if (ice_test_state(&sc->state, ICE_STATE_RESET_FAILED)) {
1834 		device_printf(sc->dev, "request to start interface cannot be completed as the device failed to reset\n");
1835 		return;
1836 	}
1837 
1838 	if (ice_test_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET)) {
1839 		device_printf(sc->dev, "request to start interface while device is prepared for impending reset\n");
1840 		return;
1841 	}
1842 
1843 	ice_update_rx_mbuf_sz(sc);
1844 
1845 	/* Update the MAC address... User might use a LAA */
1846 	err = ice_update_laa_mac(sc);
1847 	if (err) {
1848 		device_printf(dev,
1849 			      "LAA address change failed, err %s\n",
1850 			      ice_err_str(err));
1851 		return;
1852 	}
1853 
1854 	/* Initialize software Tx tracking values */
1855 	ice_init_tx_tracking(&sc->pf_vsi);
1856 
1857 	err = ice_cfg_vsi_for_tx(&sc->pf_vsi);
1858 	if (err) {
1859 		device_printf(dev,
1860 			      "Unable to configure the main VSI for Tx: %s\n",
1861 			      ice_err_str(err));
1862 		return;
1863 	}
1864 
1865 	err = ice_cfg_vsi_for_rx(&sc->pf_vsi);
1866 	if (err) {
1867 		device_printf(dev,
1868 			      "Unable to configure the main VSI for Rx: %s\n",
1869 			      ice_err_str(err));
1870 		goto err_cleanup_tx;
1871 	}
1872 
1873 	err = ice_control_rx_queues(&sc->pf_vsi, true);
1874 	if (err) {
1875 		device_printf(dev,
1876 			      "Unable to enable Rx rings for transmit: %s\n",
1877 			      ice_err_str(err));
1878 		goto err_cleanup_tx;
1879 	}
1880 
1881 	err = ice_cfg_pf_default_mac_filters(sc);
1882 	if (err) {
1883 		device_printf(dev,
1884 			      "Unable to configure default MAC filters: %s\n",
1885 			      ice_err_str(err));
1886 		goto err_stop_rx;
1887 	}
1888 
1889 	/* We use software interrupts for Tx, so we only program the hardware
1890 	 * interrupts for Rx.
1891 	 */
1892 	ice_configure_rxq_interrupts(&sc->pf_vsi);
1893 	ice_configure_rx_itr(&sc->pf_vsi);
1894 
1895 	/* Configure promiscuous mode */
1896 	ice_if_promisc_set(ctx, if_getflags(sc->ifp));
1897 
1898 	ice_set_state(&sc->state, ICE_STATE_DRIVER_INITIALIZED);
1899 	return;
1900 
1901 err_stop_rx:
1902 	ice_control_rx_queues(&sc->pf_vsi, false);
1903 err_cleanup_tx:
1904 	ice_vsi_disable_tx(&sc->pf_vsi);
1905 }
1906 
1907 /**
1908  * ice_poll_for_media_avail - Re-enable link if media is detected
1909  * @sc: device private structure
1910  *
1911  * Intended to be called from the driver's timer function, this function
1912  * sends the Get Link Status AQ command and re-enables HW link if the
1913  * command says that media is available.
1914  *
1915  * If the driver doesn't have the "NO_MEDIA" state set, then this does nothing,
1916  * since media removal events are supposed to be sent to the driver through
1917  * a link status event.
1918  */
1919 static void
1920 ice_poll_for_media_avail(struct ice_softc *sc)
1921 {
1922 	struct ice_hw *hw = &sc->hw;
1923 	struct ice_port_info *pi = hw->port_info;
1924 
1925 	if (ice_test_state(&sc->state, ICE_STATE_NO_MEDIA)) {
1926 		pi->phy.get_link_info = true;
1927 		ice_get_link_status(pi, &sc->link_up);
1928 
1929 		if (pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE) {
1930 			enum ice_status status;
1931 
1932 			/* Re-enable link and re-apply user link settings */
1933 			ice_apply_saved_phy_cfg(sc);
1934 
1935 			/* Update the OS about changes in media capability */
1936 			status = ice_add_media_types(sc, sc->media);
1937 			if (status)
1938 				device_printf(sc->dev, "Error adding device media types: %s aq_err %s\n",
1939 					      ice_status_str(status),
1940 					      ice_aq_str(hw->adminq.sq_last_status));
1941 
1942 			ice_clear_state(&sc->state, ICE_STATE_NO_MEDIA);
1943 		}
1944 	}
1945 }
1946 
1947 /**
1948  * ice_if_timer - called by iflib periodically
1949  * @ctx: iflib ctx structure
1950  * @qid: the queue this timer was called for
1951  *
1952  * This callback is triggered by iflib periodically. We use it to update the
1953  * hw statistics.
1954  *
1955  * @remark this function is not protected by the iflib CTX lock.
1956  */
1957 static void
1958 ice_if_timer(if_ctx_t ctx, uint16_t qid)
1959 {
1960 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1961 	uint64_t prev_link_xoff_rx = sc->stats.cur.link_xoff_rx;
1962 
1963 	if (qid != 0)
1964 		return;
1965 
1966 	/* Do not attempt to update stats when in recovery mode */
1967 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1968 		return;
1969 
1970 	/* Update device statistics */
1971 	ice_update_pf_stats(sc);
1972 
1973 	/*
1974 	 * For proper watchdog management, the iflib stack needs to know if
1975 	 * we've been paused during the last interval. Check if the
1976 	 * link_xoff_rx stat changed, and set the isc_pause_frames, if so.
1977 	 */
1978 	if (sc->stats.cur.link_xoff_rx != prev_link_xoff_rx)
1979 		sc->scctx->isc_pause_frames = 1;
1980 
1981 	/* Update the primary VSI stats */
1982 	ice_update_vsi_hw_stats(&sc->pf_vsi);
1983 }
1984 
1985 /**
1986  * ice_admin_timer - called periodically to trigger the admin task
1987  * @arg: callout(9) argument pointing to the device private softc structure
1988  *
1989  * Timer function used as part of a callout(9) timer that will periodically
1990  * trigger the admin task, even when the interface is down.
1991  *
1992  * @remark this function is not called by iflib and is not protected by the
1993  * iflib CTX lock.
1994  *
1995  * @remark because this is a callout function, it cannot sleep and should not
1996  * attempt taking the iflib CTX lock.
1997  */
1998 static void
1999 ice_admin_timer(void *arg)
2000 {
2001 	struct ice_softc *sc = (struct ice_softc *)arg;
2002 
2003 	/* Fire off the admin task */
2004 	iflib_admin_intr_deferred(sc->ctx);
2005 
2006 	/* Reschedule the admin timer */
2007 	callout_schedule(&sc->admin_timer, hz/2);
2008 }
2009 
2010 /**
2011  * ice_transition_recovery_mode - Transition to recovery mode
2012  * @sc: the device private softc
2013  *
2014  * Called when the driver detects that the firmware has entered recovery mode
2015  * at run time.
2016  */
2017 static void
2018 ice_transition_recovery_mode(struct ice_softc *sc)
2019 {
2020 	struct ice_vsi *vsi = &sc->pf_vsi;
2021 	int i;
2022 
2023 	device_printf(sc->dev, "Firmware recovery mode detected. Limiting functionality. Refer to Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n");
2024 
2025 	/* Tell the stack that the link has gone down */
2026 	iflib_link_state_change(sc->ctx, LINK_STATE_DOWN, 0);
2027 
2028 	/* Request that the device be re-initialized */
2029 	ice_request_stack_reinit(sc);
2030 
2031 	ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_en);
2032 	ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_cap);
2033 
2034 	ice_vsi_del_txqs_ctx(vsi);
2035 	ice_vsi_del_rxqs_ctx(vsi);
2036 
2037 	for (i = 0; i < sc->num_available_vsi; i++) {
2038 		if (sc->all_vsi[i])
2039 			ice_release_vsi(sc->all_vsi[i]);
2040 	}
2041 	sc->num_available_vsi = 0;
2042 
2043 	if (sc->all_vsi) {
2044 		free(sc->all_vsi, M_ICE);
2045 		sc->all_vsi = NULL;
2046 	}
2047 
2048 	/* Destroy the interrupt manager */
2049 	ice_resmgr_destroy(&sc->imgr);
2050 	/* Destroy the queue managers */
2051 	ice_resmgr_destroy(&sc->tx_qmgr);
2052 	ice_resmgr_destroy(&sc->rx_qmgr);
2053 
2054 	ice_deinit_hw(&sc->hw);
2055 }
2056 
2057 /**
2058  * ice_transition_safe_mode - Transition to safe mode
2059  * @sc: the device private softc
2060  *
2061  * Called when the driver attempts to reload the DDP package during a device
2062  * reset, and the new download fails. If so, we must transition to safe mode
2063  * at run time.
2064  *
2065  * @remark although safe mode normally allocates only a single queue, we can't
2066  * change the number of queues dynamically when using iflib. Due to this, we
2067  * do not attempt to reduce the number of queues.
2068  */
2069 static void
2070 ice_transition_safe_mode(struct ice_softc *sc)
2071 {
2072 	/* Indicate that we are in Safe mode */
2073 	ice_set_bit(ICE_FEATURE_SAFE_MODE, sc->feat_cap);
2074 	ice_set_bit(ICE_FEATURE_SAFE_MODE, sc->feat_en);
2075 
2076 	ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_en);
2077 	ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_cap);
2078 
2079 	ice_clear_bit(ICE_FEATURE_RSS, sc->feat_cap);
2080 	ice_clear_bit(ICE_FEATURE_RSS, sc->feat_en);
2081 }
2082 
2083 /**
2084  * ice_if_update_admin_status - update admin status
2085  * @ctx: iflib ctx structure
2086  *
2087  * Called by iflib to update the admin status. For our purposes, this means
2088  * check the adminq, and update the link status. It's ultimately triggered by
2089  * our admin interrupt, or by the ice_if_timer periodically.
2090  *
2091  * @pre assumes the caller holds the iflib CTX lock
2092  */
2093 static void
2094 ice_if_update_admin_status(if_ctx_t ctx)
2095 {
2096 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
2097 	enum ice_fw_modes fw_mode;
2098 	bool reschedule = false;
2099 	u16 pending = 0;
2100 
2101 	ASSERT_CTX_LOCKED(sc);
2102 
2103 	/* Check if the firmware entered recovery mode at run time */
2104 	fw_mode = ice_get_fw_mode(&sc->hw);
2105 	if (fw_mode == ICE_FW_MODE_REC) {
2106 		if (!ice_testandset_state(&sc->state, ICE_STATE_RECOVERY_MODE)) {
2107 			/* If we just entered recovery mode, log a warning to
2108 			 * the system administrator and deinit driver state
2109 			 * that is no longer functional.
2110 			 */
2111 			ice_transition_recovery_mode(sc);
2112 		}
2113 	} else if (fw_mode == ICE_FW_MODE_ROLLBACK) {
2114 		if (!ice_testandset_state(&sc->state, ICE_STATE_ROLLBACK_MODE)) {
2115 			/* Rollback mode isn't fatal, but we don't want to
2116 			 * repeatedly post a message about it.
2117 			 */
2118 			ice_print_rollback_msg(&sc->hw);
2119 		}
2120 	}
2121 
2122 	/* Handle global reset events */
2123 	ice_handle_reset_event(sc);
2124 
2125 	/* Handle PF reset requests */
2126 	ice_handle_pf_reset_request(sc);
2127 
2128 	/* Handle MDD events */
2129 	ice_handle_mdd_event(sc);
2130 
2131 	if (ice_test_state(&sc->state, ICE_STATE_RESET_FAILED) ||
2132 	    ice_test_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET) ||
2133 	    ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) {
2134 		/*
2135 		 * If we know the control queues are disabled, skip processing
2136 		 * the control queues entirely.
2137 		 */
2138 		;
2139 	} else if (ice_testandclear_state(&sc->state, ICE_STATE_CONTROLQ_EVENT_PENDING)) {
2140 		ice_process_ctrlq(sc, ICE_CTL_Q_ADMIN, &pending);
2141 		if (pending > 0)
2142 			reschedule = true;
2143 
2144 		ice_process_ctrlq(sc, ICE_CTL_Q_MAILBOX, &pending);
2145 		if (pending > 0)
2146 			reschedule = true;
2147 	}
2148 
2149 	/* Poll for link up */
2150 	ice_poll_for_media_avail(sc);
2151 
2152 	/* Check and update link status */
2153 	ice_update_link_status(sc, false);
2154 
2155 	/*
2156 	 * If there are still messages to process, we need to reschedule
2157 	 * ourselves. Otherwise, we can just re-enable the interrupt. We'll be
2158 	 * woken up at the next interrupt or timer event.
2159 	 */
2160 	if (reschedule) {
2161 		ice_set_state(&sc->state, ICE_STATE_CONTROLQ_EVENT_PENDING);
2162 		iflib_admin_intr_deferred(ctx);
2163 	} else {
2164 		ice_enable_intr(&sc->hw, sc->irqvs[0].me);
2165 	}
2166 }
2167 
2168 /**
2169  * ice_prepare_for_reset - Prepare device for an impending reset
2170  * @sc: The device private softc
2171  *
2172  * Prepare the driver for an impending reset, shutting down VSIs, clearing the
2173  * scheduler setup, and shutting down controlqs. Uses the
2174  * ICE_STATE_PREPARED_FOR_RESET to indicate whether we've already prepared the
2175  * driver for reset or not.
2176  */
2177 static void
2178 ice_prepare_for_reset(struct ice_softc *sc)
2179 {
2180 	struct ice_hw *hw = &sc->hw;
2181 
2182 	/* If we're already prepared, there's nothing to do */
2183 	if (ice_testandset_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET))
2184 		return;
2185 
2186 	log(LOG_INFO, "%s: preparing to reset device logic\n", sc->ifp->if_xname);
2187 
2188 	/* In recovery mode, hardware is not initialized */
2189 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
2190 		return;
2191 
2192 	/* Release the main PF VSI queue mappings */
2193 	ice_resmgr_release_map(&sc->tx_qmgr, sc->pf_vsi.tx_qmap,
2194 				    sc->pf_vsi.num_tx_queues);
2195 	ice_resmgr_release_map(&sc->rx_qmgr, sc->pf_vsi.rx_qmap,
2196 				    sc->pf_vsi.num_rx_queues);
2197 
2198 	ice_clear_hw_tbls(hw);
2199 
2200 	if (hw->port_info)
2201 		ice_sched_clear_port(hw->port_info);
2202 
2203 	ice_shutdown_all_ctrlq(hw);
2204 }
2205 
2206 /**
2207  * ice_rebuild_pf_vsi_qmap - Rebuild the main PF VSI queue mapping
2208  * @sc: the device softc pointer
2209  *
2210  * Loops over the Tx and Rx queues for the main PF VSI and reassigns the queue
2211  * mapping after a reset occurred.
2212  */
2213 static int
2214 ice_rebuild_pf_vsi_qmap(struct ice_softc *sc)
2215 {
2216 	struct ice_vsi *vsi = &sc->pf_vsi;
2217 	struct ice_tx_queue *txq;
2218 	struct ice_rx_queue *rxq;
2219 	int err, i;
2220 
2221 	/* Re-assign Tx queues from PF space to the main VSI */
2222 	err = ice_resmgr_assign_contiguous(&sc->tx_qmgr, vsi->tx_qmap,
2223 					    vsi->num_tx_queues);
2224 	if (err) {
2225 		device_printf(sc->dev, "Unable to re-assign PF Tx queues: %s\n",
2226 			      ice_err_str(err));
2227 		return (err);
2228 	}
2229 
2230 	/* Re-assign Rx queues from PF space to this VSI */
2231 	err = ice_resmgr_assign_contiguous(&sc->rx_qmgr, vsi->rx_qmap,
2232 					    vsi->num_rx_queues);
2233 	if (err) {
2234 		device_printf(sc->dev, "Unable to re-assign PF Rx queues: %s\n",
2235 			      ice_err_str(err));
2236 		goto err_release_tx_queues;
2237 	}
2238 
2239 	vsi->qmap_type = ICE_RESMGR_ALLOC_CONTIGUOUS;
2240 
2241 	/* Re-assign Tx queue tail pointers */
2242 	for (i = 0, txq = vsi->tx_queues; i < vsi->num_tx_queues; i++, txq++)
2243 		txq->tail = QTX_COMM_DBELL(vsi->tx_qmap[i]);
2244 
2245 	/* Re-assign Rx queue tail pointers */
2246 	for (i = 0, rxq = vsi->rx_queues; i < vsi->num_rx_queues; i++, rxq++)
2247 		rxq->tail = QRX_TAIL(vsi->rx_qmap[i]);
2248 
2249 	return (0);
2250 
2251 err_release_tx_queues:
2252 	ice_resmgr_release_map(&sc->tx_qmgr, sc->pf_vsi.tx_qmap,
2253 				   sc->pf_vsi.num_tx_queues);
2254 
2255 	return (err);
2256 }
2257 
2258 /* determine if the iflib context is active */
2259 #define CTX_ACTIVE(ctx) ((if_getdrvflags(iflib_get_ifp(ctx)) & IFF_DRV_RUNNING))
2260 
2261 /**
2262  * ice_rebuild_recovery_mode - Rebuild driver state while in recovery mode
2263  * @sc: The device private softc
2264  *
2265  * Handle a driver rebuild while in recovery mode. This will only rebuild the
2266  * limited functionality supported while in recovery mode.
2267  */
2268 static void
2269 ice_rebuild_recovery_mode(struct ice_softc *sc)
2270 {
2271 	device_t dev = sc->dev;
2272 
2273 	/* enable PCIe bus master */
2274 	pci_enable_busmaster(dev);
2275 
2276 	/* Configure interrupt causes for the administrative interrupt */
2277 	ice_configure_misc_interrupts(sc);
2278 
2279 	/* Enable ITR 0 right away, so that we can handle admin interrupts */
2280 	ice_enable_intr(&sc->hw, sc->irqvs[0].me);
2281 
2282 	/* Now that the rebuild is finished, we're no longer prepared to reset */
2283 	ice_clear_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET);
2284 
2285 	log(LOG_INFO, "%s: device rebuild successful\n", sc->ifp->if_xname);
2286 
2287 	/* In order to completely restore device functionality, the iflib core
2288 	 * needs to be reset. We need to request an iflib reset. Additionally,
2289 	 * because the state of IFC_DO_RESET is cached within task_fn_admin in
2290 	 * the iflib core, we also want re-run the admin task so that iflib
2291 	 * resets immediately instead of waiting for the next interrupt.
2292 	 */
2293 	ice_request_stack_reinit(sc);
2294 
2295 	return;
2296 }
2297 
2298 /**
2299  * ice_rebuild - Rebuild driver state post reset
2300  * @sc: The device private softc
2301  *
2302  * Restore driver state after a reset occurred. Restart the controlqs, setup
2303  * the hardware port, and re-enable the VSIs.
2304  */
2305 static void
2306 ice_rebuild(struct ice_softc *sc)
2307 {
2308 	struct ice_hw *hw = &sc->hw;
2309 	device_t dev = sc->dev;
2310 	enum ice_status status;
2311 	int err;
2312 
2313 	sc->rebuild_ticks = ticks;
2314 
2315 	/* If we're rebuilding, then a reset has succeeded. */
2316 	ice_clear_state(&sc->state, ICE_STATE_RESET_FAILED);
2317 
2318 	/*
2319 	 * If the firmware is in recovery mode, only restore the limited
2320 	 * functionality supported by recovery mode.
2321 	 */
2322 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) {
2323 		ice_rebuild_recovery_mode(sc);
2324 		return;
2325 	}
2326 
2327 	/* enable PCIe bus master */
2328 	pci_enable_busmaster(dev);
2329 
2330 	status = ice_init_all_ctrlq(hw);
2331 	if (status) {
2332 		device_printf(dev, "failed to re-init controlqs, err %s\n",
2333 			      ice_status_str(status));
2334 		goto err_shutdown_ctrlq;
2335 	}
2336 
2337 	/* Query the allocated resources for Tx scheduler */
2338 	status = ice_sched_query_res_alloc(hw);
2339 	if (status) {
2340 		device_printf(dev,
2341 			      "Failed to query scheduler resources, err %s aq_err %s\n",
2342 			      ice_status_str(status),
2343 			      ice_aq_str(hw->adminq.sq_last_status));
2344 		goto err_shutdown_ctrlq;
2345 	}
2346 
2347 	err = ice_send_version(sc);
2348 	if (err)
2349 		goto err_shutdown_ctrlq;
2350 
2351 	err = ice_init_link_events(sc);
2352 	if (err) {
2353 		device_printf(dev, "ice_init_link_events failed: %s\n",
2354 			      ice_err_str(err));
2355 		goto err_shutdown_ctrlq;
2356 	}
2357 
2358 	status = ice_clear_pf_cfg(hw);
2359 	if (status) {
2360 		device_printf(dev, "failed to clear PF configuration, err %s\n",
2361 			      ice_status_str(status));
2362 		goto err_shutdown_ctrlq;
2363 	}
2364 
2365 	ice_clear_pxe_mode(hw);
2366 
2367 	status = ice_get_caps(hw);
2368 	if (status) {
2369 		device_printf(dev, "failed to get capabilities, err %s\n",
2370 			      ice_status_str(status));
2371 		goto err_shutdown_ctrlq;
2372 	}
2373 
2374 	status = ice_sched_init_port(hw->port_info);
2375 	if (status) {
2376 		device_printf(dev, "failed to initialize port, err %s\n",
2377 			      ice_status_str(status));
2378 		goto err_sched_cleanup;
2379 	}
2380 
2381 	/* If we previously loaded the package, it needs to be reloaded now */
2382 	if (!ice_is_bit_set(sc->feat_en, ICE_FEATURE_SAFE_MODE)) {
2383 		status = ice_init_pkg(hw, hw->pkg_copy, hw->pkg_size);
2384 		if (status) {
2385 			ice_log_pkg_init(sc, &status);
2386 
2387 			ice_transition_safe_mode(sc);
2388 		}
2389 	}
2390 
2391 	ice_reset_pf_stats(sc);
2392 
2393 	err = ice_rebuild_pf_vsi_qmap(sc);
2394 	if (err) {
2395 		device_printf(sc->dev, "Unable to re-assign main VSI queues, err %s\n",
2396 			      ice_err_str(err));
2397 		goto err_sched_cleanup;
2398 	}
2399 	err = ice_initialize_vsi(&sc->pf_vsi);
2400 	if (err) {
2401 		device_printf(sc->dev, "Unable to re-initialize Main VSI, err %s\n",
2402 			      ice_err_str(err));
2403 		goto err_release_queue_allocations;
2404 	}
2405 
2406 	/* Replay all VSI configuration */
2407 	err = ice_replay_all_vsi_cfg(sc);
2408 	if (err)
2409 		goto err_deinit_pf_vsi;
2410 
2411 	/* Reconfigure the main PF VSI for RSS */
2412 	err = ice_config_rss(&sc->pf_vsi);
2413 	if (err) {
2414 		device_printf(sc->dev,
2415 			      "Unable to reconfigure RSS for the main VSI, err %s\n",
2416 			      ice_err_str(err));
2417 		goto err_deinit_pf_vsi;
2418 	}
2419 
2420 	/* Refresh link status */
2421 	ice_clear_state(&sc->state, ICE_STATE_LINK_STATUS_REPORTED);
2422 	sc->hw.port_info->phy.get_link_info = true;
2423 	ice_get_link_status(sc->hw.port_info, &sc->link_up);
2424 	ice_update_link_status(sc, true);
2425 
2426 	/* Configure interrupt causes for the administrative interrupt */
2427 	ice_configure_misc_interrupts(sc);
2428 
2429 	/* Enable ITR 0 right away, so that we can handle admin interrupts */
2430 	ice_enable_intr(&sc->hw, sc->irqvs[0].me);
2431 
2432 	/* Now that the rebuild is finished, we're no longer prepared to reset */
2433 	ice_clear_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET);
2434 
2435 	log(LOG_INFO, "%s: device rebuild successful\n", sc->ifp->if_xname);
2436 
2437 	/* In order to completely restore device functionality, the iflib core
2438 	 * needs to be reset. We need to request an iflib reset. Additionally,
2439 	 * because the state of IFC_DO_RESET is cached within task_fn_admin in
2440 	 * the iflib core, we also want re-run the admin task so that iflib
2441 	 * resets immediately instead of waiting for the next interrupt.
2442 	 */
2443 	ice_request_stack_reinit(sc);
2444 
2445 	return;
2446 
2447 err_deinit_pf_vsi:
2448 	ice_deinit_vsi(&sc->pf_vsi);
2449 err_release_queue_allocations:
2450 	ice_resmgr_release_map(&sc->tx_qmgr, sc->pf_vsi.tx_qmap,
2451 				    sc->pf_vsi.num_tx_queues);
2452 	ice_resmgr_release_map(&sc->rx_qmgr, sc->pf_vsi.rx_qmap,
2453 				    sc->pf_vsi.num_rx_queues);
2454 err_sched_cleanup:
2455 	ice_sched_cleanup_all(hw);
2456 err_shutdown_ctrlq:
2457 	ice_shutdown_all_ctrlq(hw);
2458 	ice_set_state(&sc->state, ICE_STATE_RESET_FAILED);
2459 	device_printf(dev, "Driver rebuild failed, please reload the device driver\n");
2460 }
2461 
2462 /**
2463  * ice_handle_reset_event - Handle reset events triggered by OICR
2464  * @sc: The device private softc
2465  *
2466  * Handle reset events triggered by an OICR notification. This includes CORER,
2467  * GLOBR, and EMPR resets triggered by software on this or any other PF or by
2468  * firmware.
2469  *
2470  * @pre assumes the iflib context lock is held, and will unlock it while
2471  * waiting for the hardware to finish reset.
2472  */
2473 static void
2474 ice_handle_reset_event(struct ice_softc *sc)
2475 {
2476 	struct ice_hw *hw = &sc->hw;
2477 	enum ice_status status;
2478 	device_t dev = sc->dev;
2479 
2480 	/* When a CORER, GLOBR, or EMPR is about to happen, the hardware will
2481 	 * trigger an OICR interrupt. Our OICR handler will determine when
2482 	 * this occurs and set the ICE_STATE_RESET_OICR_RECV bit as
2483 	 * appropriate.
2484 	 */
2485 	if (!ice_testandclear_state(&sc->state, ICE_STATE_RESET_OICR_RECV))
2486 		return;
2487 
2488 	ice_prepare_for_reset(sc);
2489 
2490 	/*
2491 	 * Release the iflib context lock and wait for the device to finish
2492 	 * resetting.
2493 	 */
2494 	IFLIB_CTX_UNLOCK(sc);
2495 	status = ice_check_reset(hw);
2496 	IFLIB_CTX_LOCK(sc);
2497 	if (status) {
2498 		device_printf(dev, "Device never came out of reset, err %s\n",
2499 			      ice_status_str(status));
2500 		ice_set_state(&sc->state, ICE_STATE_RESET_FAILED);
2501 		return;
2502 	}
2503 
2504 	/* We're done with the reset, so we can rebuild driver state */
2505 	sc->hw.reset_ongoing = false;
2506 	ice_rebuild(sc);
2507 
2508 	/* In the unlikely event that a PF reset request occurs at the same
2509 	 * time as a global reset, clear the request now. This avoids
2510 	 * resetting a second time right after we reset due to a global event.
2511 	 */
2512 	if (ice_testandclear_state(&sc->state, ICE_STATE_RESET_PFR_REQ))
2513 		device_printf(dev, "Ignoring PFR request that occurred while a reset was ongoing\n");
2514 }
2515 
2516 /**
2517  * ice_handle_pf_reset_request - Initiate PF reset requested by software
2518  * @sc: The device private softc
2519  *
2520  * Initiate a PF reset requested by software. We handle this in the admin task
2521  * so that only one thread actually handles driver preparation and cleanup,
2522  * rather than having multiple threads possibly attempt to run this code
2523  * simultaneously.
2524  *
2525  * @pre assumes the iflib context lock is held and will unlock it while
2526  * waiting for the PF reset to complete.
2527  */
2528 static void
2529 ice_handle_pf_reset_request(struct ice_softc *sc)
2530 {
2531 	struct ice_hw *hw = &sc->hw;
2532 	enum ice_status status;
2533 
2534 	/* Check for PF reset requests */
2535 	if (!ice_testandclear_state(&sc->state, ICE_STATE_RESET_PFR_REQ))
2536 		return;
2537 
2538 	/* Make sure we're prepared for reset */
2539 	ice_prepare_for_reset(sc);
2540 
2541 	/*
2542 	 * Release the iflib context lock and wait for the device to finish
2543 	 * resetting.
2544 	 */
2545 	IFLIB_CTX_UNLOCK(sc);
2546 	status = ice_reset(hw, ICE_RESET_PFR);
2547 	IFLIB_CTX_LOCK(sc);
2548 	if (status) {
2549 		device_printf(sc->dev, "device PF reset failed, err %s\n",
2550 			      ice_status_str(status));
2551 		ice_set_state(&sc->state, ICE_STATE_RESET_FAILED);
2552 		return;
2553 	}
2554 
2555 	sc->soft_stats.pfr_count++;
2556 	ice_rebuild(sc);
2557 }
2558 
2559 /**
2560  * ice_init_device_features - Init device driver features
2561  * @sc: driver softc structure
2562  *
2563  * @pre assumes that the function capabilities bits have been set up by
2564  * ice_init_hw().
2565  */
2566 static void
2567 ice_init_device_features(struct ice_softc *sc)
2568 {
2569 	/*
2570 	 * A failed pkg file download triggers safe mode, disabling advanced
2571 	 * device feature support
2572 	 */
2573 	if (ice_is_bit_set(sc->feat_en, ICE_FEATURE_SAFE_MODE))
2574 		return;
2575 
2576 	/* Set capabilities that the driver supports */
2577 	ice_set_bit(ICE_FEATURE_SRIOV, sc->feat_cap);
2578 	ice_set_bit(ICE_FEATURE_RSS, sc->feat_cap);
2579 	ice_set_bit(ICE_FEATURE_LENIENT_LINK_MODE, sc->feat_cap);
2580 	ice_set_bit(ICE_FEATURE_DEFAULT_OVERRIDE, sc->feat_cap);
2581 
2582 	/* Disable features due to hardware limitations... */
2583 	if (!sc->hw.func_caps.common_cap.rss_table_size)
2584 		ice_clear_bit(ICE_FEATURE_RSS, sc->feat_cap);
2585 
2586 	/* Disable capabilities not supported by the OS */
2587 	ice_disable_unsupported_features(sc->feat_cap);
2588 
2589 	/* RSS is always enabled for iflib */
2590 	if (ice_is_bit_set(sc->feat_cap, ICE_FEATURE_RSS))
2591 		ice_set_bit(ICE_FEATURE_RSS, sc->feat_en);
2592 }
2593 
2594 /**
2595  * ice_if_multi_set - Callback to update Multicast filters in HW
2596  * @ctx: iflib ctx structure
2597  *
2598  * Called by iflib in response to SIOCDELMULTI and SIOCADDMULTI. Must search
2599  * the if_multiaddrs list and determine which filters have been added or
2600  * removed from the list, and update HW programming to reflect the new list.
2601  *
2602  * @pre assumes the caller holds the iflib CTX lock
2603  */
2604 static void
2605 ice_if_multi_set(if_ctx_t ctx)
2606 {
2607 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
2608 	int err;
2609 
2610 	ASSERT_CTX_LOCKED(sc);
2611 
2612 	/* Do not handle multicast configuration in recovery mode */
2613 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
2614 		return;
2615 
2616 	err = ice_sync_multicast_filters(sc);
2617 	if (err) {
2618 		device_printf(sc->dev,
2619 			      "Failed to synchronize multicast filter list: %s\n",
2620 			      ice_err_str(err));
2621 		return;
2622 	}
2623 }
2624 
2625 /**
2626  * ice_if_vlan_register - Register a VLAN with the hardware
2627  * @ctx: iflib ctx pointer
2628  * @vtag: VLAN to add
2629  *
2630  * Programs the main PF VSI with a hardware filter for the given VLAN.
2631  *
2632  * @pre assumes the caller holds the iflib CTX lock
2633  */
2634 static void
2635 ice_if_vlan_register(if_ctx_t ctx, u16 vtag)
2636 {
2637 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
2638 	enum ice_status status;
2639 
2640 	ASSERT_CTX_LOCKED(sc);
2641 
2642 	/* Do not handle VLAN configuration in recovery mode */
2643 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
2644 		return;
2645 
2646 	status = ice_add_vlan_hw_filter(&sc->pf_vsi, vtag);
2647 	if (status) {
2648 		device_printf(sc->dev,
2649 			      "Failure adding VLAN %d to main VSI, err %s aq_err %s\n",
2650 			      vtag, ice_status_str(status),
2651 			      ice_aq_str(sc->hw.adminq.sq_last_status));
2652 	}
2653 }
2654 
2655 /**
2656  * ice_if_vlan_unregister - Remove a VLAN filter from the hardware
2657  * @ctx: iflib ctx pointer
2658  * @vtag: VLAN to add
2659  *
2660  * Removes the previously programmed VLAN filter from the main PF VSI.
2661  *
2662  * @pre assumes the caller holds the iflib CTX lock
2663  */
2664 static void
2665 ice_if_vlan_unregister(if_ctx_t ctx, u16 vtag)
2666 {
2667 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
2668 	enum ice_status status;
2669 
2670 	ASSERT_CTX_LOCKED(sc);
2671 
2672 	/* Do not handle VLAN configuration in recovery mode */
2673 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
2674 		return;
2675 
2676 	status = ice_remove_vlan_hw_filter(&sc->pf_vsi, vtag);
2677 	if (status) {
2678 		device_printf(sc->dev,
2679 			      "Failure removing VLAN %d from main VSI, err %s aq_err %s\n",
2680 			      vtag, ice_status_str(status),
2681 			      ice_aq_str(sc->hw.adminq.sq_last_status));
2682 	}
2683 }
2684 
2685 /**
2686  * ice_if_stop - Stop the device
2687  * @ctx: iflib context structure
2688  *
2689  * Called by iflib to stop the device and bring it down. (i.e. ifconfig ice0
2690  * down)
2691  *
2692  * @pre assumes the caller holds the iflib CTX lock
2693  */
2694 static void
2695 ice_if_stop(if_ctx_t ctx)
2696 {
2697 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
2698 
2699 	ASSERT_CTX_LOCKED(sc);
2700 
2701 	/*
2702 	 * The iflib core may call IFDI_STOP prior to the first call to
2703 	 * IFDI_INIT. This will cause us to attempt to remove MAC filters we
2704 	 * don't have, and disable Tx queues which aren't yet configured.
2705 	 * Although it is likely these extra operations are harmless, they do
2706 	 * cause spurious warning messages to be displayed, which may confuse
2707 	 * users.
2708 	 *
2709 	 * To avoid these messages, we use a state bit indicating if we've
2710 	 * been initialized. It will be set when ice_if_init is called, and
2711 	 * cleared here in ice_if_stop.
2712 	 */
2713 	if (!ice_testandclear_state(&sc->state, ICE_STATE_DRIVER_INITIALIZED))
2714 		return;
2715 
2716 	if (ice_test_state(&sc->state, ICE_STATE_RESET_FAILED)) {
2717 		device_printf(sc->dev, "request to stop interface cannot be completed as the device failed to reset\n");
2718 		return;
2719 	}
2720 
2721 	if (ice_test_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET)) {
2722 		device_printf(sc->dev, "request to stop interface while device is prepared for impending reset\n");
2723 		return;
2724 	}
2725 
2726 	/* Remove the MAC filters, stop Tx, and stop Rx. We don't check the
2727 	 * return of these functions because there's nothing we can really do
2728 	 * if they fail, and the functions already print error messages.
2729 	 * Just try to shut down as much as we can.
2730 	 */
2731 	ice_rm_pf_default_mac_filters(sc);
2732 
2733 	/* Dissociate the Tx and Rx queues from the interrupts */
2734 	ice_flush_txq_interrupts(&sc->pf_vsi);
2735 	ice_flush_rxq_interrupts(&sc->pf_vsi);
2736 
2737 	/* Disable the Tx and Rx queues */
2738 	ice_vsi_disable_tx(&sc->pf_vsi);
2739 	ice_control_rx_queues(&sc->pf_vsi, false);
2740 }
2741 
2742 /**
2743  * ice_if_get_counter - Get current value of an ifnet statistic
2744  * @ctx: iflib context pointer
2745  * @counter: ifnet counter to read
2746  *
2747  * Reads the current value of an ifnet counter for the device.
2748  *
2749  * This function is not protected by the iflib CTX lock.
2750  */
2751 static uint64_t
2752 ice_if_get_counter(if_ctx_t ctx, ift_counter counter)
2753 {
2754 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
2755 
2756 	/* Return the counter for the main PF VSI */
2757 	return ice_get_ifnet_counter(&sc->pf_vsi, counter);
2758 }
2759 
2760 /**
2761  * ice_request_stack_reinit - Request that iflib re-initialize
2762  * @sc: the device private softc
2763  *
2764  * Request that the device be brought down and up, to re-initialize. For
2765  * example, this may be called when a device reset occurs, or when Tx and Rx
2766  * queues need to be re-initialized.
2767  *
2768  * This is required because the iflib state is outside the driver, and must be
2769  * re-initialized if we need to resart Tx and Rx queues.
2770  */
2771 void
2772 ice_request_stack_reinit(struct ice_softc *sc)
2773 {
2774 	if (CTX_ACTIVE(sc->ctx)) {
2775 		iflib_request_reset(sc->ctx);
2776 		iflib_admin_intr_deferred(sc->ctx);
2777 	}
2778 }
2779 
2780 /**
2781  * ice_driver_is_detaching - Check if the driver is detaching/unloading
2782  * @sc: device private softc
2783  *
2784  * Returns true if the driver is detaching, false otherwise.
2785  *
2786  * @remark on newer kernels, take advantage of iflib_in_detach in order to
2787  * report detachment correctly as early as possible.
2788  *
2789  * @remark this function is used by various code paths that want to avoid
2790  * running if the driver is about to be removed. This includes sysctls and
2791  * other driver access points. Note that it does not fully resolve
2792  * detach-based race conditions as it is possible for a thread to race with
2793  * iflib_in_detach.
2794  */
2795 bool
2796 ice_driver_is_detaching(struct ice_softc *sc)
2797 {
2798 	return (ice_test_state(&sc->state, ICE_STATE_DETACHING) ||
2799 		iflib_in_detach(sc->ctx));
2800 }
2801 
2802 /**
2803  * ice_if_priv_ioctl - Device private ioctl handler
2804  * @ctx: iflib context pointer
2805  * @command: The ioctl command issued
2806  * @data: ioctl specific data
2807  *
2808  * iflib callback for handling custom driver specific ioctls.
2809  *
2810  * @pre Assumes that the iflib context lock is held.
2811  */
2812 static int
2813 ice_if_priv_ioctl(if_ctx_t ctx, u_long command, caddr_t data)
2814 {
2815 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
2816 	struct ifdrv *ifd;
2817 	device_t dev = sc->dev;
2818 
2819 	if (data == NULL)
2820 		return (EINVAL);
2821 
2822 	ASSERT_CTX_LOCKED(sc);
2823 
2824 	/* Make sure the command type is valid */
2825 	switch (command) {
2826 	case SIOCSDRVSPEC:
2827 	case SIOCGDRVSPEC:
2828 		/* Accepted commands */
2829 		break;
2830 	case SIOCGPRIVATE_0:
2831 		/*
2832 		 * Although we do not support this ioctl command, it's
2833 		 * expected that iflib will forward it to the IFDI_PRIV_IOCTL
2834 		 * handler. Do not print a message in this case
2835 		 */
2836 		return (ENOTSUP);
2837 	default:
2838 		/*
2839 		 * If we get a different command for this function, it's
2840 		 * definitely unexpected, so log a message indicating what
2841 		 * command we got for debugging purposes.
2842 		 */
2843 		device_printf(dev, "%s: unexpected ioctl command %08lx\n",
2844 			      __func__, command);
2845 		return (EINVAL);
2846 	}
2847 
2848 	ifd = (struct ifdrv *)data;
2849 
2850 	switch (ifd->ifd_cmd) {
2851 	case ICE_NVM_ACCESS:
2852 		return ice_handle_nvm_access_ioctl(sc, ifd);
2853 	default:
2854 		return EINVAL;
2855 	}
2856 }
2857 
2858 /**
2859  * ice_if_i2c_req - I2C request handler for iflib
2860  * @ctx: iflib context pointer
2861  * @req: The I2C parameters to use
2862  *
2863  * Read from the port's I2C eeprom using the parameters from the ioctl.
2864  *
2865  * @remark The iflib-only part is pretty simple.
2866  */
2867 static int
2868 ice_if_i2c_req(if_ctx_t ctx, struct ifi2creq *req)
2869 {
2870 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
2871 
2872 	return ice_handle_i2c_req(sc, req);
2873 }
2874 
2875