1 /******************************************************************************
2
3 Copyright (c) 2013-2020, Intel Corporation
4 All rights reserved.
5
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
8
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
11
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
15
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
19
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
31
32 ******************************************************************************/
33
34 #include "ixl_pf.h"
35
36 void
ixl_configure_tx_itr(struct ixl_pf * pf)37 ixl_configure_tx_itr(struct ixl_pf *pf)
38 {
39 struct i40e_hw *hw = &pf->hw;
40 struct ixl_vsi *vsi = &pf->vsi;
41 struct ixl_tx_queue *que = vsi->tx_queues;
42
43 vsi->tx_itr_setting = pf->tx_itr;
44
45 for (int i = 0; i < vsi->num_tx_queues; i++, que++) {
46 struct tx_ring *txr = &que->txr;
47
48 wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, i),
49 vsi->tx_itr_setting);
50 txr->itr = vsi->tx_itr_setting;
51 txr->latency = IXL_AVE_LATENCY;
52 }
53 }
54
55 void
ixl_configure_rx_itr(struct ixl_pf * pf)56 ixl_configure_rx_itr(struct ixl_pf *pf)
57 {
58 struct i40e_hw *hw = &pf->hw;
59 struct ixl_vsi *vsi = &pf->vsi;
60 struct ixl_rx_queue *que = vsi->rx_queues;
61
62 vsi->rx_itr_setting = pf->rx_itr;
63
64 for (int i = 0; i < vsi->num_rx_queues; i++, que++) {
65 struct rx_ring *rxr = &que->rxr;
66
67 wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, i),
68 vsi->rx_itr_setting);
69 rxr->itr = vsi->rx_itr_setting;
70 rxr->latency = IXL_AVE_LATENCY;
71 }
72 }
73
74 int
ixl_intr(void * arg)75 ixl_intr(void *arg)
76 {
77 struct ixl_pf *pf = arg;
78 struct i40e_hw *hw = &pf->hw;
79 struct ixl_vsi *vsi = &pf->vsi;
80 struct ixl_rx_queue *que = vsi->rx_queues;
81 u32 icr0;
82
83 ++que->irqs;
84
85 /* Clear PBA at start of ISR if using legacy interrupts */
86 if (vsi->shared->isc_intr == IFLIB_INTR_LEGACY)
87 wr32(hw, I40E_PFINT_DYN_CTL0,
88 I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
89 (IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT));
90
91 icr0 = rd32(hw, I40E_PFINT_ICR0);
92
93
94 #ifdef PCI_IOV
95 if (icr0 & I40E_PFINT_ICR0_VFLR_MASK)
96 iflib_iov_intr_deferred(vsi->ctx);
97 #endif
98
99 if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK)
100 iflib_admin_intr_deferred(vsi->ctx);
101
102 ixl_enable_intr0(hw);
103
104 if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK)
105 return (FILTER_SCHEDULE_THREAD);
106 else
107 return (FILTER_HANDLED);
108 }
109
110 /*********************************************************************
111 *
112 * MSI-X VSI Interrupt Service routine
113 *
114 **********************************************************************/
115 int
ixl_msix_que(void * arg)116 ixl_msix_que(void *arg)
117 {
118 struct ixl_rx_queue *rx_que = arg;
119
120 ++rx_que->irqs;
121
122 ixl_set_queue_rx_itr(rx_que);
123
124 return (FILTER_SCHEDULE_THREAD);
125 }
126
127 /*********************************************************************
128 *
129 * MSI-X Admin Queue Interrupt Service routine
130 *
131 **********************************************************************/
132 int
ixl_msix_adminq(void * arg)133 ixl_msix_adminq(void *arg)
134 {
135 struct ixl_pf *pf = arg;
136 struct i40e_hw *hw = &pf->hw;
137 device_t dev = pf->dev;
138 u32 reg, mask, rstat_reg;
139 bool do_task = FALSE;
140
141 DDPRINTF(dev, "begin");
142
143 ++pf->admin_irq;
144
145 reg = rd32(hw, I40E_PFINT_ICR0);
146 /*
147 * For masking off interrupt causes that need to be handled before
148 * they can be re-enabled
149 */
150 mask = rd32(hw, I40E_PFINT_ICR0_ENA);
151
152 /* Check on the cause */
153 if (reg & I40E_PFINT_ICR0_ADMINQ_MASK) {
154 mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
155 do_task = TRUE;
156 }
157
158 if (reg & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
159 mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
160 ixl_set_state(&pf->state, IXL_STATE_MDD_PENDING);
161 do_task = TRUE;
162 }
163
164 if (reg & I40E_PFINT_ICR0_GRST_MASK) {
165 const char *reset_type;
166 mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK;
167 rstat_reg = rd32(hw, I40E_GLGEN_RSTAT);
168 rstat_reg = (rstat_reg & I40E_GLGEN_RSTAT_RESET_TYPE_MASK)
169 >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT;
170 switch (rstat_reg) {
171 /* These others might be handled similarly to an EMPR reset */
172 case I40E_RESET_CORER:
173 reset_type = "CORER";
174 break;
175 case I40E_RESET_GLOBR:
176 reset_type = "GLOBR";
177 break;
178 case I40E_RESET_EMPR:
179 reset_type = "EMPR";
180 break;
181 default:
182 reset_type = "POR";
183 break;
184 }
185 device_printf(dev, "Reset Requested! (%s)\n", reset_type);
186 /* overload admin queue task to check reset progress */
187 ixl_set_state(&pf->state, IXL_STATE_RESETTING);
188 do_task = TRUE;
189 }
190
191 /*
192 * PE / PCI / ECC exceptions are all handled in the same way:
193 * mask out these three causes, then request a PF reset
194 */
195 if (reg & I40E_PFINT_ICR0_ECC_ERR_MASK)
196 device_printf(dev, "ECC Error detected!\n");
197 if (reg & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK)
198 device_printf(dev, "PCI Exception detected!\n");
199 if (reg & I40E_PFINT_ICR0_PE_CRITERR_MASK)
200 device_printf(dev, "Critical Protocol Engine Error detected!\n");
201 /* Checks against the conditions above */
202 if (reg & IXL_ICR0_CRIT_ERR_MASK) {
203 mask &= ~IXL_ICR0_CRIT_ERR_MASK;
204 ixl_set_state(&pf->state,
205 IXL_STATE_PF_RESET_REQ | IXL_STATE_PF_CRIT_ERR);
206 do_task = TRUE;
207 }
208
209 if (reg & I40E_PFINT_ICR0_HMC_ERR_MASK) {
210 reg = rd32(hw, I40E_PFHMC_ERRORINFO);
211 if (reg & I40E_PFHMC_ERRORINFO_ERROR_DETECTED_MASK) {
212 device_printf(dev, "HMC Error detected!\n");
213 device_printf(dev, "INFO 0x%08x\n", reg);
214 reg = rd32(hw, I40E_PFHMC_ERRORDATA);
215 device_printf(dev, "DATA 0x%08x\n", reg);
216 wr32(hw, I40E_PFHMC_ERRORINFO, 0);
217 }
218 }
219
220 #ifdef PCI_IOV
221 if (reg & I40E_PFINT_ICR0_VFLR_MASK) {
222 mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
223 iflib_iov_intr_deferred(pf->vsi.ctx);
224 }
225 #endif
226
227 wr32(hw, I40E_PFINT_ICR0_ENA, mask);
228 ixl_enable_intr0(hw);
229
230 if (do_task)
231 return (FILTER_SCHEDULE_THREAD);
232 else
233 return (FILTER_HANDLED);
234 }
235
236 /*
237 * Configure queue interrupt cause registers in hardware.
238 *
239 * Linked list for each vector LNKLSTN(i) -> RQCTL(i) -> TQCTL(i) -> EOL
240 */
241 void
ixl_configure_queue_intr_msix(struct ixl_pf * pf)242 ixl_configure_queue_intr_msix(struct ixl_pf *pf)
243 {
244 struct i40e_hw *hw = &pf->hw;
245 struct ixl_vsi *vsi = &pf->vsi;
246 u32 reg;
247 u16 vector = 1;
248
249 for (int i = 0; i < max(vsi->num_rx_queues, vsi->num_tx_queues); i++, vector++) {
250 /* Make sure interrupt is disabled */
251 wr32(hw, I40E_PFINT_DYN_CTLN(i), 0);
252 /* Set linked list head to point to corresponding RX queue
253 * e.g. vector 1 (LNKLSTN register 0) points to queue pair 0's RX queue */
254 reg = ((i << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT)
255 & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK) |
256 ((I40E_QUEUE_TYPE_RX << I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)
257 & I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_MASK);
258 wr32(hw, I40E_PFINT_LNKLSTN(i), reg);
259
260 reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
261 (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
262 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
263 (i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
264 (I40E_QUEUE_TYPE_TX << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
265 wr32(hw, I40E_QINT_RQCTL(i), reg);
266
267 reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
268 (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
269 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
270 (IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
271 (I40E_QUEUE_TYPE_RX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
272 wr32(hw, I40E_QINT_TQCTL(i), reg);
273 }
274 }
275
276 /*
277 * Configure for single interrupt vector operation
278 */
279 void
ixl_configure_legacy(struct ixl_pf * pf)280 ixl_configure_legacy(struct ixl_pf *pf)
281 {
282 struct i40e_hw *hw = &pf->hw;
283 struct ixl_vsi *vsi = &pf->vsi;
284 u32 reg;
285
286 vsi->rx_queues[0].rxr.itr = vsi->rx_itr_setting;
287
288 /* Setup "other" causes */
289 reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK
290 | I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK
291 | I40E_PFINT_ICR0_ENA_GRST_MASK
292 | I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK
293 | I40E_PFINT_ICR0_ENA_HMC_ERR_MASK
294 | I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK
295 | I40E_PFINT_ICR0_ENA_VFLR_MASK
296 | I40E_PFINT_ICR0_ENA_ADMINQ_MASK
297 ;
298 wr32(hw, I40E_PFINT_ICR0_ENA, reg);
299
300 /* No ITR for non-queue interrupts */
301 wr32(hw, I40E_PFINT_STAT_CTL0,
302 IXL_ITR_NONE << I40E_PFINT_STAT_CTL0_OTHER_ITR_INDX_SHIFT);
303
304 /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
305 wr32(hw, I40E_PFINT_LNKLST0, 0);
306
307 /* Associate the queue pair to the vector and enable the q int */
308 reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK
309 | (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT)
310 | (I40E_QUEUE_TYPE_TX << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
311 wr32(hw, I40E_QINT_RQCTL(0), reg);
312
313 reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK
314 | (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)
315 | (IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
316 wr32(hw, I40E_QINT_TQCTL(0), reg);
317 }
318
319 void
ixl_free_pci_resources(struct ixl_pf * pf)320 ixl_free_pci_resources(struct ixl_pf *pf)
321 {
322 struct ixl_vsi *vsi = &pf->vsi;
323 device_t dev = iflib_get_dev(vsi->ctx);
324 struct ixl_rx_queue *rx_que = vsi->rx_queues;
325
326 /* We may get here before stations are set up */
327 if (rx_que == NULL)
328 goto early;
329
330 /*
331 ** Release all MSI-X VSI resources:
332 */
333 iflib_irq_free(vsi->ctx, &vsi->irq);
334
335 for (int i = 0; i < vsi->num_rx_queues; i++, rx_que++)
336 iflib_irq_free(vsi->ctx, &rx_que->que_irq);
337 early:
338 if (pf->pci_mem != NULL)
339 bus_release_resource(dev, SYS_RES_MEMORY,
340 rman_get_rid(pf->pci_mem), pf->pci_mem);
341 }
342
343 /*********************************************************************
344 *
345 * Setup networking device structure and register an interface.
346 *
347 **********************************************************************/
348 int
ixl_setup_interface(device_t dev,struct ixl_pf * pf)349 ixl_setup_interface(device_t dev, struct ixl_pf *pf)
350 {
351 struct ixl_vsi *vsi = &pf->vsi;
352 if_ctx_t ctx = vsi->ctx;
353 struct i40e_hw *hw = &pf->hw;
354 if_t ifp = iflib_get_ifp(ctx);
355 struct i40e_aq_get_phy_abilities_resp abilities;
356 enum i40e_status_code aq_error = 0;
357
358 INIT_DBG_DEV(dev, "begin");
359
360 vsi->shared->isc_max_frame_size =
361 if_getmtu(ifp) + ETHER_HDR_LEN + ETHER_CRC_LEN
362 + ETHER_VLAN_ENCAP_LEN;
363
364 if (IXL_PF_IN_RECOVERY_MODE(pf))
365 goto only_auto;
366
367 aq_error = i40e_aq_get_phy_capabilities(hw,
368 FALSE, TRUE, &abilities, NULL);
369 /* May need delay to detect fiber correctly */
370 if (aq_error == I40E_ERR_UNKNOWN_PHY) {
371 i40e_msec_delay(200);
372 aq_error = i40e_aq_get_phy_capabilities(hw, FALSE,
373 TRUE, &abilities, NULL);
374 }
375 if (aq_error) {
376 if (aq_error == I40E_ERR_UNKNOWN_PHY)
377 device_printf(dev, "Unknown PHY type detected!\n");
378 else
379 device_printf(dev,
380 "Error getting supported media types, err %d,"
381 " AQ error %d\n", aq_error, hw->aq.asq_last_status);
382 } else {
383 pf->supported_speeds = abilities.link_speed;
384 if_setbaudrate(ifp, ixl_max_aq_speed_to_value(pf->supported_speeds));
385
386 ixl_add_ifmedia(vsi->media, hw->phy.phy_types);
387 }
388
389 only_auto:
390 /* Use autoselect media by default */
391 ifmedia_add(vsi->media, IFM_ETHER | IFM_AUTO, 0, NULL);
392 ifmedia_set(vsi->media, IFM_ETHER | IFM_AUTO);
393
394 return (0);
395 }
396
397 /*
398 ** Run when the Admin Queue gets a link state change interrupt.
399 */
400 void
ixl_link_event(struct ixl_pf * pf,struct i40e_arq_event_info * e)401 ixl_link_event(struct ixl_pf *pf, struct i40e_arq_event_info *e)
402 {
403 struct i40e_hw *hw = &pf->hw;
404 device_t dev = iflib_get_dev(pf->vsi.ctx);
405 struct i40e_link_status *link_info = &hw->phy.link_info;
406
407 /* Driver needs to re-enable delivering of link status events
408 * by FW after each event reception. Call i40e_get_link_status
409 * to do that. To not lose information about link state changes,
410 * which happened between receiving an event and the call,
411 * do not rely on status from event but use most recent
412 * status information retrieved by the call. */
413 hw->phy.get_link_info = TRUE;
414 i40e_get_link_status(hw, &pf->link_up);
415
416 /* Print out message if an unqualified module is found */
417 if ((link_info->link_info & I40E_AQ_MEDIA_AVAILABLE) &&
418 (pf->advertised_speed) &&
419 (if_getflags(pf->vsi.ifp) & IFF_UP) &&
420 (!(link_info->an_info & I40E_AQ_QUALIFIED_MODULE)) &&
421 (!(link_info->link_info & I40E_AQ_LINK_UP)))
422 device_printf(dev, "Link failed because "
423 "an unqualified module was detected!\n");
424
425 /* OS link info is updated elsewhere */
426 }
427
428 /*********************************************************************
429 *
430 * Initialize the VSI: this handles contexts, which means things
431 * like the number of descriptors, buffer size,
432 * plus we init the rings thru this function.
433 *
434 **********************************************************************/
435 int
ixl_initialize_vsi(struct ixl_vsi * vsi)436 ixl_initialize_vsi(struct ixl_vsi *vsi)
437 {
438 struct ixl_pf *pf = vsi->back;
439 if_softc_ctx_t scctx = iflib_get_softc_ctx(vsi->ctx);
440 struct ixl_tx_queue *tx_que = vsi->tx_queues;
441 struct ixl_rx_queue *rx_que = vsi->rx_queues;
442 device_t dev = iflib_get_dev(vsi->ctx);
443 struct i40e_hw *hw = vsi->hw;
444 struct i40e_vsi_context ctxt;
445 int tc_queues;
446 int err = 0;
447
448 memset(&ctxt, 0, sizeof(ctxt));
449 ctxt.seid = vsi->seid;
450 if (pf->veb_seid != 0)
451 ctxt.uplink_seid = pf->veb_seid;
452 ctxt.pf_num = hw->pf_id;
453 err = i40e_aq_get_vsi_params(hw, &ctxt, NULL);
454 if (err) {
455 device_printf(dev, "i40e_aq_get_vsi_params() failed, error %d"
456 " aq_error %d\n", err, hw->aq.asq_last_status);
457 return (err);
458 }
459 ixl_dbg(pf, IXL_DBG_SWITCH_INFO,
460 "get_vsi_params: seid: %d, uplinkseid: %d, vsi_number: %d, "
461 "vsis_allocated: %d, vsis_unallocated: %d, flags: 0x%x, "
462 "pfnum: %d, vfnum: %d, stat idx: %d, enabled: %d\n", ctxt.seid,
463 ctxt.uplink_seid, ctxt.vsi_number,
464 ctxt.vsis_allocated, ctxt.vsis_unallocated,
465 ctxt.flags, ctxt.pf_num, ctxt.vf_num,
466 ctxt.info.stat_counter_idx, ctxt.info.up_enable_bits);
467 /*
468 ** Set the queue and traffic class bits
469 ** - when multiple traffic classes are supported
470 ** this will need to be more robust.
471 */
472 ctxt.info.valid_sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
473 ctxt.info.mapping_flags |= I40E_AQ_VSI_QUE_MAP_CONTIG;
474 /* In contig mode, que_mapping[0] is first queue index used by this VSI */
475 ctxt.info.queue_mapping[0] = 0;
476 /*
477 * This VSI will only use traffic class 0; start traffic class 0's
478 * queue allocation at queue 0, and assign it 2^tc_queues queues (though
479 * the driver may not use all of them).
480 */
481 tc_queues = fls(pf->qtag.num_allocated) - 1;
482 ctxt.info.tc_mapping[0] = ((pf->qtag.first_qidx << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT)
483 & I40E_AQ_VSI_TC_QUE_OFFSET_MASK) |
484 ((tc_queues << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT)
485 & I40E_AQ_VSI_TC_QUE_NUMBER_MASK);
486
487 /* Set VLAN receive stripping mode */
488 ctxt.info.valid_sections |= I40E_AQ_VSI_PROP_VLAN_VALID;
489 ctxt.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL;
490 if (if_getcapenable(vsi->ifp) & IFCAP_VLAN_HWTAGGING)
491 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
492 else
493 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
494
495 #ifdef IXL_IW
496 /* Set TCP Enable for iWARP capable VSI */
497 if (ixl_enable_iwarp && pf->iw_enabled) {
498 ctxt.info.valid_sections |=
499 htole16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
500 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA;
501 }
502 #endif
503 /* Save VSI number and info for use later */
504 vsi->vsi_num = ctxt.vsi_number;
505 bcopy(&ctxt.info, &vsi->info, sizeof(vsi->info));
506
507 ctxt.flags = htole16(I40E_AQ_VSI_TYPE_PF);
508
509 err = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
510 if (err) {
511 device_printf(dev, "i40e_aq_update_vsi_params() failed, error %d,"
512 " aq_error %d\n", err, hw->aq.asq_last_status);
513 return (err);
514 }
515
516 for (int i = 0; i < vsi->num_tx_queues; i++, tx_que++) {
517 struct tx_ring *txr = &tx_que->txr;
518 struct i40e_hmc_obj_txq tctx;
519 u32 txctl;
520
521 /* Setup the HMC TX Context */
522 bzero(&tctx, sizeof(tctx));
523 tctx.new_context = 1;
524 tctx.base = (txr->tx_paddr/IXL_TX_CTX_BASE_UNITS);
525 tctx.qlen = scctx->isc_ntxd[0];
526 tctx.fc_ena = 0; /* Disable FCoE */
527 /*
528 * This value needs to pulled from the VSI that this queue
529 * is assigned to. Index into array is traffic class.
530 */
531 tctx.rdylist = vsi->info.qs_handle[0];
532 /*
533 * Set these to enable Head Writeback
534 * - Address is last entry in TX ring (reserved for HWB index)
535 * Leave these as 0 for Descriptor Writeback
536 */
537 if (vsi->enable_head_writeback) {
538 tctx.head_wb_ena = 1;
539 tctx.head_wb_addr = txr->tx_paddr +
540 (scctx->isc_ntxd[0] * sizeof(struct i40e_tx_desc));
541 } else {
542 tctx.head_wb_ena = 0;
543 tctx.head_wb_addr = 0;
544 }
545 tctx.rdylist_act = 0;
546 err = i40e_clear_lan_tx_queue_context(hw, i);
547 if (err) {
548 device_printf(dev, "Unable to clear TX context\n");
549 break;
550 }
551 err = i40e_set_lan_tx_queue_context(hw, i, &tctx);
552 if (err) {
553 device_printf(dev, "Unable to set TX context\n");
554 break;
555 }
556 /* Associate the ring with this PF */
557 txctl = I40E_QTX_CTL_PF_QUEUE;
558 txctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
559 I40E_QTX_CTL_PF_INDX_MASK);
560 wr32(hw, I40E_QTX_CTL(i), txctl);
561 ixl_flush(hw);
562
563 /* Do ring (re)init */
564 ixl_init_tx_ring(vsi, tx_que);
565 }
566 for (int i = 0; i < vsi->num_rx_queues; i++, rx_que++) {
567 struct rx_ring *rxr = &rx_que->rxr;
568 struct i40e_hmc_obj_rxq rctx;
569
570 /* Next setup the HMC RX Context */
571 rxr->mbuf_sz = iflib_get_rx_mbuf_sz(vsi->ctx);
572
573 u16 max_rxmax = rxr->mbuf_sz * hw->func_caps.rx_buf_chain_len;
574
575 /* Set up an RX context for the HMC */
576 memset(&rctx, 0, sizeof(struct i40e_hmc_obj_rxq));
577 rctx.dbuff = rxr->mbuf_sz >> I40E_RXQ_CTX_DBUFF_SHIFT;
578 /* ignore header split for now */
579 rctx.hbuff = 0 >> I40E_RXQ_CTX_HBUFF_SHIFT;
580 rctx.rxmax = (scctx->isc_max_frame_size < max_rxmax) ?
581 scctx->isc_max_frame_size : max_rxmax;
582 rctx.dtype = 0;
583 rctx.dsize = 1; /* do 32byte descriptors */
584 rctx.hsplit_0 = 0; /* no header split */
585 rctx.base = (rxr->rx_paddr/IXL_RX_CTX_BASE_UNITS);
586 rctx.qlen = scctx->isc_nrxd[0];
587 rctx.tphrdesc_ena = 1;
588 rctx.tphwdesc_ena = 1;
589 rctx.tphdata_ena = 0; /* Header Split related */
590 rctx.tphhead_ena = 0; /* Header Split related */
591 rctx.lrxqthresh = 1; /* Interrupt at <64 desc avail */
592 rctx.crcstrip = 1;
593 rctx.l2tsel = 1;
594 rctx.showiv = 1; /* Strip inner VLAN header */
595 rctx.fc_ena = 0; /* Disable FCoE */
596 rctx.prefena = 1; /* Prefetch descriptors */
597
598 err = i40e_clear_lan_rx_queue_context(hw, i);
599 if (err) {
600 device_printf(dev,
601 "Unable to clear RX context %d\n", i);
602 break;
603 }
604 err = i40e_set_lan_rx_queue_context(hw, i, &rctx);
605 if (err) {
606 device_printf(dev, "Unable to set RX context %d\n", i);
607 break;
608 }
609 wr32(vsi->hw, I40E_QRX_TAIL(i), 0);
610 }
611 return (err);
612 }
613
614
615 /*
616 ** Provide a update to the queue RX
617 ** interrupt moderation value.
618 */
619 void
ixl_set_queue_rx_itr(struct ixl_rx_queue * que)620 ixl_set_queue_rx_itr(struct ixl_rx_queue *que)
621 {
622 struct ixl_vsi *vsi = que->vsi;
623 struct ixl_pf *pf = (struct ixl_pf *)vsi->back;
624 struct i40e_hw *hw = vsi->hw;
625 struct rx_ring *rxr = &que->rxr;
626 u16 rx_itr;
627 u16 rx_latency = 0;
628 int rx_bytes;
629
630 /* Idle, do nothing */
631 if (rxr->bytes == 0)
632 return;
633
634 if (pf->dynamic_rx_itr) {
635 rx_bytes = rxr->bytes/rxr->itr;
636 rx_itr = rxr->itr;
637
638 /* Adjust latency range */
639 switch (rxr->latency) {
640 case IXL_LOW_LATENCY:
641 if (rx_bytes > 10) {
642 rx_latency = IXL_AVE_LATENCY;
643 rx_itr = IXL_ITR_20K;
644 }
645 break;
646 case IXL_AVE_LATENCY:
647 if (rx_bytes > 20) {
648 rx_latency = IXL_BULK_LATENCY;
649 rx_itr = IXL_ITR_8K;
650 } else if (rx_bytes <= 10) {
651 rx_latency = IXL_LOW_LATENCY;
652 rx_itr = IXL_ITR_100K;
653 }
654 break;
655 case IXL_BULK_LATENCY:
656 if (rx_bytes <= 20) {
657 rx_latency = IXL_AVE_LATENCY;
658 rx_itr = IXL_ITR_20K;
659 }
660 break;
661 }
662
663 rxr->latency = rx_latency;
664
665 if (rx_itr != rxr->itr) {
666 /* do an exponential smoothing */
667 rx_itr = (10 * rx_itr * rxr->itr) /
668 ((9 * rx_itr) + rxr->itr);
669 rxr->itr = min(rx_itr, IXL_MAX_ITR);
670 wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR,
671 rxr->me), rxr->itr);
672 }
673 } else { /* We may have have toggled to non-dynamic */
674 if (vsi->rx_itr_setting & IXL_ITR_DYNAMIC)
675 vsi->rx_itr_setting = pf->rx_itr;
676 /* Update the hardware if needed */
677 if (rxr->itr != vsi->rx_itr_setting) {
678 rxr->itr = vsi->rx_itr_setting;
679 wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR,
680 rxr->me), rxr->itr);
681 }
682 }
683 rxr->bytes = 0;
684 rxr->packets = 0;
685 }
686
687
688 /*
689 ** Provide a update to the queue TX
690 ** interrupt moderation value.
691 */
692 void
ixl_set_queue_tx_itr(struct ixl_tx_queue * que)693 ixl_set_queue_tx_itr(struct ixl_tx_queue *que)
694 {
695 struct ixl_vsi *vsi = que->vsi;
696 struct ixl_pf *pf = (struct ixl_pf *)vsi->back;
697 struct i40e_hw *hw = vsi->hw;
698 struct tx_ring *txr = &que->txr;
699 u16 tx_itr;
700 u16 tx_latency = 0;
701 int tx_bytes;
702
703
704 /* Idle, do nothing */
705 if (txr->bytes == 0)
706 return;
707
708 if (pf->dynamic_tx_itr) {
709 tx_bytes = txr->bytes/txr->itr;
710 tx_itr = txr->itr;
711
712 switch (txr->latency) {
713 case IXL_LOW_LATENCY:
714 if (tx_bytes > 10) {
715 tx_latency = IXL_AVE_LATENCY;
716 tx_itr = IXL_ITR_20K;
717 }
718 break;
719 case IXL_AVE_LATENCY:
720 if (tx_bytes > 20) {
721 tx_latency = IXL_BULK_LATENCY;
722 tx_itr = IXL_ITR_8K;
723 } else if (tx_bytes <= 10) {
724 tx_latency = IXL_LOW_LATENCY;
725 tx_itr = IXL_ITR_100K;
726 }
727 break;
728 case IXL_BULK_LATENCY:
729 if (tx_bytes <= 20) {
730 tx_latency = IXL_AVE_LATENCY;
731 tx_itr = IXL_ITR_20K;
732 }
733 break;
734 }
735
736 txr->latency = tx_latency;
737
738 if (tx_itr != txr->itr) {
739 /* do an exponential smoothing */
740 tx_itr = (10 * tx_itr * txr->itr) /
741 ((9 * tx_itr) + txr->itr);
742 txr->itr = min(tx_itr, IXL_MAX_ITR);
743 wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR,
744 txr->me), txr->itr);
745 }
746
747 } else { /* We may have have toggled to non-dynamic */
748 if (vsi->tx_itr_setting & IXL_ITR_DYNAMIC)
749 vsi->tx_itr_setting = pf->tx_itr;
750 /* Update the hardware if needed */
751 if (txr->itr != vsi->tx_itr_setting) {
752 txr->itr = vsi->tx_itr_setting;
753 wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR,
754 txr->me), txr->itr);
755 }
756 }
757 txr->bytes = 0;
758 txr->packets = 0;
759 return;
760 }
761
762 #ifdef IXL_DEBUG
763 /**
764 * ixl_sysctl_qtx_tail_handler
765 * Retrieves I40E_QTX_TAIL value from hardware
766 * for a sysctl.
767 */
768 int
ixl_sysctl_qtx_tail_handler(SYSCTL_HANDLER_ARGS)769 ixl_sysctl_qtx_tail_handler(SYSCTL_HANDLER_ARGS)
770 {
771 struct ixl_tx_queue *tx_que;
772 int error;
773 u32 val;
774
775 tx_que = ((struct ixl_tx_queue *)oidp->oid_arg1);
776 if (!tx_que) return 0;
777
778 val = rd32(tx_que->vsi->hw, tx_que->txr.tail);
779 error = sysctl_handle_int(oidp, &val, 0, req);
780 if (error || !req->newptr)
781 return error;
782 return (0);
783 }
784
785 /**
786 * ixl_sysctl_qrx_tail_handler
787 * Retrieves I40E_QRX_TAIL value from hardware
788 * for a sysctl.
789 */
790 int
ixl_sysctl_qrx_tail_handler(SYSCTL_HANDLER_ARGS)791 ixl_sysctl_qrx_tail_handler(SYSCTL_HANDLER_ARGS)
792 {
793 struct ixl_rx_queue *rx_que;
794 int error;
795 u32 val;
796
797 rx_que = ((struct ixl_rx_queue *)oidp->oid_arg1);
798 if (!rx_que) return 0;
799
800 val = rd32(rx_que->vsi->hw, rx_que->rxr.tail);
801 error = sysctl_handle_int(oidp, &val, 0, req);
802 if (error || !req->newptr)
803 return error;
804 return (0);
805 }
806 #endif
807
808 void
ixl_add_hw_stats(struct ixl_pf * pf)809 ixl_add_hw_stats(struct ixl_pf *pf)
810 {
811 struct ixl_vsi *vsi = &pf->vsi;
812 device_t dev = iflib_get_dev(vsi->ctx);
813 struct i40e_hw_port_stats *pf_stats = &pf->stats;
814
815 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
816 struct sysctl_oid *tree = device_get_sysctl_tree(dev);
817 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
818
819 /* Driver statistics */
820 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "admin_irq",
821 CTLFLAG_RD, &pf->admin_irq,
822 "Admin Queue IRQs received");
823
824 sysctl_ctx_init(&vsi->sysctl_ctx);
825 ixl_vsi_add_sysctls(vsi, "pf", true);
826
827 ixl_add_sysctls_mac_stats(ctx, child, pf_stats);
828 }
829
830 void
ixl_set_rss_hlut(struct ixl_pf * pf)831 ixl_set_rss_hlut(struct ixl_pf *pf)
832 {
833 struct i40e_hw *hw = &pf->hw;
834 struct ixl_vsi *vsi = &pf->vsi;
835 device_t dev = iflib_get_dev(vsi->ctx);
836 int i, que_id;
837 int lut_entry_width;
838 u32 lut = 0;
839 enum i40e_status_code status;
840
841 lut_entry_width = pf->hw.func_caps.rss_table_entry_width;
842
843 /* Populate the LUT with max no. of queues in round robin fashion */
844 u8 hlut_buf[512];
845 for (i = 0; i < pf->hw.func_caps.rss_table_size; i++) {
846 #ifdef RSS
847 /*
848 * Fetch the RSS bucket id for the given indirection entry.
849 * Cap it at the number of configured buckets (which is
850 * num_queues.)
851 */
852 que_id = rss_get_indirection_to_bucket(i);
853 que_id = que_id % vsi->num_rx_queues;
854 #else
855 que_id = i % vsi->num_rx_queues;
856 #endif
857 lut = (que_id & ((0x1 << lut_entry_width) - 1));
858 hlut_buf[i] = lut;
859 }
860
861 if (hw->mac.type == I40E_MAC_X722) {
862 status = i40e_aq_set_rss_lut(hw, vsi->vsi_num, TRUE, hlut_buf, sizeof(hlut_buf));
863 if (status)
864 device_printf(dev, "i40e_aq_set_rss_lut status %s, error %s\n",
865 i40e_stat_str(hw, status), i40e_aq_str(hw, hw->aq.asq_last_status));
866 } else {
867 for (i = 0; i < pf->hw.func_caps.rss_table_size >> 2; i++)
868 wr32(hw, I40E_PFQF_HLUT(i), ((u32 *)hlut_buf)[i]);
869 ixl_flush(hw);
870 }
871 }
872
873 /* For PF VSI only */
874 int
ixl_enable_rings(struct ixl_vsi * vsi)875 ixl_enable_rings(struct ixl_vsi *vsi)
876 {
877 struct ixl_pf *pf = vsi->back;
878 int error = 0;
879
880 for (int i = 0; i < vsi->num_tx_queues; i++)
881 error = ixl_enable_tx_ring(pf, &pf->qtag, i);
882
883 for (int i = 0; i < vsi->num_rx_queues; i++)
884 error = ixl_enable_rx_ring(pf, &pf->qtag, i);
885
886 return (error);
887 }
888
889 int
ixl_disable_rings(struct ixl_pf * pf,struct ixl_vsi * vsi,struct ixl_pf_qtag * qtag)890 ixl_disable_rings(struct ixl_pf *pf, struct ixl_vsi *vsi, struct ixl_pf_qtag *qtag)
891 {
892 int error = 0;
893
894 for (int i = 0; i < vsi->num_tx_queues; i++)
895 error = ixl_disable_tx_ring(pf, qtag, i);
896
897 for (int i = 0; i < vsi->num_rx_queues; i++)
898 error = ixl_disable_rx_ring(pf, qtag, i);
899
900 return (error);
901 }
902
903 void
ixl_enable_intr(struct ixl_vsi * vsi)904 ixl_enable_intr(struct ixl_vsi *vsi)
905 {
906 struct i40e_hw *hw = vsi->hw;
907 struct ixl_rx_queue *que = vsi->rx_queues;
908
909 if (vsi->shared->isc_intr == IFLIB_INTR_MSIX) {
910 for (int i = 0; i < vsi->num_rx_queues; i++, que++)
911 ixl_enable_queue(hw, que->rxr.me);
912 } else
913 ixl_enable_intr0(hw);
914 }
915
916 void
ixl_disable_rings_intr(struct ixl_vsi * vsi)917 ixl_disable_rings_intr(struct ixl_vsi *vsi)
918 {
919 struct i40e_hw *hw = vsi->hw;
920 struct ixl_rx_queue *que = vsi->rx_queues;
921
922 for (int i = 0; i < vsi->num_rx_queues; i++, que++)
923 ixl_disable_queue(hw, que->rxr.me);
924 }
925
926 int
ixl_prepare_for_reset(struct ixl_pf * pf,bool is_up)927 ixl_prepare_for_reset(struct ixl_pf *pf, bool is_up)
928 {
929 struct i40e_hw *hw = &pf->hw;
930 device_t dev = pf->dev;
931 int error = 0;
932
933 if (is_up)
934 ixl_if_stop(pf->vsi.ctx);
935
936 ixl_shutdown_hmc(pf);
937
938 ixl_disable_intr0(hw);
939
940 error = i40e_shutdown_adminq(hw);
941 if (error)
942 device_printf(dev,
943 "Shutdown Admin queue failed with code %d\n", error);
944
945 ixl_pf_qmgr_release(&pf->qmgr, &pf->qtag);
946 return (error);
947 }
948
949 int
ixl_rebuild_hw_structs_after_reset(struct ixl_pf * pf,bool is_up)950 ixl_rebuild_hw_structs_after_reset(struct ixl_pf *pf, bool is_up)
951 {
952 struct i40e_hw *hw = &pf->hw;
953 struct ixl_vsi *vsi = &pf->vsi;
954 device_t dev = pf->dev;
955 enum i40e_get_fw_lldp_status_resp lldp_status;
956 int error = 0;
957
958 device_printf(dev, "Rebuilding driver state...\n");
959
960 /* Setup */
961 error = i40e_init_adminq(hw);
962 if (error != 0 && error != I40E_ERR_FIRMWARE_API_VERSION) {
963 device_printf(dev, "Unable to initialize Admin Queue, error %d\n",
964 error);
965 goto ixl_rebuild_hw_structs_after_reset_err;
966 }
967
968 if (IXL_PF_IN_RECOVERY_MODE(pf)) {
969 /* Keep admin queue interrupts active while driver is loaded */
970 if (vsi->shared->isc_intr == IFLIB_INTR_MSIX) {
971 ixl_configure_intr0_msix(pf);
972 ixl_enable_intr0(hw);
973 }
974
975 return (0);
976 }
977
978 i40e_clear_pxe_mode(hw);
979
980 error = ixl_get_hw_capabilities(pf);
981 if (error) {
982 device_printf(dev, "ixl_get_hw_capabilities failed: %d\n", error);
983 goto ixl_rebuild_hw_structs_after_reset_err;
984 }
985
986 error = ixl_setup_hmc(pf);
987 if (error)
988 goto ixl_rebuild_hw_structs_after_reset_err;
989
990 /* reserve a contiguous allocation for the PF's VSI */
991 error = ixl_pf_qmgr_alloc_contiguous(&pf->qmgr, vsi->num_tx_queues, &pf->qtag);
992 if (error) {
993 device_printf(dev, "Failed to reserve queues for PF LAN VSI, error %d\n",
994 error);
995 }
996
997 error = ixl_switch_config(pf);
998 if (error) {
999 device_printf(dev, "ixl_rebuild_hw_structs_after_reset: ixl_switch_config() failed: %d\n",
1000 error);
1001 error = EIO;
1002 goto ixl_rebuild_hw_structs_after_reset_err;
1003 }
1004
1005 error = i40e_aq_set_phy_int_mask(hw, IXL_DEFAULT_PHY_INT_MASK,
1006 NULL);
1007 if (error) {
1008 device_printf(dev, "init: i40e_aq_set_phy_mask() failed: err %d,"
1009 " aq_err %d\n", error, hw->aq.asq_last_status);
1010 error = EIO;
1011 goto ixl_rebuild_hw_structs_after_reset_err;
1012 }
1013
1014 u8 set_fc_err_mask;
1015 error = i40e_set_fc(hw, &set_fc_err_mask, true);
1016 if (error) {
1017 device_printf(dev, "init: setting link flow control failed; retcode %d,"
1018 " fc_err_mask 0x%02x\n", error, set_fc_err_mask);
1019 error = EIO;
1020 goto ixl_rebuild_hw_structs_after_reset_err;
1021 }
1022
1023 /* Remove default filters reinstalled by FW on reset */
1024 ixl_del_default_hw_filters(vsi);
1025
1026 /* Receive broadcast Ethernet frames */
1027 i40e_aq_set_vsi_broadcast(&pf->hw, vsi->seid, TRUE, NULL);
1028
1029 /* Determine link state */
1030 ixl_attach_get_link_status(pf);
1031
1032 i40e_aq_set_dcb_parameters(hw, TRUE, NULL);
1033
1034 /* Query device FW LLDP status */
1035 if (i40e_get_fw_lldp_status(hw, &lldp_status) == I40E_SUCCESS) {
1036 if (lldp_status == I40E_GET_FW_LLDP_STATUS_DISABLED) {
1037 ixl_set_state(&pf->state,
1038 IXL_STATE_FW_LLDP_DISABLED);
1039 } else {
1040 ixl_clear_state(&pf->state,
1041 IXL_STATE_FW_LLDP_DISABLED);
1042 }
1043 }
1044
1045 /* Keep admin queue interrupts active while driver is loaded */
1046 if (vsi->shared->isc_intr == IFLIB_INTR_MSIX) {
1047 ixl_configure_intr0_msix(pf);
1048 ixl_enable_intr0(hw);
1049 }
1050
1051 if (is_up) {
1052 iflib_request_reset(vsi->ctx);
1053 iflib_admin_intr_deferred(vsi->ctx);
1054 }
1055
1056 device_printf(dev, "Rebuilding driver state done.\n");
1057 return (0);
1058
1059 ixl_rebuild_hw_structs_after_reset_err:
1060 device_printf(dev, "Reload the driver to recover\n");
1061 return (error);
1062 }
1063
1064 /*
1065 ** Set flow control using sysctl:
1066 ** 0 - off
1067 ** 1 - rx pause
1068 ** 2 - tx pause
1069 ** 3 - full
1070 */
1071 int
ixl_sysctl_set_flowcntl(SYSCTL_HANDLER_ARGS)1072 ixl_sysctl_set_flowcntl(SYSCTL_HANDLER_ARGS)
1073 {
1074 struct ixl_pf *pf = (struct ixl_pf *)arg1;
1075 struct i40e_hw *hw = &pf->hw;
1076 device_t dev = pf->dev;
1077 int requested_fc, error = 0;
1078 enum i40e_status_code aq_error = 0;
1079 u8 fc_aq_err = 0;
1080
1081 /* Get request */
1082 requested_fc = pf->fc;
1083 error = sysctl_handle_int(oidp, &requested_fc, 0, req);
1084 if ((error) || (req->newptr == NULL))
1085 return (error);
1086 if (requested_fc < 0 || requested_fc > 3) {
1087 device_printf(dev,
1088 "Invalid fc mode; valid modes are 0 through 3\n");
1089 return (EINVAL);
1090 }
1091
1092 /* Set fc ability for port */
1093 hw->fc.requested_mode = requested_fc;
1094 aq_error = i40e_set_fc(hw, &fc_aq_err, TRUE);
1095 if (aq_error) {
1096 device_printf(dev,
1097 "%s: Error setting Flow Control mode %d; fc_err %#x\n",
1098 __func__, aq_error, fc_aq_err);
1099 return (EIO);
1100 }
1101 pf->fc = requested_fc;
1102
1103 return (0);
1104 }
1105