1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2010-2016 Solarflare Communications Inc.
5 * All rights reserved.
6 *
7 * This software was developed in part by Philip Paeps under contract for
8 * Solarflare Communications, Inc.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions are met:
12 *
13 * 1. Redistributions of source code must retain the above copyright notice,
14 * this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright notice,
16 * this list of conditions and the following disclaimer in the documentation
17 * and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
21 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
23 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
24 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
25 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
26 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
27 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
28 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
29 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 *
31 * The views and conclusions contained in the software and documentation are
32 * those of the authors and should not be interpreted as representing official
33 * policies, either expressed or implied, of the FreeBSD Project.
34 */
35
36 #include <sys/cdefs.h>
37 #include "opt_rss.h"
38
39 #include <sys/param.h>
40 #include <sys/bus.h>
41 #include <sys/kernel.h>
42 #include <sys/malloc.h>
43 #include <sys/queue.h>
44 #include <sys/rman.h>
45 #include <sys/syslog.h>
46 #include <sys/taskqueue.h>
47
48 #include <machine/bus.h>
49 #include <machine/resource.h>
50
51 #include <dev/pci/pcireg.h>
52 #include <dev/pci/pcivar.h>
53
54 #ifdef RSS
55 #include <net/rss_config.h>
56 #endif
57
58 #include "common/efx.h"
59
60 #include "sfxge.h"
61
62 static int
sfxge_intr_line_filter(void * arg)63 sfxge_intr_line_filter(void *arg)
64 {
65 struct sfxge_evq *evq;
66 struct sfxge_softc *sc;
67 efx_nic_t *enp;
68 struct sfxge_intr *intr;
69 boolean_t fatal;
70 uint32_t qmask;
71
72 evq = (struct sfxge_evq *)arg;
73 sc = evq->sc;
74 enp = sc->enp;
75 intr = &sc->intr;
76
77 KASSERT(intr != NULL, ("intr == NULL"));
78 KASSERT(intr->type == EFX_INTR_LINE,
79 ("intr->type != EFX_INTR_LINE"));
80
81 if (intr->state != SFXGE_INTR_STARTED)
82 return (FILTER_STRAY);
83
84 (void)efx_intr_status_line(enp, &fatal, &qmask);
85
86 if (fatal) {
87 (void) efx_intr_disable(enp);
88 (void) efx_intr_fatal(enp);
89 return (FILTER_HANDLED);
90 }
91
92 if (qmask != 0) {
93 intr->zero_count = 0;
94 return (FILTER_SCHEDULE_THREAD);
95 }
96
97 /* SF bug 15783: If the function is not asserting its IRQ and
98 * we read the queue mask on the cycle before a flag is added
99 * to the mask, this inhibits the function from asserting the
100 * IRQ even though we don't see the flag set. To work around
101 * this, we must re-prime all event queues and report the IRQ
102 * as handled when we see a mask of zero. To allow for shared
103 * IRQs, we don't repeat this if we see a mask of zero twice
104 * or more in a row.
105 */
106 if (intr->zero_count++ == 0) {
107 if (evq->init_state == SFXGE_EVQ_STARTED) {
108 if (efx_ev_qpending(evq->common, evq->read_ptr))
109 return (FILTER_SCHEDULE_THREAD);
110 efx_ev_qprime(evq->common, evq->read_ptr);
111 return (FILTER_HANDLED);
112 }
113 }
114
115 return (FILTER_STRAY);
116 }
117
118 static void
sfxge_intr_line(void * arg)119 sfxge_intr_line(void *arg)
120 {
121 struct sfxge_evq *evq = arg;
122
123 (void)sfxge_ev_qpoll(evq);
124 }
125
126 static void
sfxge_intr_message(void * arg)127 sfxge_intr_message(void *arg)
128 {
129 struct sfxge_evq *evq;
130 struct sfxge_softc *sc;
131 efx_nic_t *enp;
132 struct sfxge_intr *intr;
133 unsigned int index;
134 boolean_t fatal;
135
136 evq = (struct sfxge_evq *)arg;
137 sc = evq->sc;
138 enp = sc->enp;
139 intr = &sc->intr;
140 index = evq->index;
141
142 KASSERT(intr != NULL, ("intr == NULL"));
143 KASSERT(intr->type == EFX_INTR_MESSAGE,
144 ("intr->type != EFX_INTR_MESSAGE"));
145
146 if (__predict_false(intr->state != SFXGE_INTR_STARTED))
147 return;
148
149 (void)efx_intr_status_message(enp, index, &fatal);
150
151 if (fatal) {
152 (void)efx_intr_disable(enp);
153 (void)efx_intr_fatal(enp);
154 return;
155 }
156
157 (void)sfxge_ev_qpoll(evq);
158 }
159
160 static int
sfxge_intr_bus_enable(struct sfxge_softc * sc)161 sfxge_intr_bus_enable(struct sfxge_softc *sc)
162 {
163 struct sfxge_intr *intr;
164 struct sfxge_intr_hdl *table;
165 driver_filter_t *filter;
166 driver_intr_t *handler;
167 int index;
168 int err;
169
170 intr = &sc->intr;
171 table = intr->table;
172
173 switch (intr->type) {
174 case EFX_INTR_MESSAGE:
175 filter = NULL; /* not shared */
176 handler = sfxge_intr_message;
177 break;
178
179 case EFX_INTR_LINE:
180 filter = sfxge_intr_line_filter;
181 handler = sfxge_intr_line;
182 break;
183
184 default:
185 KASSERT(0, ("Invalid interrupt type"));
186 return (EINVAL);
187 }
188
189 /* Try to add the handlers */
190 for (index = 0; index < intr->n_alloc; index++) {
191 if ((err = bus_setup_intr(sc->dev, table[index].eih_res,
192 INTR_MPSAFE|INTR_TYPE_NET, filter, handler,
193 sc->evq[index], &table[index].eih_tag)) != 0) {
194 goto fail;
195 }
196 if (intr->n_alloc > 1)
197 bus_describe_intr(sc->dev, table[index].eih_res,
198 table[index].eih_tag, "%d", index);
199 #ifdef RSS
200 bus_bind_intr(sc->dev, table[index].eih_res,
201 rss_getcpu(index));
202 #else
203 bus_bind_intr(sc->dev, table[index].eih_res, index);
204 #endif
205 }
206
207 return (0);
208
209 fail:
210 /* Remove remaining handlers */
211 while (--index >= 0)
212 bus_teardown_intr(sc->dev, table[index].eih_res,
213 table[index].eih_tag);
214
215 return (err);
216 }
217
218 static void
sfxge_intr_bus_disable(struct sfxge_softc * sc)219 sfxge_intr_bus_disable(struct sfxge_softc *sc)
220 {
221 struct sfxge_intr *intr;
222 struct sfxge_intr_hdl *table;
223 int i;
224
225 intr = &sc->intr;
226 table = intr->table;
227
228 /* Remove all handlers */
229 for (i = 0; i < intr->n_alloc; i++)
230 bus_teardown_intr(sc->dev, table[i].eih_res,
231 table[i].eih_tag);
232 }
233
234 static int
sfxge_intr_alloc(struct sfxge_softc * sc,int count)235 sfxge_intr_alloc(struct sfxge_softc *sc, int count)
236 {
237 device_t dev;
238 struct sfxge_intr_hdl *table;
239 struct sfxge_intr *intr;
240 struct resource *res;
241 int rid;
242 int error;
243 int i;
244
245 dev = sc->dev;
246 intr = &sc->intr;
247 error = 0;
248
249 table = malloc(count * sizeof(struct sfxge_intr_hdl),
250 M_SFXGE, M_WAITOK);
251 intr->table = table;
252
253 for (i = 0; i < count; i++) {
254 rid = i + 1;
255 res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
256 RF_SHAREABLE | RF_ACTIVE);
257 if (res == NULL) {
258 device_printf(dev, "Couldn't allocate interrupts for "
259 "message %d\n", rid);
260 error = ENOMEM;
261 break;
262 }
263 table[i].eih_rid = rid;
264 table[i].eih_res = res;
265 }
266
267 if (error != 0) {
268 count = i - 1;
269 for (i = 0; i < count; i++)
270 bus_release_resource(dev, SYS_RES_IRQ,
271 table[i].eih_rid, table[i].eih_res);
272 }
273
274 return (error);
275 }
276
277 static void
sfxge_intr_teardown_msix(struct sfxge_softc * sc)278 sfxge_intr_teardown_msix(struct sfxge_softc *sc)
279 {
280 device_t dev;
281 struct resource *resp;
282 int rid;
283
284 dev = sc->dev;
285 resp = sc->intr.msix_res;
286
287 rid = rman_get_rid(resp);
288 bus_release_resource(dev, SYS_RES_MEMORY, rid, resp);
289 }
290
291 static int
sfxge_intr_setup_msix(struct sfxge_softc * sc)292 sfxge_intr_setup_msix(struct sfxge_softc *sc)
293 {
294 struct sfxge_intr *intr;
295 struct resource *resp;
296 device_t dev;
297 int count;
298 int rid;
299
300 dev = sc->dev;
301 intr = &sc->intr;
302
303 /* Check if MSI-X is available. */
304 count = pci_msix_count(dev);
305 if (count == 0)
306 return (EINVAL);
307
308 /* Do not try to allocate more than already estimated EVQ maximum */
309 KASSERT(sc->evq_max > 0, ("evq_max is zero"));
310 count = MIN(count, sc->evq_max);
311
312 rid = PCIR_BAR(4);
313 resp = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE);
314 if (resp == NULL)
315 return (ENOMEM);
316
317 if (pci_alloc_msix(dev, &count) != 0) {
318 bus_release_resource(dev, SYS_RES_MEMORY, rid, resp);
319 return (ENOMEM);
320 }
321
322 /* Allocate interrupt handlers. */
323 if (sfxge_intr_alloc(sc, count) != 0) {
324 bus_release_resource(dev, SYS_RES_MEMORY, rid, resp);
325 pci_release_msi(dev);
326 return (ENOMEM);
327 }
328
329 intr->type = EFX_INTR_MESSAGE;
330 intr->n_alloc = count;
331 intr->msix_res = resp;
332
333 return (0);
334 }
335
336 static int
sfxge_intr_setup_msi(struct sfxge_softc * sc)337 sfxge_intr_setup_msi(struct sfxge_softc *sc)
338 {
339 struct sfxge_intr *intr;
340 device_t dev;
341 int count;
342 int error;
343
344 dev = sc->dev;
345 intr = &sc->intr;
346
347 /*
348 * Check if MSI is available. All messages must be written to
349 * the same address and on x86 this means the IRQs have the
350 * same CPU affinity. So we only ever allocate 1.
351 */
352 count = pci_msi_count(dev) ? 1 : 0;
353 if (count == 0)
354 return (EINVAL);
355
356 if ((error = pci_alloc_msi(dev, &count)) != 0)
357 return (ENOMEM);
358
359 /* Allocate interrupt handler. */
360 if (sfxge_intr_alloc(sc, count) != 0) {
361 pci_release_msi(dev);
362 return (ENOMEM);
363 }
364
365 intr->type = EFX_INTR_MESSAGE;
366 intr->n_alloc = count;
367
368 return (0);
369 }
370
371 static int
sfxge_intr_setup_fixed(struct sfxge_softc * sc)372 sfxge_intr_setup_fixed(struct sfxge_softc *sc)
373 {
374 struct sfxge_intr_hdl *table;
375 struct sfxge_intr *intr;
376 struct resource *res;
377 device_t dev;
378 int rid;
379
380 dev = sc->dev;
381 intr = &sc->intr;
382
383 rid = 0;
384 res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
385 RF_SHAREABLE | RF_ACTIVE);
386 if (res == NULL)
387 return (ENOMEM);
388
389 table = malloc(sizeof(struct sfxge_intr_hdl), M_SFXGE, M_WAITOK);
390 table[0].eih_rid = rid;
391 table[0].eih_res = res;
392
393 intr->type = EFX_INTR_LINE;
394 intr->n_alloc = 1;
395 intr->table = table;
396
397 return (0);
398 }
399
400 static const char *const __sfxge_err[] = {
401 "",
402 "SRAM out-of-bounds",
403 "Buffer ID out-of-bounds",
404 "Internal memory parity",
405 "Receive buffer ownership",
406 "Transmit buffer ownership",
407 "Receive descriptor ownership",
408 "Transmit descriptor ownership",
409 "Event queue ownership",
410 "Event queue FIFO overflow",
411 "Illegal address",
412 "SRAM parity"
413 };
414
415 void
sfxge_err(efsys_identifier_t * arg,unsigned int code,uint32_t dword0,uint32_t dword1)416 sfxge_err(efsys_identifier_t *arg, unsigned int code, uint32_t dword0,
417 uint32_t dword1)
418 {
419 struct sfxge_softc *sc = (struct sfxge_softc *)arg;
420 device_t dev = sc->dev;
421
422 log(LOG_WARNING, "[%s%d] FATAL ERROR: %s (0x%08x%08x)",
423 device_get_name(dev), device_get_unit(dev),
424 __sfxge_err[code], dword1, dword0);
425 }
426
427 void
sfxge_intr_stop(struct sfxge_softc * sc)428 sfxge_intr_stop(struct sfxge_softc *sc)
429 {
430 struct sfxge_intr *intr;
431
432 intr = &sc->intr;
433
434 KASSERT(intr->state == SFXGE_INTR_STARTED,
435 ("Interrupts not started"));
436
437 intr->state = SFXGE_INTR_INITIALIZED;
438
439 /* Disable interrupts at the NIC */
440 efx_intr_disable(sc->enp);
441
442 /* Disable interrupts at the bus */
443 sfxge_intr_bus_disable(sc);
444
445 /* Tear down common code interrupt bits. */
446 efx_intr_fini(sc->enp);
447 }
448
449 int
sfxge_intr_start(struct sfxge_softc * sc)450 sfxge_intr_start(struct sfxge_softc *sc)
451 {
452 struct sfxge_intr *intr;
453 efsys_mem_t *esmp;
454 int rc;
455
456 intr = &sc->intr;
457 esmp = &intr->status;
458
459 KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
460 ("Interrupts not initialized"));
461
462 /* Zero the memory. */
463 (void)memset(esmp->esm_base, 0, EFX_INTR_SIZE);
464
465 /* Initialize common code interrupt bits. */
466 (void)efx_intr_init(sc->enp, intr->type, esmp);
467
468 /* Enable interrupts at the bus */
469 if ((rc = sfxge_intr_bus_enable(sc)) != 0)
470 goto fail;
471
472 intr->state = SFXGE_INTR_STARTED;
473
474 /* Enable interrupts at the NIC */
475 efx_intr_enable(sc->enp);
476
477 return (0);
478
479 fail:
480 /* Tear down common code interrupt bits. */
481 efx_intr_fini(sc->enp);
482
483 intr->state = SFXGE_INTR_INITIALIZED;
484
485 return (rc);
486 }
487
488 void
sfxge_intr_fini(struct sfxge_softc * sc)489 sfxge_intr_fini(struct sfxge_softc *sc)
490 {
491 struct sfxge_intr_hdl *table;
492 struct sfxge_intr *intr;
493 efsys_mem_t *esmp;
494 device_t dev;
495 int i;
496
497 dev = sc->dev;
498 intr = &sc->intr;
499 esmp = &intr->status;
500 table = intr->table;
501
502 KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
503 ("intr->state != SFXGE_INTR_INITIALIZED"));
504
505 /* Free DMA memory. */
506 sfxge_dma_free(esmp);
507
508 /* Free interrupt handles. */
509 for (i = 0; i < intr->n_alloc; i++)
510 bus_release_resource(dev, SYS_RES_IRQ,
511 table[i].eih_rid, table[i].eih_res);
512
513 if (table[0].eih_rid != 0)
514 pci_release_msi(dev);
515
516 if (intr->msix_res != NULL)
517 sfxge_intr_teardown_msix(sc);
518
519 /* Free the handle table */
520 free(table, M_SFXGE);
521 intr->table = NULL;
522 intr->n_alloc = 0;
523
524 /* Clear the interrupt type */
525 intr->type = EFX_INTR_INVALID;
526
527 intr->state = SFXGE_INTR_UNINITIALIZED;
528 }
529
530 int
sfxge_intr_init(struct sfxge_softc * sc)531 sfxge_intr_init(struct sfxge_softc *sc)
532 {
533 device_t dev;
534 struct sfxge_intr *intr;
535 efsys_mem_t *esmp;
536 int rc;
537
538 dev = sc->dev;
539 intr = &sc->intr;
540 esmp = &intr->status;
541
542 KASSERT(intr->state == SFXGE_INTR_UNINITIALIZED,
543 ("Interrupts already initialized"));
544
545 /* Try to setup MSI-X or MSI interrupts if available. */
546 if ((rc = sfxge_intr_setup_msix(sc)) == 0)
547 device_printf(dev, "Using MSI-X interrupts\n");
548 else if ((rc = sfxge_intr_setup_msi(sc)) == 0)
549 device_printf(dev, "Using MSI interrupts\n");
550 else if ((rc = sfxge_intr_setup_fixed(sc)) == 0) {
551 device_printf(dev, "Using fixed interrupts\n");
552 } else {
553 device_printf(dev, "Couldn't setup interrupts\n");
554 return (ENOMEM);
555 }
556
557 /* Set up DMA for interrupts. */
558 if ((rc = sfxge_dma_alloc(sc, EFX_INTR_SIZE, esmp)) != 0)
559 return (ENOMEM);
560
561 intr->state = SFXGE_INTR_INITIALIZED;
562
563 return (0);
564 }
565