1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2010-2016 Solarflare Communications Inc. 5 * All rights reserved. 6 * 7 * This software was developed in part by Philip Paeps under contract for 8 * Solarflare Communications, Inc. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions are met: 12 * 13 * 1. Redistributions of source code must retain the above copyright notice, 14 * this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright notice, 16 * this list of conditions and the following disclaimer in the documentation 17 * and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 21 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 23 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 24 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 25 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 26 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 28 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 29 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 * 31 * The views and conclusions contained in the software and documentation are 32 * those of the authors and should not be interpreted as representing official 33 * policies, either expressed or implied, of the FreeBSD Project. 34 */ 35 36 #include <sys/cdefs.h> 37 __FBSDID("$FreeBSD$"); 38 39 #include <sys/param.h> 40 #include <sys/kernel.h> 41 #include <sys/malloc.h> 42 #include <sys/param.h> 43 #include <sys/queue.h> 44 #include <sys/systm.h> 45 #include <sys/taskqueue.h> 46 47 #include "common/efx.h" 48 49 #include "sfxge.h" 50 51 static void 52 sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop) 53 { 54 struct sfxge_softc *sc; 55 unsigned int index; 56 struct sfxge_rxq *rxq; 57 struct sfxge_txq *txq; 58 59 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 60 61 sc = evq->sc; 62 index = evq->index; 63 rxq = sc->rxq[index]; 64 65 if ((txq = evq->txq) != NULL) { 66 evq->txq = NULL; 67 evq->txqs = &(evq->txq); 68 69 do { 70 struct sfxge_txq *next; 71 72 next = txq->next; 73 txq->next = NULL; 74 75 KASSERT(txq->evq_index == index, 76 ("txq->evq_index != index")); 77 78 if (txq->pending != txq->completed) 79 sfxge_tx_qcomplete(txq, evq); 80 81 txq = next; 82 } while (txq != NULL); 83 } 84 85 if (rxq->pending != rxq->completed) 86 sfxge_rx_qcomplete(rxq, eop); 87 } 88 89 static struct sfxge_rxq * 90 sfxge_get_rxq_by_label(struct sfxge_evq *evq, uint32_t label) 91 { 92 struct sfxge_rxq *rxq; 93 94 KASSERT(label == 0, ("unexpected rxq label != 0")); 95 96 rxq = evq->sc->rxq[evq->index]; 97 98 KASSERT(rxq != NULL, ("rxq == NULL")); 99 KASSERT(evq->index == rxq->index, ("evq->index != rxq->index")); 100 101 return (rxq); 102 } 103 104 static boolean_t 105 sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size, 106 uint16_t flags) 107 { 108 struct sfxge_evq *evq; 109 struct sfxge_softc *sc; 110 struct sfxge_rxq *rxq; 111 unsigned int stop; 112 unsigned int delta; 113 struct sfxge_rx_sw_desc *rx_desc; 114 115 evq = arg; 116 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 117 118 sc = evq->sc; 119 120 if (evq->exception) 121 goto done; 122 123 rxq = sfxge_get_rxq_by_label(evq, label); 124 if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED)) 125 goto done; 126 127 stop = (id + 1) & rxq->ptr_mask; 128 id = rxq->pending & rxq->ptr_mask; 129 delta = (stop >= id) ? (stop - id) : (rxq->entries - id + stop); 130 rxq->pending += delta; 131 132 if (delta != 1) { 133 if ((delta <= 0) || 134 (delta > efx_nic_cfg_get(sc->enp)->enc_rx_batch_max)) { 135 evq->exception = B_TRUE; 136 137 device_printf(sc->dev, "RX completion out of order" 138 " (id=%#x delta=%u flags=%#x); resetting\n", 139 id, delta, flags); 140 sfxge_schedule_reset(sc); 141 142 goto done; 143 } 144 } 145 146 rx_desc = &rxq->queue[id]; 147 148 prefetch_read_many(rx_desc->mbuf); 149 150 for (; id != stop; id = (id + 1) & rxq->ptr_mask) { 151 rx_desc = &rxq->queue[id]; 152 KASSERT(rx_desc->flags == EFX_DISCARD, 153 ("rx_desc->flags != EFX_DISCARD")); 154 rx_desc->flags = flags; 155 156 KASSERT(size < (1 << 16), ("size > (1 << 16)")); 157 rx_desc->size = (uint16_t)size; 158 } 159 160 evq->rx_done++; 161 162 if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH) 163 sfxge_ev_qcomplete(evq, B_FALSE); 164 165 done: 166 return (evq->rx_done >= SFXGE_EV_BATCH); 167 } 168 169 static boolean_t 170 sfxge_ev_exception(void *arg, uint32_t code, uint32_t data) 171 { 172 struct sfxge_evq *evq; 173 struct sfxge_softc *sc; 174 175 evq = (struct sfxge_evq *)arg; 176 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 177 178 sc = evq->sc; 179 180 DBGPRINT(sc->dev, "[%d] %s", evq->index, 181 (code == EFX_EXCEPTION_RX_RECOVERY) ? "RX_RECOVERY" : 182 (code == EFX_EXCEPTION_RX_DSC_ERROR) ? "RX_DSC_ERROR" : 183 (code == EFX_EXCEPTION_TX_DSC_ERROR) ? "TX_DSC_ERROR" : 184 (code == EFX_EXCEPTION_UNKNOWN_SENSOREVT) ? "UNKNOWN_SENSOREVT" : 185 (code == EFX_EXCEPTION_FWALERT_SRAM) ? "FWALERT_SRAM" : 186 (code == EFX_EXCEPTION_UNKNOWN_FWALERT) ? "UNKNOWN_FWALERT" : 187 (code == EFX_EXCEPTION_RX_ERROR) ? "RX_ERROR" : 188 (code == EFX_EXCEPTION_TX_ERROR) ? "TX_ERROR" : 189 (code == EFX_EXCEPTION_EV_ERROR) ? "EV_ERROR" : 190 "UNKNOWN"); 191 192 evq->exception = B_TRUE; 193 194 if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) { 195 device_printf(sc->dev, 196 "hardware exception (code=%u); resetting\n", 197 code); 198 sfxge_schedule_reset(sc); 199 } 200 201 return (B_FALSE); 202 } 203 204 static boolean_t 205 sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index) 206 { 207 struct sfxge_evq *evq; 208 struct sfxge_softc *sc; 209 struct sfxge_rxq *rxq; 210 unsigned int index; 211 uint16_t magic; 212 213 evq = (struct sfxge_evq *)arg; 214 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 215 216 sc = evq->sc; 217 rxq = sc->rxq[rxq_index]; 218 219 KASSERT(rxq != NULL, ("rxq == NULL")); 220 221 /* Resend a software event on the correct queue */ 222 index = rxq->index; 223 if (index == evq->index) { 224 sfxge_rx_qflush_done(rxq); 225 return (B_FALSE); 226 } 227 228 evq = sc->evq[index]; 229 magic = sfxge_sw_ev_rxq_magic(SFXGE_SW_EV_RX_QFLUSH_DONE, rxq); 230 231 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 232 ("evq not started")); 233 efx_ev_qpost(evq->common, magic); 234 235 return (B_FALSE); 236 } 237 238 static boolean_t 239 sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index) 240 { 241 struct sfxge_evq *evq; 242 struct sfxge_softc *sc; 243 struct sfxge_rxq *rxq; 244 unsigned int index; 245 uint16_t magic; 246 247 evq = (struct sfxge_evq *)arg; 248 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 249 250 sc = evq->sc; 251 rxq = sc->rxq[rxq_index]; 252 253 KASSERT(rxq != NULL, ("rxq == NULL")); 254 255 /* Resend a software event on the correct queue */ 256 index = rxq->index; 257 evq = sc->evq[index]; 258 magic = sfxge_sw_ev_rxq_magic(SFXGE_SW_EV_RX_QFLUSH_FAILED, rxq); 259 260 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 261 ("evq not started")); 262 efx_ev_qpost(evq->common, magic); 263 264 return (B_FALSE); 265 } 266 267 static struct sfxge_txq * 268 sfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label) 269 { 270 unsigned int index; 271 272 KASSERT((evq->index == 0 && label < SFXGE_TXQ_NTYPES) || 273 (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM), ("unexpected txq label")); 274 index = (evq->index == 0) ? label : (evq->index - 1 + SFXGE_TXQ_NTYPES); 275 return (evq->sc->txq[index]); 276 } 277 278 static boolean_t 279 sfxge_ev_tx(void *arg, uint32_t label, uint32_t id) 280 { 281 struct sfxge_evq *evq; 282 struct sfxge_txq *txq; 283 unsigned int stop; 284 unsigned int delta; 285 286 evq = (struct sfxge_evq *)arg; 287 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 288 289 txq = sfxge_get_txq_by_label(evq, label); 290 291 KASSERT(txq != NULL, ("txq == NULL")); 292 KASSERT(evq->index == txq->evq_index, 293 ("evq->index != txq->evq_index")); 294 295 if (__predict_false(txq->init_state != SFXGE_TXQ_STARTED)) 296 goto done; 297 298 stop = (id + 1) & txq->ptr_mask; 299 id = txq->pending & txq->ptr_mask; 300 301 delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop); 302 txq->pending += delta; 303 304 evq->tx_done++; 305 306 if (txq->next == NULL && 307 evq->txqs != &(txq->next)) { 308 *(evq->txqs) = txq; 309 evq->txqs = &(txq->next); 310 } 311 312 if (txq->pending - txq->completed >= SFXGE_TX_BATCH) 313 sfxge_tx_qcomplete(txq, evq); 314 315 done: 316 return (evq->tx_done >= SFXGE_EV_BATCH); 317 } 318 319 static boolean_t 320 sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index) 321 { 322 struct sfxge_evq *evq; 323 struct sfxge_softc *sc; 324 struct sfxge_txq *txq; 325 uint16_t magic; 326 327 evq = (struct sfxge_evq *)arg; 328 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 329 330 sc = evq->sc; 331 txq = sc->txq[txq_index]; 332 333 KASSERT(txq != NULL, ("txq == NULL")); 334 KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED, 335 ("txq not initialized")); 336 337 if (txq->evq_index == evq->index) { 338 sfxge_tx_qflush_done(txq); 339 return (B_FALSE); 340 } 341 342 /* Resend a software event on the correct queue */ 343 evq = sc->evq[txq->evq_index]; 344 magic = sfxge_sw_ev_txq_magic(SFXGE_SW_EV_TX_QFLUSH_DONE, txq); 345 346 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 347 ("evq not started")); 348 efx_ev_qpost(evq->common, magic); 349 350 return (B_FALSE); 351 } 352 353 static boolean_t 354 sfxge_ev_software(void *arg, uint16_t magic) 355 { 356 struct sfxge_evq *evq; 357 struct sfxge_softc *sc; 358 unsigned int label; 359 360 evq = (struct sfxge_evq *)arg; 361 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 362 363 sc = evq->sc; 364 365 label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK; 366 magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK; 367 368 switch (magic) { 369 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QFLUSH_DONE): 370 sfxge_rx_qflush_done(sfxge_get_rxq_by_label(evq, label)); 371 break; 372 373 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QFLUSH_FAILED): 374 sfxge_rx_qflush_failed(sfxge_get_rxq_by_label(evq, label)); 375 break; 376 377 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QREFILL): 378 sfxge_rx_qrefill(sfxge_get_rxq_by_label(evq, label)); 379 break; 380 381 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_TX_QFLUSH_DONE): { 382 struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label); 383 384 KASSERT(txq != NULL, ("txq == NULL")); 385 KASSERT(evq->index == txq->evq_index, 386 ("evq->index != txq->evq_index")); 387 388 sfxge_tx_qflush_done(txq); 389 break; 390 } 391 default: 392 break; 393 } 394 395 return (B_FALSE); 396 } 397 398 static boolean_t 399 sfxge_ev_sram(void *arg, uint32_t code) 400 { 401 (void)arg; 402 (void)code; 403 404 switch (code) { 405 case EFX_SRAM_UPDATE: 406 EFSYS_PROBE(sram_update); 407 break; 408 409 case EFX_SRAM_CLEAR: 410 EFSYS_PROBE(sram_clear); 411 break; 412 413 case EFX_SRAM_ILLEGAL_CLEAR: 414 EFSYS_PROBE(sram_illegal_clear); 415 break; 416 417 default: 418 KASSERT(B_FALSE, ("Impossible SRAM event")); 419 break; 420 } 421 422 return (B_FALSE); 423 } 424 425 static boolean_t 426 sfxge_ev_timer(void *arg, uint32_t index) 427 { 428 (void)arg; 429 (void)index; 430 431 return (B_FALSE); 432 } 433 434 static boolean_t 435 sfxge_ev_wake_up(void *arg, uint32_t index) 436 { 437 (void)arg; 438 (void)index; 439 440 return (B_FALSE); 441 } 442 443 #if EFSYS_OPT_QSTATS 444 445 static void 446 sfxge_ev_stat_update(struct sfxge_softc *sc) 447 { 448 struct sfxge_evq *evq; 449 unsigned int index; 450 clock_t now; 451 452 SFXGE_ADAPTER_LOCK(sc); 453 454 if (__predict_false(sc->evq[0]->init_state != SFXGE_EVQ_STARTED)) 455 goto out; 456 457 now = ticks; 458 if ((unsigned int)(now - sc->ev_stats_update_time) < (unsigned int)hz) 459 goto out; 460 461 sc->ev_stats_update_time = now; 462 463 /* Add event counts from each event queue in turn */ 464 for (index = 0; index < sc->evq_count; index++) { 465 evq = sc->evq[index]; 466 SFXGE_EVQ_LOCK(evq); 467 efx_ev_qstats_update(evq->common, sc->ev_stats); 468 SFXGE_EVQ_UNLOCK(evq); 469 } 470 out: 471 SFXGE_ADAPTER_UNLOCK(sc); 472 } 473 474 static int 475 sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS) 476 { 477 struct sfxge_softc *sc = arg1; 478 unsigned int id = arg2; 479 480 sfxge_ev_stat_update(sc); 481 482 return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id]))); 483 } 484 485 static void 486 sfxge_ev_stat_init(struct sfxge_softc *sc) 487 { 488 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); 489 struct sysctl_oid_list *stat_list; 490 unsigned int id; 491 char name[40]; 492 493 stat_list = SYSCTL_CHILDREN(sc->stats_node); 494 495 for (id = 0; id < EV_NQSTATS; id++) { 496 snprintf(name, sizeof(name), "ev_%s", 497 efx_ev_qstat_name(sc->enp, id)); 498 SYSCTL_ADD_PROC( 499 ctx, stat_list, 500 OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD, 501 sc, id, sfxge_ev_stat_handler, "Q", 502 ""); 503 } 504 } 505 506 #endif /* EFSYS_OPT_QSTATS */ 507 508 static void 509 sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us) 510 { 511 struct sfxge_evq *evq; 512 efx_evq_t *eep; 513 514 evq = sc->evq[idx]; 515 eep = evq->common; 516 517 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 518 ("evq->init_state != SFXGE_EVQ_STARTED")); 519 520 (void)efx_ev_qmoderate(eep, us); 521 } 522 523 static int 524 sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS) 525 { 526 struct sfxge_softc *sc = arg1; 527 struct sfxge_intr *intr = &sc->intr; 528 unsigned int moderation; 529 int error; 530 unsigned int index; 531 532 SFXGE_ADAPTER_LOCK(sc); 533 534 if (req->newptr != NULL) { 535 if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation))) 536 != 0) 537 goto out; 538 539 /* We may not be calling efx_ev_qmoderate() now, 540 * so we have to range-check the value ourselves. 541 */ 542 if (moderation > 543 efx_nic_cfg_get(sc->enp)->enc_evq_timer_max_us) { 544 error = EINVAL; 545 goto out; 546 } 547 548 sc->ev_moderation = moderation; 549 if (intr->state == SFXGE_INTR_STARTED) { 550 for (index = 0; index < sc->evq_count; index++) 551 sfxge_ev_qmoderate(sc, index, moderation); 552 } 553 } else { 554 error = SYSCTL_OUT(req, &sc->ev_moderation, 555 sizeof(sc->ev_moderation)); 556 } 557 558 out: 559 SFXGE_ADAPTER_UNLOCK(sc); 560 561 return (error); 562 } 563 564 static boolean_t 565 sfxge_ev_initialized(void *arg) 566 { 567 struct sfxge_evq *evq; 568 569 evq = (struct sfxge_evq *)arg; 570 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 571 572 /* Init done events may be duplicated on 7xxx */ 573 KASSERT(evq->init_state == SFXGE_EVQ_STARTING || 574 evq->init_state == SFXGE_EVQ_STARTED, 575 ("evq not starting")); 576 577 evq->init_state = SFXGE_EVQ_STARTED; 578 579 return (0); 580 } 581 582 static boolean_t 583 sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode) 584 { 585 struct sfxge_evq *evq; 586 struct sfxge_softc *sc; 587 588 evq = (struct sfxge_evq *)arg; 589 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 590 591 sc = evq->sc; 592 593 sfxge_mac_link_update(sc, link_mode); 594 595 return (0); 596 } 597 598 static const efx_ev_callbacks_t sfxge_ev_callbacks = { 599 .eec_initialized = sfxge_ev_initialized, 600 .eec_rx = sfxge_ev_rx, 601 .eec_tx = sfxge_ev_tx, 602 .eec_exception = sfxge_ev_exception, 603 .eec_rxq_flush_done = sfxge_ev_rxq_flush_done, 604 .eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed, 605 .eec_txq_flush_done = sfxge_ev_txq_flush_done, 606 .eec_software = sfxge_ev_software, 607 .eec_sram = sfxge_ev_sram, 608 .eec_wake_up = sfxge_ev_wake_up, 609 .eec_timer = sfxge_ev_timer, 610 .eec_link_change = sfxge_ev_link_change, 611 }; 612 613 614 int 615 sfxge_ev_qpoll(struct sfxge_evq *evq) 616 { 617 int rc; 618 619 SFXGE_EVQ_LOCK(evq); 620 621 if (__predict_false(evq->init_state != SFXGE_EVQ_STARTING && 622 evq->init_state != SFXGE_EVQ_STARTED)) { 623 rc = EINVAL; 624 goto fail; 625 } 626 627 /* Synchronize the DMA memory for reading */ 628 bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map, 629 BUS_DMASYNC_POSTREAD); 630 631 KASSERT(evq->rx_done == 0, ("evq->rx_done != 0")); 632 KASSERT(evq->tx_done == 0, ("evq->tx_done != 0")); 633 KASSERT(evq->txq == NULL, ("evq->txq != NULL")); 634 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 635 636 /* Poll the queue */ 637 efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq); 638 639 evq->rx_done = 0; 640 evq->tx_done = 0; 641 642 /* Perform any pending completion processing */ 643 sfxge_ev_qcomplete(evq, B_TRUE); 644 645 /* Re-prime the event queue for interrupts */ 646 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 647 goto fail; 648 649 SFXGE_EVQ_UNLOCK(evq); 650 651 return (0); 652 653 fail: 654 SFXGE_EVQ_UNLOCK(evq); 655 return (rc); 656 } 657 658 static void 659 sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index) 660 { 661 struct sfxge_evq *evq; 662 663 evq = sc->evq[index]; 664 665 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 666 ("evq->init_state != SFXGE_EVQ_STARTED")); 667 668 SFXGE_EVQ_LOCK(evq); 669 evq->init_state = SFXGE_EVQ_INITIALIZED; 670 evq->read_ptr = 0; 671 evq->exception = B_FALSE; 672 673 #if EFSYS_OPT_QSTATS 674 /* Add event counts before discarding the common evq state */ 675 efx_ev_qstats_update(evq->common, sc->ev_stats); 676 #endif 677 678 efx_ev_qdestroy(evq->common); 679 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 680 EFX_EVQ_NBUFS(evq->entries)); 681 SFXGE_EVQ_UNLOCK(evq); 682 } 683 684 static int 685 sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index) 686 { 687 struct sfxge_evq *evq; 688 efsys_mem_t *esmp; 689 int count; 690 int rc; 691 692 evq = sc->evq[index]; 693 esmp = &evq->mem; 694 695 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 696 ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 697 698 /* Clear all events. */ 699 (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries)); 700 701 /* Program the buffer table. */ 702 if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp, 703 EFX_EVQ_NBUFS(evq->entries))) != 0) 704 return (rc); 705 706 /* Create the common code event queue. */ 707 if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries, 708 evq->buf_base_id, sc->ev_moderation, EFX_EVQ_FLAGS_TYPE_AUTO, 709 &evq->common)) != 0) 710 goto fail; 711 712 SFXGE_EVQ_LOCK(evq); 713 714 /* Prime the event queue for interrupts */ 715 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 716 goto fail2; 717 718 evq->init_state = SFXGE_EVQ_STARTING; 719 720 SFXGE_EVQ_UNLOCK(evq); 721 722 /* Wait for the initialization event */ 723 count = 0; 724 do { 725 /* Pause for 100 ms */ 726 pause("sfxge evq init", hz / 10); 727 728 /* Check to see if the test event has been processed */ 729 if (evq->init_state == SFXGE_EVQ_STARTED) 730 goto done; 731 732 } while (++count < 20); 733 734 rc = ETIMEDOUT; 735 goto fail3; 736 737 done: 738 return (0); 739 740 fail3: 741 SFXGE_EVQ_LOCK(evq); 742 evq->init_state = SFXGE_EVQ_INITIALIZED; 743 fail2: 744 SFXGE_EVQ_UNLOCK(evq); 745 efx_ev_qdestroy(evq->common); 746 fail: 747 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 748 EFX_EVQ_NBUFS(evq->entries)); 749 750 return (rc); 751 } 752 753 void 754 sfxge_ev_stop(struct sfxge_softc *sc) 755 { 756 struct sfxge_intr *intr; 757 efx_nic_t *enp; 758 int index; 759 760 intr = &sc->intr; 761 enp = sc->enp; 762 763 KASSERT(intr->state == SFXGE_INTR_STARTED, 764 ("Interrupts not started")); 765 766 /* Stop the event queue(s) */ 767 index = sc->evq_count; 768 while (--index >= 0) 769 sfxge_ev_qstop(sc, index); 770 771 /* Tear down the event module */ 772 efx_ev_fini(enp); 773 } 774 775 int 776 sfxge_ev_start(struct sfxge_softc *sc) 777 { 778 struct sfxge_intr *intr; 779 int index; 780 int rc; 781 782 intr = &sc->intr; 783 784 KASSERT(intr->state == SFXGE_INTR_STARTED, 785 ("intr->state != SFXGE_INTR_STARTED")); 786 787 /* Initialize the event module */ 788 if ((rc = efx_ev_init(sc->enp)) != 0) 789 return (rc); 790 791 /* Start the event queues */ 792 for (index = 0; index < sc->evq_count; index++) { 793 if ((rc = sfxge_ev_qstart(sc, index)) != 0) 794 goto fail; 795 } 796 797 return (0); 798 799 fail: 800 /* Stop the event queue(s) */ 801 while (--index >= 0) 802 sfxge_ev_qstop(sc, index); 803 804 /* Tear down the event module */ 805 efx_ev_fini(sc->enp); 806 807 return (rc); 808 } 809 810 static void 811 sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index) 812 { 813 struct sfxge_evq *evq; 814 815 evq = sc->evq[index]; 816 817 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 818 ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 819 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 820 821 sfxge_dma_free(&evq->mem); 822 823 sc->evq[index] = NULL; 824 825 SFXGE_EVQ_LOCK_DESTROY(evq); 826 827 free(evq, M_SFXGE); 828 } 829 830 static int 831 sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index) 832 { 833 struct sfxge_evq *evq; 834 efsys_mem_t *esmp; 835 int rc; 836 837 KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX")); 838 839 evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK); 840 evq->sc = sc; 841 evq->index = index; 842 sc->evq[index] = evq; 843 esmp = &evq->mem; 844 845 /* Build an event queue with room for one event per tx and rx buffer, 846 * plus some extra for link state events and MCDI completions. 847 * There are three tx queues in the first event queue and one in 848 * other. 849 */ 850 if (index == 0) 851 evq->entries = 852 ROUNDUP_POW_OF_TWO(sc->rxq_entries + 853 3 * sc->txq_entries + 854 128); 855 else 856 evq->entries = 857 ROUNDUP_POW_OF_TWO(sc->rxq_entries + 858 sc->txq_entries + 859 128); 860 861 /* Initialise TX completion list */ 862 evq->txqs = &evq->txq; 863 864 /* Allocate DMA space. */ 865 if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0) 866 return (rc); 867 868 /* Allocate buffer table entries. */ 869 sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries), 870 &evq->buf_base_id); 871 872 SFXGE_EVQ_LOCK_INIT(evq, device_get_nameunit(sc->dev), index); 873 874 evq->init_state = SFXGE_EVQ_INITIALIZED; 875 876 return (0); 877 } 878 879 void 880 sfxge_ev_fini(struct sfxge_softc *sc) 881 { 882 struct sfxge_intr *intr; 883 int index; 884 885 intr = &sc->intr; 886 887 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 888 ("intr->state != SFXGE_INTR_INITIALIZED")); 889 890 sc->ev_moderation = 0; 891 892 /* Tear down the event queue(s). */ 893 index = sc->evq_count; 894 while (--index >= 0) 895 sfxge_ev_qfini(sc, index); 896 897 sc->evq_count = 0; 898 } 899 900 int 901 sfxge_ev_init(struct sfxge_softc *sc) 902 { 903 struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev); 904 struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev); 905 struct sfxge_intr *intr; 906 int index; 907 int rc; 908 909 intr = &sc->intr; 910 911 sc->evq_count = intr->n_alloc; 912 913 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 914 ("intr->state != SFXGE_INTR_INITIALIZED")); 915 916 /* Set default interrupt moderation; add a sysctl to 917 * read and change it. 918 */ 919 sc->ev_moderation = SFXGE_MODERATION; 920 SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 921 OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW, 922 sc, 0, sfxge_int_mod_handler, "IU", 923 "sfxge interrupt moderation (us)"); 924 925 /* 926 * Initialize the event queue(s) - one per interrupt. 927 */ 928 for (index = 0; index < sc->evq_count; index++) { 929 if ((rc = sfxge_ev_qinit(sc, index)) != 0) 930 goto fail; 931 } 932 933 #if EFSYS_OPT_QSTATS 934 sfxge_ev_stat_init(sc); 935 #endif 936 937 return (0); 938 939 fail: 940 while (--index >= 0) 941 sfxge_ev_qfini(sc, index); 942 943 sc->evq_count = 0; 944 return (rc); 945 } 946