1 /*- 2 * Copyright (c) 2010-2011 Solarflare Communications, Inc. 3 * All rights reserved. 4 * 5 * This software was developed in part by Philip Paeps under contract for 6 * Solarflare Communications, Inc. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include <sys/param.h> 34 #include <sys/kernel.h> 35 #include <sys/malloc.h> 36 #include <sys/param.h> 37 #include <sys/queue.h> 38 #include <sys/systm.h> 39 #include <sys/taskqueue.h> 40 41 #include "common/efx.h" 42 43 #include "sfxge.h" 44 45 static void 46 sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop) 47 { 48 struct sfxge_softc *sc; 49 unsigned int index; 50 struct sfxge_rxq *rxq; 51 struct sfxge_txq *txq; 52 53 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 54 55 sc = evq->sc; 56 index = evq->index; 57 rxq = sc->rxq[index]; 58 59 if ((txq = evq->txq) != NULL) { 60 evq->txq = NULL; 61 evq->txqs = &(evq->txq); 62 63 do { 64 struct sfxge_txq *next; 65 66 next = txq->next; 67 txq->next = NULL; 68 69 KASSERT(txq->evq_index == index, 70 ("txq->evq_index != index")); 71 72 if (txq->pending != txq->completed) 73 sfxge_tx_qcomplete(txq, evq); 74 75 txq = next; 76 } while (txq != NULL); 77 } 78 79 if (rxq->pending != rxq->completed) 80 sfxge_rx_qcomplete(rxq, eop); 81 } 82 83 static boolean_t 84 sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size, 85 uint16_t flags) 86 { 87 struct sfxge_evq *evq; 88 struct sfxge_softc *sc; 89 struct sfxge_rxq *rxq; 90 unsigned int expected; 91 struct sfxge_rx_sw_desc *rx_desc; 92 93 evq = arg; 94 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 95 96 sc = evq->sc; 97 98 if (evq->exception) 99 goto done; 100 101 rxq = sc->rxq[label]; 102 KASSERT(rxq != NULL, ("rxq == NULL")); 103 KASSERT(evq->index == rxq->index, 104 ("evq->index != rxq->index")); 105 106 if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED)) 107 goto done; 108 109 expected = rxq->pending++ & rxq->ptr_mask; 110 if (id != expected) { 111 evq->exception = B_TRUE; 112 113 device_printf(sc->dev, "RX completion out of order" 114 " (id=%#x expected=%#x flags=%#x); resetting\n", 115 id, expected, flags); 116 sfxge_schedule_reset(sc); 117 118 goto done; 119 } 120 121 rx_desc = &rxq->queue[id]; 122 123 KASSERT(rx_desc->flags == EFX_DISCARD, 124 ("rx_desc->flags != EFX_DISCARD")); 125 rx_desc->flags = flags; 126 127 KASSERT(size < (1 << 16), ("size > (1 << 16)")); 128 rx_desc->size = (uint16_t)size; 129 prefetch_read_many(rx_desc->mbuf); 130 131 evq->rx_done++; 132 133 if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH) 134 sfxge_ev_qcomplete(evq, B_FALSE); 135 136 done: 137 return (evq->rx_done >= SFXGE_EV_BATCH); 138 } 139 140 static boolean_t 141 sfxge_ev_exception(void *arg, uint32_t code, uint32_t data) 142 { 143 struct sfxge_evq *evq; 144 struct sfxge_softc *sc; 145 146 evq = (struct sfxge_evq *)arg; 147 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 148 149 sc = evq->sc; 150 151 evq->exception = B_TRUE; 152 153 if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) { 154 device_printf(sc->dev, 155 "hardware exception (code=%u); resetting\n", 156 code); 157 sfxge_schedule_reset(sc); 158 } 159 160 return (B_FALSE); 161 } 162 163 static boolean_t 164 sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index) 165 { 166 struct sfxge_evq *evq; 167 struct sfxge_softc *sc; 168 struct sfxge_rxq *rxq; 169 unsigned int index; 170 unsigned int label; 171 uint16_t magic; 172 173 evq = (struct sfxge_evq *)arg; 174 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 175 176 sc = evq->sc; 177 rxq = sc->rxq[rxq_index]; 178 179 KASSERT(rxq != NULL, ("rxq == NULL")); 180 181 /* Resend a software event on the correct queue */ 182 index = rxq->index; 183 evq = sc->evq[index]; 184 185 label = rxq_index; 186 KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 187 ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != level")); 188 magic = SFXGE_MAGIC_RX_QFLUSH_DONE | label; 189 190 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 191 ("evq not started")); 192 efx_ev_qpost(evq->common, magic); 193 194 return (B_FALSE); 195 } 196 197 static boolean_t 198 sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index) 199 { 200 struct sfxge_evq *evq; 201 struct sfxge_softc *sc; 202 struct sfxge_rxq *rxq; 203 unsigned int index; 204 unsigned int label; 205 uint16_t magic; 206 207 evq = (struct sfxge_evq *)arg; 208 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 209 210 sc = evq->sc; 211 rxq = sc->rxq[rxq_index]; 212 213 KASSERT(rxq != NULL, ("rxq == NULL")); 214 215 /* Resend a software event on the correct queue */ 216 index = rxq->index; 217 evq = sc->evq[index]; 218 219 label = rxq_index; 220 KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 221 ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label")); 222 magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label; 223 224 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 225 ("evq not started")); 226 efx_ev_qpost(evq->common, magic); 227 228 return (B_FALSE); 229 } 230 231 static struct sfxge_txq * 232 sfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label) 233 { 234 unsigned int index; 235 236 KASSERT((evq->index == 0 && label < SFXGE_TXQ_NTYPES) || 237 (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM), ("unexpected txq label")); 238 index = (evq->index == 0) ? label : (evq->index - 1 + SFXGE_TXQ_NTYPES); 239 return (evq->sc->txq[index]); 240 } 241 242 static boolean_t 243 sfxge_ev_tx(void *arg, uint32_t label, uint32_t id) 244 { 245 struct sfxge_evq *evq; 246 struct sfxge_txq *txq; 247 unsigned int stop; 248 unsigned int delta; 249 250 evq = (struct sfxge_evq *)arg; 251 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 252 253 txq = sfxge_get_txq_by_label(evq, label); 254 255 KASSERT(txq != NULL, ("txq == NULL")); 256 KASSERT(evq->index == txq->evq_index, 257 ("evq->index != txq->evq_index")); 258 259 if (__predict_false(txq->init_state != SFXGE_TXQ_STARTED)) 260 goto done; 261 262 stop = (id + 1) & txq->ptr_mask; 263 id = txq->pending & txq->ptr_mask; 264 265 delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop); 266 txq->pending += delta; 267 268 evq->tx_done++; 269 270 if (txq->next == NULL && 271 evq->txqs != &(txq->next)) { 272 *(evq->txqs) = txq; 273 evq->txqs = &(txq->next); 274 } 275 276 if (txq->pending - txq->completed >= SFXGE_TX_BATCH) 277 sfxge_tx_qcomplete(txq, evq); 278 279 done: 280 return (evq->tx_done >= SFXGE_EV_BATCH); 281 } 282 283 static boolean_t 284 sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index) 285 { 286 struct sfxge_evq *evq; 287 struct sfxge_softc *sc; 288 struct sfxge_txq *txq; 289 unsigned int label; 290 uint16_t magic; 291 292 evq = (struct sfxge_evq *)arg; 293 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 294 295 sc = evq->sc; 296 txq = sc->txq[txq_index]; 297 298 KASSERT(txq != NULL, ("txq == NULL")); 299 KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED, 300 ("txq not initialized")); 301 302 /* Resend a software event on the correct queue */ 303 evq = sc->evq[txq->evq_index]; 304 305 label = txq->type; 306 KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 307 ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label")); 308 magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label; 309 310 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 311 ("evq not started")); 312 efx_ev_qpost(evq->common, magic); 313 314 return (B_FALSE); 315 } 316 317 static boolean_t 318 sfxge_ev_software(void *arg, uint16_t magic) 319 { 320 struct sfxge_evq *evq; 321 struct sfxge_softc *sc; 322 unsigned int label; 323 324 evq = (struct sfxge_evq *)arg; 325 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 326 327 sc = evq->sc; 328 329 label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK; 330 magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK; 331 332 switch (magic) { 333 case SFXGE_MAGIC_RX_QFLUSH_DONE: { 334 struct sfxge_rxq *rxq = sc->rxq[label]; 335 336 KASSERT(rxq != NULL, ("rxq == NULL")); 337 KASSERT(evq->index == rxq->index, 338 ("evq->index != rxq->index")); 339 340 sfxge_rx_qflush_done(rxq); 341 break; 342 } 343 case SFXGE_MAGIC_RX_QFLUSH_FAILED: { 344 struct sfxge_rxq *rxq = sc->rxq[label]; 345 346 KASSERT(rxq != NULL, ("rxq == NULL")); 347 KASSERT(evq->index == rxq->index, 348 ("evq->index != rxq->index")); 349 350 sfxge_rx_qflush_failed(rxq); 351 break; 352 } 353 case SFXGE_MAGIC_RX_QREFILL: { 354 struct sfxge_rxq *rxq = sc->rxq[label]; 355 356 KASSERT(rxq != NULL, ("rxq == NULL")); 357 KASSERT(evq->index == rxq->index, 358 ("evq->index != rxq->index")); 359 360 sfxge_rx_qrefill(rxq); 361 break; 362 } 363 case SFXGE_MAGIC_TX_QFLUSH_DONE: { 364 struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label); 365 366 KASSERT(txq != NULL, ("txq == NULL")); 367 KASSERT(evq->index == txq->evq_index, 368 ("evq->index != txq->evq_index")); 369 370 sfxge_tx_qflush_done(txq); 371 break; 372 } 373 default: 374 break; 375 } 376 377 return (B_FALSE); 378 } 379 380 static boolean_t 381 sfxge_ev_sram(void *arg, uint32_t code) 382 { 383 (void)arg; 384 (void)code; 385 386 switch (code) { 387 case EFX_SRAM_UPDATE: 388 EFSYS_PROBE(sram_update); 389 break; 390 391 case EFX_SRAM_CLEAR: 392 EFSYS_PROBE(sram_clear); 393 break; 394 395 case EFX_SRAM_ILLEGAL_CLEAR: 396 EFSYS_PROBE(sram_illegal_clear); 397 break; 398 399 default: 400 KASSERT(B_FALSE, ("Impossible SRAM event")); 401 break; 402 } 403 404 return (B_FALSE); 405 } 406 407 static boolean_t 408 sfxge_ev_timer(void *arg, uint32_t index) 409 { 410 (void)arg; 411 (void)index; 412 413 return (B_FALSE); 414 } 415 416 static boolean_t 417 sfxge_ev_wake_up(void *arg, uint32_t index) 418 { 419 (void)arg; 420 (void)index; 421 422 return (B_FALSE); 423 } 424 425 #if EFSYS_OPT_QSTATS 426 427 static void 428 sfxge_ev_stat_update(struct sfxge_softc *sc) 429 { 430 struct sfxge_evq *evq; 431 unsigned int index; 432 clock_t now; 433 434 SFXGE_ADAPTER_LOCK(sc); 435 436 if (__predict_false(sc->evq[0]->init_state != SFXGE_EVQ_STARTED)) 437 goto out; 438 439 now = ticks; 440 if (now - sc->ev_stats_update_time < hz) 441 goto out; 442 443 sc->ev_stats_update_time = now; 444 445 /* Add event counts from each event queue in turn */ 446 for (index = 0; index < sc->evq_count; index++) { 447 evq = sc->evq[index]; 448 SFXGE_EVQ_LOCK(evq); 449 efx_ev_qstats_update(evq->common, sc->ev_stats); 450 SFXGE_EVQ_UNLOCK(evq); 451 } 452 out: 453 SFXGE_ADAPTER_UNLOCK(sc); 454 } 455 456 static int 457 sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS) 458 { 459 struct sfxge_softc *sc = arg1; 460 unsigned int id = arg2; 461 462 sfxge_ev_stat_update(sc); 463 464 return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id]))); 465 } 466 467 static void 468 sfxge_ev_stat_init(struct sfxge_softc *sc) 469 { 470 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); 471 struct sysctl_oid_list *stat_list; 472 unsigned int id; 473 char name[40]; 474 475 stat_list = SYSCTL_CHILDREN(sc->stats_node); 476 477 for (id = 0; id < EV_NQSTATS; id++) { 478 snprintf(name, sizeof(name), "ev_%s", 479 efx_ev_qstat_name(sc->enp, id)); 480 SYSCTL_ADD_PROC( 481 ctx, stat_list, 482 OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD, 483 sc, id, sfxge_ev_stat_handler, "Q", 484 ""); 485 } 486 } 487 488 #endif /* EFSYS_OPT_QSTATS */ 489 490 static void 491 sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us) 492 { 493 struct sfxge_evq *evq; 494 efx_evq_t *eep; 495 496 evq = sc->evq[idx]; 497 eep = evq->common; 498 499 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 500 ("evq->init_state != SFXGE_EVQ_STARTED")); 501 502 (void)efx_ev_qmoderate(eep, us); 503 } 504 505 static int 506 sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS) 507 { 508 struct sfxge_softc *sc = arg1; 509 struct sfxge_intr *intr = &sc->intr; 510 unsigned int moderation; 511 int error; 512 unsigned int index; 513 514 SFXGE_ADAPTER_LOCK(sc); 515 516 if (req->newptr != NULL) { 517 if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation))) 518 != 0) 519 goto out; 520 521 /* We may not be calling efx_ev_qmoderate() now, 522 * so we have to range-check the value ourselves. 523 */ 524 if (moderation > 525 efx_nic_cfg_get(sc->enp)->enc_evq_timer_max_us) { 526 error = EINVAL; 527 goto out; 528 } 529 530 sc->ev_moderation = moderation; 531 if (intr->state == SFXGE_INTR_STARTED) { 532 for (index = 0; index < sc->evq_count; index++) 533 sfxge_ev_qmoderate(sc, index, moderation); 534 } 535 } else { 536 error = SYSCTL_OUT(req, &sc->ev_moderation, 537 sizeof(sc->ev_moderation)); 538 } 539 540 out: 541 SFXGE_ADAPTER_UNLOCK(sc); 542 543 return (error); 544 } 545 546 static boolean_t 547 sfxge_ev_initialized(void *arg) 548 { 549 struct sfxge_evq *evq; 550 551 evq = (struct sfxge_evq *)arg; 552 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 553 554 KASSERT(evq->init_state == SFXGE_EVQ_STARTING, 555 ("evq not starting")); 556 557 evq->init_state = SFXGE_EVQ_STARTED; 558 559 return (0); 560 } 561 562 static boolean_t 563 sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode) 564 { 565 struct sfxge_evq *evq; 566 struct sfxge_softc *sc; 567 568 evq = (struct sfxge_evq *)arg; 569 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 570 571 sc = evq->sc; 572 573 sfxge_mac_link_update(sc, link_mode); 574 575 return (0); 576 } 577 578 static const efx_ev_callbacks_t sfxge_ev_callbacks = { 579 .eec_initialized = sfxge_ev_initialized, 580 .eec_rx = sfxge_ev_rx, 581 .eec_tx = sfxge_ev_tx, 582 .eec_exception = sfxge_ev_exception, 583 .eec_rxq_flush_done = sfxge_ev_rxq_flush_done, 584 .eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed, 585 .eec_txq_flush_done = sfxge_ev_txq_flush_done, 586 .eec_software = sfxge_ev_software, 587 .eec_sram = sfxge_ev_sram, 588 .eec_wake_up = sfxge_ev_wake_up, 589 .eec_timer = sfxge_ev_timer, 590 .eec_link_change = sfxge_ev_link_change, 591 }; 592 593 594 int 595 sfxge_ev_qpoll(struct sfxge_evq *evq) 596 { 597 int rc; 598 599 SFXGE_EVQ_LOCK(evq); 600 601 if (__predict_false(evq->init_state != SFXGE_EVQ_STARTING && 602 evq->init_state != SFXGE_EVQ_STARTED)) { 603 rc = EINVAL; 604 goto fail; 605 } 606 607 /* Synchronize the DMA memory for reading */ 608 bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map, 609 BUS_DMASYNC_POSTREAD); 610 611 KASSERT(evq->rx_done == 0, ("evq->rx_done != 0")); 612 KASSERT(evq->tx_done == 0, ("evq->tx_done != 0")); 613 KASSERT(evq->txq == NULL, ("evq->txq != NULL")); 614 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 615 616 /* Poll the queue */ 617 efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq); 618 619 evq->rx_done = 0; 620 evq->tx_done = 0; 621 622 /* Perform any pending completion processing */ 623 sfxge_ev_qcomplete(evq, B_TRUE); 624 625 /* Re-prime the event queue for interrupts */ 626 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 627 goto fail; 628 629 SFXGE_EVQ_UNLOCK(evq); 630 631 return (0); 632 633 fail: 634 SFXGE_EVQ_UNLOCK(evq); 635 return (rc); 636 } 637 638 static void 639 sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index) 640 { 641 struct sfxge_evq *evq; 642 643 evq = sc->evq[index]; 644 645 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 646 ("evq->init_state != SFXGE_EVQ_STARTED")); 647 648 SFXGE_EVQ_LOCK(evq); 649 evq->init_state = SFXGE_EVQ_INITIALIZED; 650 evq->read_ptr = 0; 651 evq->exception = B_FALSE; 652 653 #if EFSYS_OPT_QSTATS 654 /* Add event counts before discarding the common evq state */ 655 efx_ev_qstats_update(evq->common, sc->ev_stats); 656 #endif 657 658 efx_ev_qdestroy(evq->common); 659 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 660 EFX_EVQ_NBUFS(evq->entries)); 661 SFXGE_EVQ_UNLOCK(evq); 662 } 663 664 static int 665 sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index) 666 { 667 struct sfxge_evq *evq; 668 efsys_mem_t *esmp; 669 int count; 670 int rc; 671 672 evq = sc->evq[index]; 673 esmp = &evq->mem; 674 675 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 676 ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 677 678 /* Clear all events. */ 679 (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries)); 680 681 /* Program the buffer table. */ 682 if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp, 683 EFX_EVQ_NBUFS(evq->entries))) != 0) 684 return (rc); 685 686 /* Create the common code event queue. */ 687 if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries, 688 evq->buf_base_id, &evq->common)) != 0) 689 goto fail; 690 691 SFXGE_EVQ_LOCK(evq); 692 693 /* Set the default moderation */ 694 (void)efx_ev_qmoderate(evq->common, sc->ev_moderation); 695 696 /* Prime the event queue for interrupts */ 697 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 698 goto fail2; 699 700 evq->init_state = SFXGE_EVQ_STARTING; 701 702 SFXGE_EVQ_UNLOCK(evq); 703 704 /* Wait for the initialization event */ 705 count = 0; 706 do { 707 /* Pause for 100 ms */ 708 pause("sfxge evq init", hz / 10); 709 710 /* Check to see if the test event has been processed */ 711 if (evq->init_state == SFXGE_EVQ_STARTED) 712 goto done; 713 714 } while (++count < 20); 715 716 rc = ETIMEDOUT; 717 goto fail3; 718 719 done: 720 return (0); 721 722 fail3: 723 SFXGE_EVQ_LOCK(evq); 724 evq->init_state = SFXGE_EVQ_INITIALIZED; 725 fail2: 726 SFXGE_EVQ_UNLOCK(evq); 727 efx_ev_qdestroy(evq->common); 728 fail: 729 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 730 EFX_EVQ_NBUFS(evq->entries)); 731 732 return (rc); 733 } 734 735 void 736 sfxge_ev_stop(struct sfxge_softc *sc) 737 { 738 struct sfxge_intr *intr; 739 efx_nic_t *enp; 740 int index; 741 742 intr = &sc->intr; 743 enp = sc->enp; 744 745 KASSERT(intr->state == SFXGE_INTR_STARTED, 746 ("Interrupts not started")); 747 748 /* Stop the event queue(s) */ 749 index = sc->evq_count; 750 while (--index >= 0) 751 sfxge_ev_qstop(sc, index); 752 753 /* Tear down the event module */ 754 efx_ev_fini(enp); 755 } 756 757 int 758 sfxge_ev_start(struct sfxge_softc *sc) 759 { 760 struct sfxge_intr *intr; 761 int index; 762 int rc; 763 764 intr = &sc->intr; 765 766 KASSERT(intr->state == SFXGE_INTR_STARTED, 767 ("intr->state != SFXGE_INTR_STARTED")); 768 769 /* Initialize the event module */ 770 if ((rc = efx_ev_init(sc->enp)) != 0) 771 return (rc); 772 773 /* Start the event queues */ 774 for (index = 0; index < sc->evq_count; index++) { 775 if ((rc = sfxge_ev_qstart(sc, index)) != 0) 776 goto fail; 777 } 778 779 return (0); 780 781 fail: 782 /* Stop the event queue(s) */ 783 while (--index >= 0) 784 sfxge_ev_qstop(sc, index); 785 786 /* Tear down the event module */ 787 efx_ev_fini(sc->enp); 788 789 return (rc); 790 } 791 792 static void 793 sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index) 794 { 795 struct sfxge_evq *evq; 796 797 evq = sc->evq[index]; 798 799 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 800 ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 801 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 802 803 sfxge_dma_free(&evq->mem); 804 805 sc->evq[index] = NULL; 806 807 SFXGE_EVQ_LOCK_DESTROY(evq); 808 809 free(evq, M_SFXGE); 810 } 811 812 static int 813 sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index) 814 { 815 struct sfxge_evq *evq; 816 efsys_mem_t *esmp; 817 int rc; 818 819 KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX")); 820 821 evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK); 822 evq->sc = sc; 823 evq->index = index; 824 sc->evq[index] = evq; 825 esmp = &evq->mem; 826 827 /* Build an event queue with room for one event per tx and rx buffer, 828 * plus some extra for link state events and MCDI completions. 829 * There are three tx queues in the first event queue and one in 830 * other. 831 */ 832 if (index == 0) 833 evq->entries = 834 ROUNDUP_POW_OF_TWO(sc->rxq_entries + 835 3 * sc->txq_entries + 836 128); 837 else 838 evq->entries = 839 ROUNDUP_POW_OF_TWO(sc->rxq_entries + 840 sc->txq_entries + 841 128); 842 843 /* Initialise TX completion list */ 844 evq->txqs = &evq->txq; 845 846 /* Allocate DMA space. */ 847 if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0) 848 return (rc); 849 850 /* Allocate buffer table entries. */ 851 sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries), 852 &evq->buf_base_id); 853 854 SFXGE_EVQ_LOCK_INIT(evq, device_get_nameunit(sc->dev), index); 855 856 evq->init_state = SFXGE_EVQ_INITIALIZED; 857 858 return (0); 859 } 860 861 void 862 sfxge_ev_fini(struct sfxge_softc *sc) 863 { 864 struct sfxge_intr *intr; 865 int index; 866 867 intr = &sc->intr; 868 869 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 870 ("intr->state != SFXGE_INTR_INITIALIZED")); 871 872 sc->ev_moderation = 0; 873 874 /* Tear down the event queue(s). */ 875 index = sc->evq_count; 876 while (--index >= 0) 877 sfxge_ev_qfini(sc, index); 878 879 sc->evq_count = 0; 880 } 881 882 int 883 sfxge_ev_init(struct sfxge_softc *sc) 884 { 885 struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev); 886 struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev); 887 struct sfxge_intr *intr; 888 int index; 889 int rc; 890 891 intr = &sc->intr; 892 893 sc->evq_count = intr->n_alloc; 894 895 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 896 ("intr->state != SFXGE_INTR_INITIALIZED")); 897 898 /* Set default interrupt moderation; add a sysctl to 899 * read and change it. 900 */ 901 sc->ev_moderation = SFXGE_MODERATION; 902 SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 903 OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW, 904 sc, 0, sfxge_int_mod_handler, "IU", 905 "sfxge interrupt moderation (us)"); 906 907 /* 908 * Initialize the event queue(s) - one per interrupt. 909 */ 910 for (index = 0; index < sc->evq_count; index++) { 911 if ((rc = sfxge_ev_qinit(sc, index)) != 0) 912 goto fail; 913 } 914 915 #if EFSYS_OPT_QSTATS 916 sfxge_ev_stat_init(sc); 917 #endif 918 919 return (0); 920 921 fail: 922 while (--index >= 0) 923 sfxge_ev_qfini(sc, index); 924 925 sc->evq_count = 0; 926 return (rc); 927 } 928