16f443ebcSRyan Zezeski /* 26f443ebcSRyan Zezeski * This file and its contents are supplied under the terms of the 36f443ebcSRyan Zezeski * Common Development and Distribution License ("CDDL"), version 1.0. 46f443ebcSRyan Zezeski * You may only use this file in accordance with the terms of version 56f443ebcSRyan Zezeski * 1.0 of the CDDL. 66f443ebcSRyan Zezeski * 76f443ebcSRyan Zezeski * A full copy of the text of the CDDL should have accompanied this 86f443ebcSRyan Zezeski * source. A copy of the CDDL is also available via the Internet at 96f443ebcSRyan Zezeski * http://www.illumos.org/license/CDDL. 106f443ebcSRyan Zezeski */ 116f443ebcSRyan Zezeski 126f443ebcSRyan Zezeski /* 13*eebd18daSAndy Fiddaman * Copyright 2024 Oxide Computer Company 146f443ebcSRyan Zezeski */ 156f443ebcSRyan Zezeski #include "ena.h" 166f443ebcSRyan Zezeski 176f443ebcSRyan Zezeski static void 186f443ebcSRyan Zezeski ena_refill_rx(ena_rxq_t *rxq, uint16_t num) 196f443ebcSRyan Zezeski { 206f443ebcSRyan Zezeski VERIFY3P(rxq, !=, NULL); 216f443ebcSRyan Zezeski ASSERT(MUTEX_HELD(&rxq->er_lock)); 226f443ebcSRyan Zezeski ASSERT3U(num, <=, rxq->er_sq_num_descs); 236f443ebcSRyan Zezeski uint16_t tail_mod = rxq->er_sq_tail_idx & (rxq->er_sq_num_descs - 1); 246f443ebcSRyan Zezeski 256f443ebcSRyan Zezeski while (num != 0) { 266f443ebcSRyan Zezeski enahw_rx_desc_t *desc = &rxq->er_sq_descs[tail_mod]; 276f443ebcSRyan Zezeski ena_rx_ctrl_block_t *rcb = &rxq->er_rcbs[tail_mod]; 286f443ebcSRyan Zezeski uint16_t phase = rxq->er_sq_phase; 296f443ebcSRyan Zezeski 306f443ebcSRyan Zezeski VERIFY3U(tail_mod, <, rxq->er_sq_num_descs); 316f443ebcSRyan Zezeski VERIFY3P(desc, !=, NULL); 326f443ebcSRyan Zezeski VERIFY3P(rcb, !=, NULL); 336f443ebcSRyan Zezeski VERIFY3P(desc, >=, rxq->er_sq_descs); 346f443ebcSRyan Zezeski VERIFY3P(desc, <=, 356f443ebcSRyan Zezeski (rxq->er_sq_descs + rxq->er_sq_num_descs - 1)); 366f443ebcSRyan Zezeski 376f443ebcSRyan Zezeski desc->erd_length = rcb->ercb_dma.edb_len; 386f443ebcSRyan Zezeski desc->erd_req_id = tail_mod; 396f443ebcSRyan Zezeski VERIFY3P(rcb->ercb_dma.edb_cookie, !=, NULL); 406f443ebcSRyan Zezeski ena_set_dma_addr_values(rxq->er_ena, 416f443ebcSRyan Zezeski rcb->ercb_dma.edb_cookie->dmac_laddress, 426f443ebcSRyan Zezeski &desc->erd_buff_addr_lo, &desc->erd_buff_addr_hi); 43*eebd18daSAndy Fiddaman 44*eebd18daSAndy Fiddaman ENAHW_RX_DESC_CLEAR_CTRL(desc); 456f443ebcSRyan Zezeski ENAHW_RX_DESC_SET_PHASE(desc, phase); 466f443ebcSRyan Zezeski ENAHW_RX_DESC_SET_FIRST(desc); 476f443ebcSRyan Zezeski ENAHW_RX_DESC_SET_LAST(desc); 486f443ebcSRyan Zezeski ENAHW_RX_DESC_SET_COMP_REQ(desc); 496f443ebcSRyan Zezeski DTRACE_PROBE1(ena__refill__rx, enahw_rx_desc_t *, desc); 506f443ebcSRyan Zezeski rxq->er_sq_tail_idx++; 516f443ebcSRyan Zezeski tail_mod = rxq->er_sq_tail_idx & (rxq->er_sq_num_descs - 1); 526f443ebcSRyan Zezeski 536f443ebcSRyan Zezeski if (tail_mod == 0) { 54*eebd18daSAndy Fiddaman rxq->er_sq_phase ^= 1; 556f443ebcSRyan Zezeski } 566f443ebcSRyan Zezeski 576f443ebcSRyan Zezeski num--; 586f443ebcSRyan Zezeski } 596f443ebcSRyan Zezeski 606f443ebcSRyan Zezeski ENA_DMA_SYNC(rxq->er_sq_dma, DDI_DMA_SYNC_FORDEV); 616f443ebcSRyan Zezeski ena_hw_abs_write32(rxq->er_ena, rxq->er_sq_db_addr, 626f443ebcSRyan Zezeski rxq->er_sq_tail_idx); 636f443ebcSRyan Zezeski } 646f443ebcSRyan Zezeski 656f443ebcSRyan Zezeski void 666f443ebcSRyan Zezeski ena_free_rx_dma(ena_rxq_t *rxq) 676f443ebcSRyan Zezeski { 686f443ebcSRyan Zezeski if (rxq->er_rcbs != NULL) { 696f443ebcSRyan Zezeski for (uint_t i = 0; i < rxq->er_sq_num_descs; i++) { 706f443ebcSRyan Zezeski ena_rx_ctrl_block_t *rcb = &rxq->er_rcbs[i]; 716f443ebcSRyan Zezeski ena_dma_free(&rcb->ercb_dma); 726f443ebcSRyan Zezeski } 736f443ebcSRyan Zezeski 746f443ebcSRyan Zezeski kmem_free(rxq->er_rcbs, 756f443ebcSRyan Zezeski sizeof (*rxq->er_rcbs) * rxq->er_sq_num_descs); 766f443ebcSRyan Zezeski 776f443ebcSRyan Zezeski rxq->er_rcbs = NULL; 786f443ebcSRyan Zezeski } 796f443ebcSRyan Zezeski 806f443ebcSRyan Zezeski ena_dma_free(&rxq->er_cq_dma); 816f443ebcSRyan Zezeski rxq->er_cq_descs = NULL; 826f443ebcSRyan Zezeski rxq->er_cq_num_descs = 0; 836f443ebcSRyan Zezeski 846f443ebcSRyan Zezeski ena_dma_free(&rxq->er_sq_dma); 856f443ebcSRyan Zezeski rxq->er_sq_descs = NULL; 866f443ebcSRyan Zezeski rxq->er_sq_num_descs = 0; 876f443ebcSRyan Zezeski 886f443ebcSRyan Zezeski rxq->er_state &= ~ENA_RXQ_STATE_HOST_ALLOC; 896f443ebcSRyan Zezeski } 906f443ebcSRyan Zezeski 916f443ebcSRyan Zezeski static int 926f443ebcSRyan Zezeski ena_alloc_rx_dma(ena_rxq_t *rxq) 936f443ebcSRyan Zezeski { 946f443ebcSRyan Zezeski ena_t *ena = rxq->er_ena; 956f443ebcSRyan Zezeski size_t cq_descs_sz; 966f443ebcSRyan Zezeski size_t sq_descs_sz; 976f443ebcSRyan Zezeski ena_dma_conf_t conf; 986f443ebcSRyan Zezeski int err = 0; 996f443ebcSRyan Zezeski 1006f443ebcSRyan Zezeski cq_descs_sz = rxq->er_cq_num_descs * sizeof (*rxq->er_cq_descs); 1016f443ebcSRyan Zezeski sq_descs_sz = rxq->er_sq_num_descs * sizeof (*rxq->er_sq_descs); 102*eebd18daSAndy Fiddaman /* BEGIN CSTYLED */ 1036f443ebcSRyan Zezeski conf = (ena_dma_conf_t) { 1046f443ebcSRyan Zezeski .edc_size = sq_descs_sz, 1056f443ebcSRyan Zezeski .edc_align = ENAHW_IO_SQ_DESC_BUF_ALIGNMENT, 1066f443ebcSRyan Zezeski .edc_sgl = 1, 1076f443ebcSRyan Zezeski .edc_endian = DDI_NEVERSWAP_ACC, 1086f443ebcSRyan Zezeski .edc_stream = B_FALSE, 1096f443ebcSRyan Zezeski }; 110*eebd18daSAndy Fiddaman /* END CSTYLED */ 1116f443ebcSRyan Zezeski 1126f443ebcSRyan Zezeski if (!ena_dma_alloc(ena, &rxq->er_sq_dma, &conf, sq_descs_sz)) { 1136f443ebcSRyan Zezeski return (ENOMEM); 1146f443ebcSRyan Zezeski } 1156f443ebcSRyan Zezeski 1166f443ebcSRyan Zezeski rxq->er_sq_descs = (void *)rxq->er_sq_dma.edb_va; 1176f443ebcSRyan Zezeski rxq->er_rcbs = kmem_zalloc(sizeof (*rxq->er_rcbs) * 1186f443ebcSRyan Zezeski rxq->er_sq_num_descs, KM_SLEEP); 1196f443ebcSRyan Zezeski 1206f443ebcSRyan Zezeski for (uint_t i = 0; i < rxq->er_sq_num_descs; i++) { 1216f443ebcSRyan Zezeski ena_rx_ctrl_block_t *rcb = &rxq->er_rcbs[i]; 1226f443ebcSRyan Zezeski ena_dma_conf_t buf_conf = { 1236f443ebcSRyan Zezeski .edc_size = ena->ena_rx_buf_sz, 1246f443ebcSRyan Zezeski .edc_align = 1, 1256f443ebcSRyan Zezeski .edc_sgl = ena->ena_rx_sgl_max_sz, 1266f443ebcSRyan Zezeski .edc_endian = DDI_NEVERSWAP_ACC, 1276f443ebcSRyan Zezeski .edc_stream = B_TRUE, 1286f443ebcSRyan Zezeski }; 1296f443ebcSRyan Zezeski 1306f443ebcSRyan Zezeski if (!ena_dma_alloc(ena, &rcb->ercb_dma, &buf_conf, 1316f443ebcSRyan Zezeski ena->ena_rx_buf_sz)) { 1326f443ebcSRyan Zezeski err = ENOMEM; 1336f443ebcSRyan Zezeski goto error; 1346f443ebcSRyan Zezeski } 1356f443ebcSRyan Zezeski } 1366f443ebcSRyan Zezeski 137*eebd18daSAndy Fiddaman /* BEGIN CSTYLED */ 1386f443ebcSRyan Zezeski conf = (ena_dma_conf_t) { 1396f443ebcSRyan Zezeski .edc_size = cq_descs_sz, 1406f443ebcSRyan Zezeski .edc_align = ENAHW_IO_CQ_DESC_BUF_ALIGNMENT, 1416f443ebcSRyan Zezeski .edc_sgl = 1, 1426f443ebcSRyan Zezeski .edc_endian = DDI_NEVERSWAP_ACC, 1436f443ebcSRyan Zezeski .edc_stream = B_FALSE, 1446f443ebcSRyan Zezeski }; 145*eebd18daSAndy Fiddaman /* END CSTYLED */ 1466f443ebcSRyan Zezeski 1476f443ebcSRyan Zezeski if (!ena_dma_alloc(ena, &rxq->er_cq_dma, &conf, cq_descs_sz)) { 1486f443ebcSRyan Zezeski err = ENOMEM; 1496f443ebcSRyan Zezeski goto error; 1506f443ebcSRyan Zezeski } 1516f443ebcSRyan Zezeski 1526f443ebcSRyan Zezeski rxq->er_cq_descs = (void *)rxq->er_cq_dma.edb_va; 1536f443ebcSRyan Zezeski rxq->er_state |= ENA_RXQ_STATE_HOST_ALLOC; 1546f443ebcSRyan Zezeski return (0); 1556f443ebcSRyan Zezeski 1566f443ebcSRyan Zezeski error: 1576f443ebcSRyan Zezeski ena_free_rx_dma(rxq); 1586f443ebcSRyan Zezeski return (err); 1596f443ebcSRyan Zezeski } 1606f443ebcSRyan Zezeski 1616f443ebcSRyan Zezeski boolean_t 1626f443ebcSRyan Zezeski ena_alloc_rxq(ena_rxq_t *rxq) 1636f443ebcSRyan Zezeski { 1646f443ebcSRyan Zezeski int ret = 0; 1656f443ebcSRyan Zezeski ena_t *ena = rxq->er_ena; 1666f443ebcSRyan Zezeski uint16_t cq_hw_idx, sq_hw_idx; 1676f443ebcSRyan Zezeski uint32_t *cq_unmask_addr, *cq_headdb, *cq_numanode; 1686f443ebcSRyan Zezeski uint32_t *sq_db_addr; 1696f443ebcSRyan Zezeski 1706f443ebcSRyan Zezeski /* 1716f443ebcSRyan Zezeski * First, allocate the Rx data buffers. 1726f443ebcSRyan Zezeski */ 1736f443ebcSRyan Zezeski if ((ret = ena_alloc_rx_dma(rxq)) != 0) { 1746f443ebcSRyan Zezeski ena_err(ena, "failed to allocate Rx queue %u data buffers: %d", 1756f443ebcSRyan Zezeski rxq->er_rxqs_idx, ret); 1766f443ebcSRyan Zezeski return (B_FALSE); 1776f443ebcSRyan Zezeski } 1786f443ebcSRyan Zezeski 1796f443ebcSRyan Zezeski ASSERT(rxq->er_state & ENA_RXQ_STATE_HOST_ALLOC); 1806f443ebcSRyan Zezeski 1816f443ebcSRyan Zezeski /* 1826f443ebcSRyan Zezeski * Second, create the Completion Queue. 1836f443ebcSRyan Zezeski */ 1846f443ebcSRyan Zezeski ret = ena_create_cq(ena, rxq->er_cq_num_descs, 1856f443ebcSRyan Zezeski rxq->er_cq_dma.edb_cookie->dmac_laddress, B_FALSE, 1866f443ebcSRyan Zezeski rxq->er_intr_vector, &cq_hw_idx, &cq_unmask_addr, &cq_headdb, 1876f443ebcSRyan Zezeski &cq_numanode); 1886f443ebcSRyan Zezeski 1896f443ebcSRyan Zezeski if (ret != 0) { 1906f443ebcSRyan Zezeski ena_err(ena, "failed to create Rx CQ %u: %d", rxq->er_rxqs_idx, 1916f443ebcSRyan Zezeski ret); 1926f443ebcSRyan Zezeski return (B_FALSE); 1936f443ebcSRyan Zezeski } 1946f443ebcSRyan Zezeski 1956f443ebcSRyan Zezeski /* The phase must always start on 1. */ 1966f443ebcSRyan Zezeski rxq->er_cq_phase = 1; 1976f443ebcSRyan Zezeski rxq->er_cq_head_idx = 0; 1986f443ebcSRyan Zezeski rxq->er_cq_hw_idx = cq_hw_idx; 1996f443ebcSRyan Zezeski rxq->er_cq_unmask_addr = cq_unmask_addr; 2006f443ebcSRyan Zezeski rxq->er_cq_head_db_addr = cq_headdb; 2016f443ebcSRyan Zezeski rxq->er_cq_numa_addr = cq_numanode; 2026f443ebcSRyan Zezeski rxq->er_state |= ENA_RXQ_STATE_CQ_CREATED; 2036f443ebcSRyan Zezeski 2046f443ebcSRyan Zezeski /* 2056f443ebcSRyan Zezeski * Third, create the Submission Queue to match with the above 2066f443ebcSRyan Zezeski * CQ. At this time we force the SQ and CQ to have the same 2076f443ebcSRyan Zezeski * number of descriptors as we only use a 1:1 completion 2086f443ebcSRyan Zezeski * policy. However, in the future, we could loosen this and 2096f443ebcSRyan Zezeski * use an on-demand completion policy and the two could have a 2106f443ebcSRyan Zezeski * different number of descriptors. 2116f443ebcSRyan Zezeski */ 2126f443ebcSRyan Zezeski ASSERT3U(rxq->er_sq_num_descs, ==, rxq->er_cq_num_descs); 2136f443ebcSRyan Zezeski ret = ena_create_sq(ena, rxq->er_sq_num_descs, 2146f443ebcSRyan Zezeski rxq->er_sq_dma.edb_cookie->dmac_laddress, B_FALSE, cq_hw_idx, 2156f443ebcSRyan Zezeski &sq_hw_idx, &sq_db_addr); 2166f443ebcSRyan Zezeski 2176f443ebcSRyan Zezeski if (ret != 0) { 2186f443ebcSRyan Zezeski ena_err(ena, "failed to create Rx SQ %u: %d", rxq->er_rxqs_idx, 2196f443ebcSRyan Zezeski ret); 2206f443ebcSRyan Zezeski return (B_FALSE); 2216f443ebcSRyan Zezeski } 2226f443ebcSRyan Zezeski 2236f443ebcSRyan Zezeski ASSERT3P(sq_db_addr, !=, NULL); 2246f443ebcSRyan Zezeski rxq->er_sq_hw_idx = sq_hw_idx; 2256f443ebcSRyan Zezeski rxq->er_sq_db_addr = sq_db_addr; 2266f443ebcSRyan Zezeski /* The phase must always start on 1. */ 2276f443ebcSRyan Zezeski rxq->er_sq_phase = 1; 2286f443ebcSRyan Zezeski rxq->er_sq_tail_idx = 0; 2296f443ebcSRyan Zezeski rxq->er_sq_avail_descs = rxq->er_sq_num_descs; 2306f443ebcSRyan Zezeski rxq->er_mode = ENA_RXQ_MODE_INTR; 2316f443ebcSRyan Zezeski rxq->er_state |= ENA_RXQ_STATE_SQ_CREATED; 2326f443ebcSRyan Zezeski 2336f443ebcSRyan Zezeski return (B_TRUE); 2346f443ebcSRyan Zezeski } 2356f443ebcSRyan Zezeski 2366f443ebcSRyan Zezeski void 2376f443ebcSRyan Zezeski ena_cleanup_rxq(ena_rxq_t *rxq) 2386f443ebcSRyan Zezeski { 2396f443ebcSRyan Zezeski int ret = 0; 2406f443ebcSRyan Zezeski ena_t *ena = rxq->er_ena; 2416f443ebcSRyan Zezeski 2426f443ebcSRyan Zezeski if ((rxq->er_state & ENA_RXQ_STATE_SQ_CREATED) != 0) { 2436f443ebcSRyan Zezeski ret = ena_destroy_sq(ena, rxq->er_sq_hw_idx, B_FALSE); 2446f443ebcSRyan Zezeski 2456f443ebcSRyan Zezeski if (ret != 0) { 2466f443ebcSRyan Zezeski ena_err(ena, "failed to destroy Rx SQ %u: %d", 2476f443ebcSRyan Zezeski rxq->er_rxqs_idx, ret); 2486f443ebcSRyan Zezeski } 2496f443ebcSRyan Zezeski 2506f443ebcSRyan Zezeski rxq->er_sq_hw_idx = 0; 2516f443ebcSRyan Zezeski rxq->er_sq_db_addr = NULL; 2526f443ebcSRyan Zezeski rxq->er_sq_tail_idx = 0; 2536f443ebcSRyan Zezeski rxq->er_sq_phase = 0; 2546f443ebcSRyan Zezeski rxq->er_state &= ~ENA_RXQ_STATE_SQ_CREATED; 255*eebd18daSAndy Fiddaman rxq->er_state &= ~ENA_RXQ_STATE_SQ_FILLED; 2566f443ebcSRyan Zezeski } 2576f443ebcSRyan Zezeski 2586f443ebcSRyan Zezeski if ((rxq->er_state & ENA_RXQ_STATE_CQ_CREATED) != 0) { 2596f443ebcSRyan Zezeski ret = ena_destroy_cq(ena, rxq->er_cq_hw_idx); 2606f443ebcSRyan Zezeski 2616f443ebcSRyan Zezeski if (ret != 0) { 2626f443ebcSRyan Zezeski ena_err(ena, "failed to destroy Rx CQ %u: %d", 2636f443ebcSRyan Zezeski rxq->er_rxqs_idx, ret); 2646f443ebcSRyan Zezeski } 2656f443ebcSRyan Zezeski 2666f443ebcSRyan Zezeski rxq->er_cq_hw_idx = 0; 2676f443ebcSRyan Zezeski rxq->er_cq_head_idx = 0; 2686f443ebcSRyan Zezeski rxq->er_cq_phase = 0; 2696f443ebcSRyan Zezeski rxq->er_cq_head_db_addr = NULL; 2706f443ebcSRyan Zezeski rxq->er_cq_unmask_addr = NULL; 2716f443ebcSRyan Zezeski rxq->er_cq_numa_addr = NULL; 2726f443ebcSRyan Zezeski rxq->er_state &= ~ENA_RXQ_STATE_CQ_CREATED; 2736f443ebcSRyan Zezeski } 2746f443ebcSRyan Zezeski 2756f443ebcSRyan Zezeski ena_free_rx_dma(rxq); 2766f443ebcSRyan Zezeski ASSERT3S(rxq->er_state, ==, ENA_RXQ_STATE_NONE); 2776f443ebcSRyan Zezeski } 2786f443ebcSRyan Zezeski 2796f443ebcSRyan Zezeski void 2806f443ebcSRyan Zezeski ena_ring_rx_stop(mac_ring_driver_t rh) 2816f443ebcSRyan Zezeski { 2826f443ebcSRyan Zezeski ena_rxq_t *rxq = (ena_rxq_t *)rh; 2836f443ebcSRyan Zezeski uint32_t intr_ctrl; 2846f443ebcSRyan Zezeski 2856f443ebcSRyan Zezeski intr_ctrl = ena_hw_abs_read32(rxq->er_ena, rxq->er_cq_unmask_addr); 2866f443ebcSRyan Zezeski ENAHW_REG_INTR_MASK(intr_ctrl); 2876f443ebcSRyan Zezeski ena_hw_abs_write32(rxq->er_ena, rxq->er_cq_unmask_addr, intr_ctrl); 2886f443ebcSRyan Zezeski 2896f443ebcSRyan Zezeski rxq->er_state &= ~ENA_RXQ_STATE_RUNNING; 2906f443ebcSRyan Zezeski rxq->er_state &= ~ENA_RXQ_STATE_READY; 2916f443ebcSRyan Zezeski } 2926f443ebcSRyan Zezeski 2936f443ebcSRyan Zezeski int 2946f443ebcSRyan Zezeski ena_ring_rx_start(mac_ring_driver_t rh, uint64_t gen_num) 2956f443ebcSRyan Zezeski { 2966f443ebcSRyan Zezeski ena_rxq_t *rxq = (ena_rxq_t *)rh; 2976f443ebcSRyan Zezeski ena_t *ena = rxq->er_ena; 2986f443ebcSRyan Zezeski uint32_t intr_ctrl; 2996f443ebcSRyan Zezeski 300*eebd18daSAndy Fiddaman ena_dbg(ena, "ring_rx_start %p: state %x", rxq, rxq->er_state); 301*eebd18daSAndy Fiddaman 3026f443ebcSRyan Zezeski mutex_enter(&rxq->er_lock); 303*eebd18daSAndy Fiddaman if ((rxq->er_state & ENA_RXQ_STATE_SQ_FILLED) == 0) { 304*eebd18daSAndy Fiddaman /* 305*eebd18daSAndy Fiddaman * The ENA controller gets upset and sets the fatal error bit 306*eebd18daSAndy Fiddaman * in its status register if we write a value to an RX SQ's 307*eebd18daSAndy Fiddaman * doorbell that is past its current head. This makes sense as 308*eebd18daSAndy Fiddaman * it would represent there being more descriptors available 309*eebd18daSAndy Fiddaman * than can fit in the ring. For this reason, we make sure that 310*eebd18daSAndy Fiddaman * we only fill the ring once, even if it is started multiple 311*eebd18daSAndy Fiddaman * times. 312*eebd18daSAndy Fiddaman * The `- 1` below is harder to explain. If we completely fill 313*eebd18daSAndy Fiddaman * the SQ ring, then at some time later that seems to be 314*eebd18daSAndy Fiddaman * independent of how many times we've been around the ring, 315*eebd18daSAndy Fiddaman * the ENA controller will set the fatal error bit and stop 316*eebd18daSAndy Fiddaman * responding. Leaving a gap prevents this somehow and it is 317*eebd18daSAndy Fiddaman * what the other open source drivers do. 318*eebd18daSAndy Fiddaman */ 319*eebd18daSAndy Fiddaman ena_refill_rx(rxq, rxq->er_sq_num_descs - 1); 320*eebd18daSAndy Fiddaman rxq->er_state |= ENA_RXQ_STATE_SQ_FILLED; 321*eebd18daSAndy Fiddaman } 3226f443ebcSRyan Zezeski rxq->er_m_gen_num = gen_num; 3236f443ebcSRyan Zezeski rxq->er_intr_limit = ena->ena_rxq_intr_limit; 3246f443ebcSRyan Zezeski mutex_exit(&rxq->er_lock); 3256f443ebcSRyan Zezeski 3266f443ebcSRyan Zezeski rxq->er_state |= ENA_RXQ_STATE_READY; 3276f443ebcSRyan Zezeski 3286f443ebcSRyan Zezeski intr_ctrl = ena_hw_abs_read32(ena, rxq->er_cq_unmask_addr); 3296f443ebcSRyan Zezeski ENAHW_REG_INTR_UNMASK(intr_ctrl); 3306f443ebcSRyan Zezeski ena_hw_abs_write32(ena, rxq->er_cq_unmask_addr, intr_ctrl); 3316f443ebcSRyan Zezeski rxq->er_state |= ENA_RXQ_STATE_RUNNING; 3326f443ebcSRyan Zezeski return (0); 3336f443ebcSRyan Zezeski } 3346f443ebcSRyan Zezeski 3356f443ebcSRyan Zezeski mblk_t * 3366f443ebcSRyan Zezeski ena_ring_rx(ena_rxq_t *rxq, int poll_bytes) 3376f443ebcSRyan Zezeski { 3386f443ebcSRyan Zezeski ena_t *ena = rxq->er_ena; 3396f443ebcSRyan Zezeski uint16_t head_mod = rxq->er_cq_head_idx & (rxq->er_cq_num_descs - 1); 3406f443ebcSRyan Zezeski uint64_t total_bytes = 0; 3416f443ebcSRyan Zezeski uint64_t num_frames = 0; 3426f443ebcSRyan Zezeski enahw_rx_cdesc_t *cdesc; 3436f443ebcSRyan Zezeski boolean_t polling = B_TRUE; 3446f443ebcSRyan Zezeski mblk_t *head = NULL; 3456f443ebcSRyan Zezeski mblk_t *tail = NULL; 3466f443ebcSRyan Zezeski 3476f443ebcSRyan Zezeski ASSERT(MUTEX_HELD(&rxq->er_lock)); 3486f443ebcSRyan Zezeski ENA_DMA_SYNC(rxq->er_cq_dma, DDI_DMA_SYNC_FORKERNEL); 3496f443ebcSRyan Zezeski 3506f443ebcSRyan Zezeski if (poll_bytes == ENA_INTERRUPT_MODE) { 3516f443ebcSRyan Zezeski polling = B_FALSE; 3526f443ebcSRyan Zezeski } 3536f443ebcSRyan Zezeski 3546f443ebcSRyan Zezeski cdesc = &rxq->er_cq_descs[head_mod]; 3556f443ebcSRyan Zezeski VERIFY3P(cdesc, >=, rxq->er_cq_descs); 3566f443ebcSRyan Zezeski VERIFY3P(cdesc, <=, (rxq->er_cq_descs + rxq->er_cq_num_descs - 1)); 3576f443ebcSRyan Zezeski 3586f443ebcSRyan Zezeski while (ENAHW_RX_CDESC_PHASE(cdesc) == rxq->er_cq_phase) { 3596f443ebcSRyan Zezeski boolean_t first, last; 3606f443ebcSRyan Zezeski ena_rx_ctrl_block_t *rcb; 3616f443ebcSRyan Zezeski uint16_t req_id; 3626f443ebcSRyan Zezeski mblk_t *mp; 3636f443ebcSRyan Zezeski enahw_io_l3_proto_t l3proto; 3646f443ebcSRyan Zezeski enahw_io_l4_proto_t l4proto; 3656f443ebcSRyan Zezeski boolean_t l4csum_checked; 3666f443ebcSRyan Zezeski uint32_t hflags = 0; 3676f443ebcSRyan Zezeski 3686f443ebcSRyan Zezeski VERIFY3U(head_mod, <, rxq->er_cq_num_descs); 3696f443ebcSRyan Zezeski /* 3706f443ebcSRyan Zezeski * Currently, all incoming frames fit in a single Rx 3716f443ebcSRyan Zezeski * buffer (erd_length > total frame size). In the 3726f443ebcSRyan Zezeski * future, if we decide to loan buffers which are 3736f443ebcSRyan Zezeski * smaller, we will need to modify this code to read 3746f443ebcSRyan Zezeski * one or more descriptors (based on frame size). 3756f443ebcSRyan Zezeski * 3766f443ebcSRyan Zezeski * For this reason we do not expect any frame to span 3776f443ebcSRyan Zezeski * multiple descriptors. Therefore, we drop any data 3786f443ebcSRyan Zezeski * not delivered as a single descriptor, i.e., where 3796f443ebcSRyan Zezeski * 'first' and 'last' are both true. 3806f443ebcSRyan Zezeski */ 3816f443ebcSRyan Zezeski first = ENAHW_RX_CDESC_FIRST(cdesc); 3826f443ebcSRyan Zezeski last = ENAHW_RX_CDESC_LAST(cdesc); 3836f443ebcSRyan Zezeski 3846f443ebcSRyan Zezeski if (!first || !last) { 3856f443ebcSRyan Zezeski mutex_enter(&rxq->er_stat_lock); 3866f443ebcSRyan Zezeski rxq->er_stat.ers_multi_desc.value.ui64++; 3876f443ebcSRyan Zezeski mutex_exit(&rxq->er_stat_lock); 3886f443ebcSRyan Zezeski goto next_desc; 3896f443ebcSRyan Zezeski } 3906f443ebcSRyan Zezeski 3916f443ebcSRyan Zezeski req_id = cdesc->erc_req_id; 3926f443ebcSRyan Zezeski VERIFY3U(req_id, <, rxq->er_cq_num_descs); 3936f443ebcSRyan Zezeski rcb = &rxq->er_rcbs[req_id]; 3946f443ebcSRyan Zezeski rcb->ercb_offset = cdesc->erc_offset; 3956f443ebcSRyan Zezeski rcb->ercb_length = cdesc->erc_length; 3966f443ebcSRyan Zezeski ASSERT3U(rcb->ercb_length, <=, ena->ena_max_frame_total); 3976f443ebcSRyan Zezeski mp = allocb(rcb->ercb_length + ENA_RX_BUF_IPHDR_ALIGNMENT, 0); 3986f443ebcSRyan Zezeski 3996f443ebcSRyan Zezeski /* 4006f443ebcSRyan Zezeski * If we can't allocate an mblk, things are looking 4016f443ebcSRyan Zezeski * grim. Forget about this frame and move on. 4026f443ebcSRyan Zezeski */ 4036f443ebcSRyan Zezeski if (mp == NULL) { 4046f443ebcSRyan Zezeski mutex_enter(&rxq->er_stat_lock); 4056f443ebcSRyan Zezeski rxq->er_stat.ers_allocb_fail.value.ui64++; 4066f443ebcSRyan Zezeski mutex_exit(&rxq->er_stat_lock); 4076f443ebcSRyan Zezeski goto next_desc; 4086f443ebcSRyan Zezeski } 4096f443ebcSRyan Zezeski 4106f443ebcSRyan Zezeski /* 4116f443ebcSRyan Zezeski * As we pull frames we need to link them together as 4126f443ebcSRyan Zezeski * one chain to be delivered up to mac. 4136f443ebcSRyan Zezeski */ 4146f443ebcSRyan Zezeski if (head == NULL) { 4156f443ebcSRyan Zezeski head = mp; 4166f443ebcSRyan Zezeski } else { 4176f443ebcSRyan Zezeski tail->b_next = mp; 4186f443ebcSRyan Zezeski } 4196f443ebcSRyan Zezeski 4206f443ebcSRyan Zezeski tail = mp; 4216f443ebcSRyan Zezeski 4226f443ebcSRyan Zezeski /* 4236f443ebcSRyan Zezeski * We need to make sure the bytes are copied to the 4246f443ebcSRyan Zezeski * correct offset to achieve 4-byte IP header 4256f443ebcSRyan Zezeski * alignment. 4266f443ebcSRyan Zezeski * 4276f443ebcSRyan Zezeski * If we start using desballoc on the buffers, then we 4286f443ebcSRyan Zezeski * will need to make sure to apply this offset to the 4296f443ebcSRyan Zezeski * DMA buffers as well. Though it may be the case the 4306f443ebcSRyan Zezeski * device does this implicitly and that's what 4316f443ebcSRyan Zezeski * cdesc->erc_offset is for; we don't know because 4326f443ebcSRyan Zezeski * it's not documented. 4336f443ebcSRyan Zezeski */ 4346f443ebcSRyan Zezeski mp->b_wptr += ENA_RX_BUF_IPHDR_ALIGNMENT; 4356f443ebcSRyan Zezeski mp->b_rptr += ENA_RX_BUF_IPHDR_ALIGNMENT; 4366f443ebcSRyan Zezeski bcopy(rcb->ercb_dma.edb_va + rcb->ercb_offset, mp->b_wptr, 4376f443ebcSRyan Zezeski rcb->ercb_length); 4386f443ebcSRyan Zezeski mp->b_wptr += rcb->ercb_length; 4396f443ebcSRyan Zezeski total_bytes += rcb->ercb_length; 4406f443ebcSRyan Zezeski VERIFY3P(mp->b_wptr, >, mp->b_rptr); 4416f443ebcSRyan Zezeski VERIFY3P(mp->b_wptr, <=, mp->b_datap->db_lim); 4426f443ebcSRyan Zezeski 4436f443ebcSRyan Zezeski l3proto = ENAHW_RX_CDESC_L3_PROTO(cdesc); 4446f443ebcSRyan Zezeski l4proto = ENAHW_RX_CDESC_L4_PROTO(cdesc); 4456f443ebcSRyan Zezeski 4466f443ebcSRyan Zezeski /* 4476f443ebcSRyan Zezeski * When it comes to bad TCP/IP checksums we do not 4486f443ebcSRyan Zezeski * discard the packet at this level. Instead, we let 4496f443ebcSRyan Zezeski * it percolate up for further processing and tracking 4506f443ebcSRyan Zezeski * by the upstream TCP/IP stack. 4516f443ebcSRyan Zezeski */ 4526f443ebcSRyan Zezeski if (ena->ena_rx_l3_ipv4_csum && 4536f443ebcSRyan Zezeski l3proto == ENAHW_IO_L3_PROTO_IPV4) { 4546f443ebcSRyan Zezeski boolean_t l3_csum_err = 4556f443ebcSRyan Zezeski ENAHW_RX_CDESC_L3_CSUM_ERR(cdesc); 4566f443ebcSRyan Zezeski 4576f443ebcSRyan Zezeski if (l3_csum_err) { 4586f443ebcSRyan Zezeski mutex_enter(&rxq->er_stat_lock); 4596f443ebcSRyan Zezeski rxq->er_stat.ers_hck_ipv4_err.value.ui64++; 4606f443ebcSRyan Zezeski mutex_exit(&rxq->er_stat_lock); 4616f443ebcSRyan Zezeski } else { 4626f443ebcSRyan Zezeski hflags |= HCK_IPV4_HDRCKSUM_OK; 4636f443ebcSRyan Zezeski } 4646f443ebcSRyan Zezeski } 4656f443ebcSRyan Zezeski 4666f443ebcSRyan Zezeski l4csum_checked = ENAHW_RX_CDESC_L4_CSUM_CHECKED(cdesc); 4676f443ebcSRyan Zezeski 4686f443ebcSRyan Zezeski if (ena->ena_rx_l4_ipv4_csum && l4csum_checked && 4696f443ebcSRyan Zezeski l4proto == ENAHW_IO_L4_PROTO_TCP) { 4706f443ebcSRyan Zezeski boolean_t l4_csum_err = 4716f443ebcSRyan Zezeski ENAHW_RX_CDESC_L4_CSUM_ERR(cdesc); 4726f443ebcSRyan Zezeski 4736f443ebcSRyan Zezeski if (l4_csum_err) { 4746f443ebcSRyan Zezeski mutex_enter(&rxq->er_stat_lock); 4756f443ebcSRyan Zezeski rxq->er_stat.ers_hck_l4_err.value.ui64++; 4766f443ebcSRyan Zezeski mutex_exit(&rxq->er_stat_lock); 4776f443ebcSRyan Zezeski } else { 4786f443ebcSRyan Zezeski hflags |= HCK_FULLCKSUM_OK; 4796f443ebcSRyan Zezeski } 4806f443ebcSRyan Zezeski } 4816f443ebcSRyan Zezeski 4826f443ebcSRyan Zezeski if (hflags != 0) { 4836f443ebcSRyan Zezeski mac_hcksum_set(mp, 0, 0, 0, 0, hflags); 4846f443ebcSRyan Zezeski } 4856f443ebcSRyan Zezeski 4866f443ebcSRyan Zezeski next_desc: 4876f443ebcSRyan Zezeski /* 4886f443ebcSRyan Zezeski * Technically, if we arrived here due to a failure, 4896f443ebcSRyan Zezeski * then we did not read a new frame. However, we count 4906f443ebcSRyan Zezeski * it all the same anyways in order to count it as 4916f443ebcSRyan Zezeski * progress to the interrupt work limit. The failure 4926f443ebcSRyan Zezeski * stats will allow us to differentiate good frames 4936f443ebcSRyan Zezeski * from bad. 4946f443ebcSRyan Zezeski */ 4956f443ebcSRyan Zezeski num_frames++; 4966f443ebcSRyan Zezeski rxq->er_cq_head_idx++; 4976f443ebcSRyan Zezeski head_mod = rxq->er_cq_head_idx & (rxq->er_cq_num_descs - 1); 4986f443ebcSRyan Zezeski 4996f443ebcSRyan Zezeski if (head_mod == 0) { 500*eebd18daSAndy Fiddaman rxq->er_cq_phase ^= 1; 5016f443ebcSRyan Zezeski } 5026f443ebcSRyan Zezeski 5036f443ebcSRyan Zezeski if (polling && (total_bytes > poll_bytes)) { 5046f443ebcSRyan Zezeski break; 5056f443ebcSRyan Zezeski } else if (!polling && (num_frames >= rxq->er_intr_limit)) { 5066f443ebcSRyan Zezeski mutex_enter(&rxq->er_stat_lock); 5076f443ebcSRyan Zezeski rxq->er_stat.ers_intr_limit.value.ui64++; 5086f443ebcSRyan Zezeski mutex_exit(&rxq->er_stat_lock); 5096f443ebcSRyan Zezeski break; 5106f443ebcSRyan Zezeski } 5116f443ebcSRyan Zezeski 5126f443ebcSRyan Zezeski cdesc = &rxq->er_cq_descs[head_mod]; 5136f443ebcSRyan Zezeski VERIFY3P(cdesc, >=, rxq->er_cq_descs); 5146f443ebcSRyan Zezeski VERIFY3P(cdesc, <=, 5156f443ebcSRyan Zezeski (rxq->er_cq_descs + rxq->er_cq_num_descs - 1)); 5166f443ebcSRyan Zezeski } 5176f443ebcSRyan Zezeski 518*eebd18daSAndy Fiddaman if (num_frames > 0) { 5196f443ebcSRyan Zezeski mutex_enter(&rxq->er_stat_lock); 5206f443ebcSRyan Zezeski rxq->er_stat.ers_packets.value.ui64 += num_frames; 5216f443ebcSRyan Zezeski rxq->er_stat.ers_bytes.value.ui64 += total_bytes; 5226f443ebcSRyan Zezeski mutex_exit(&rxq->er_stat_lock); 5236f443ebcSRyan Zezeski 524*eebd18daSAndy Fiddaman DTRACE_PROBE4(rx__frames, mblk_t *, head, boolean_t, polling, 525*eebd18daSAndy Fiddaman uint64_t, num_frames, uint64_t, total_bytes); 5266f443ebcSRyan Zezeski ena_refill_rx(rxq, num_frames); 527*eebd18daSAndy Fiddaman } 528*eebd18daSAndy Fiddaman 5296f443ebcSRyan Zezeski return (head); 5306f443ebcSRyan Zezeski } 5316f443ebcSRyan Zezeski 5326f443ebcSRyan Zezeski void 5336f443ebcSRyan Zezeski ena_rx_intr_work(ena_rxq_t *rxq) 5346f443ebcSRyan Zezeski { 5356f443ebcSRyan Zezeski mblk_t *mp; 5366f443ebcSRyan Zezeski 5376f443ebcSRyan Zezeski mutex_enter(&rxq->er_lock); 5386f443ebcSRyan Zezeski mp = ena_ring_rx(rxq, ENA_INTERRUPT_MODE); 5396f443ebcSRyan Zezeski mutex_exit(&rxq->er_lock); 5406f443ebcSRyan Zezeski 5416f443ebcSRyan Zezeski if (mp == NULL) { 5426f443ebcSRyan Zezeski return; 5436f443ebcSRyan Zezeski } 5446f443ebcSRyan Zezeski 5456f443ebcSRyan Zezeski mac_rx_ring(rxq->er_ena->ena_mh, rxq->er_mrh, mp, rxq->er_m_gen_num); 5466f443ebcSRyan Zezeski } 5476f443ebcSRyan Zezeski 5486f443ebcSRyan Zezeski mblk_t * 5496f443ebcSRyan Zezeski ena_ring_rx_poll(void *rh, int poll_bytes) 5506f443ebcSRyan Zezeski { 5516f443ebcSRyan Zezeski ena_rxq_t *rxq = rh; 5526f443ebcSRyan Zezeski mblk_t *mp; 5536f443ebcSRyan Zezeski 5546f443ebcSRyan Zezeski ASSERT3S(poll_bytes, >, 0); 5556f443ebcSRyan Zezeski 5566f443ebcSRyan Zezeski mutex_enter(&rxq->er_lock); 5576f443ebcSRyan Zezeski mp = ena_ring_rx(rxq, poll_bytes); 5586f443ebcSRyan Zezeski mutex_exit(&rxq->er_lock); 5596f443ebcSRyan Zezeski 5606f443ebcSRyan Zezeski return (mp); 5616f443ebcSRyan Zezeski } 562