1758cc3dcSJack F Vogel /****************************************************************************** 2758cc3dcSJack F Vogel 3758cc3dcSJack F Vogel Copyright (c) 2001-2014, Intel Corporation 4758cc3dcSJack F Vogel All rights reserved. 5758cc3dcSJack F Vogel 6758cc3dcSJack F Vogel Redistribution and use in source and binary forms, with or without 7758cc3dcSJack F Vogel modification, are permitted provided that the following conditions are met: 8758cc3dcSJack F Vogel 9758cc3dcSJack F Vogel 1. Redistributions of source code must retain the above copyright notice, 10758cc3dcSJack F Vogel this list of conditions and the following disclaimer. 11758cc3dcSJack F Vogel 12758cc3dcSJack F Vogel 2. Redistributions in binary form must reproduce the above copyright 13758cc3dcSJack F Vogel notice, this list of conditions and the following disclaimer in the 14758cc3dcSJack F Vogel documentation and/or other materials provided with the distribution. 15758cc3dcSJack F Vogel 16758cc3dcSJack F Vogel 3. Neither the name of the Intel Corporation nor the names of its 17758cc3dcSJack F Vogel contributors may be used to endorse or promote products derived from 18758cc3dcSJack F Vogel this software without specific prior written permission. 19758cc3dcSJack F Vogel 20758cc3dcSJack F Vogel THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21758cc3dcSJack F Vogel AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22758cc3dcSJack F Vogel IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23758cc3dcSJack F Vogel ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24758cc3dcSJack F Vogel LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25758cc3dcSJack F Vogel CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26758cc3dcSJack F Vogel SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27758cc3dcSJack F Vogel INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28758cc3dcSJack F Vogel CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29758cc3dcSJack F Vogel ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30758cc3dcSJack F Vogel POSSIBILITY OF SUCH DAMAGE. 31758cc3dcSJack F Vogel 32758cc3dcSJack F Vogel ******************************************************************************/ 33758cc3dcSJack F Vogel /*$FreeBSD$*/ 34758cc3dcSJack F Vogel 35758cc3dcSJack F Vogel 36758cc3dcSJack F Vogel #ifndef IXGBE_STANDALONE_BUILD 37758cc3dcSJack F Vogel #include "opt_inet.h" 38758cc3dcSJack F Vogel #include "opt_inet6.h" 39758cc3dcSJack F Vogel #include "opt_rss.h" 40758cc3dcSJack F Vogel #endif 41758cc3dcSJack F Vogel 42758cc3dcSJack F Vogel #include "ixgbe.h" 43758cc3dcSJack F Vogel 44758cc3dcSJack F Vogel #ifdef RSS 45758cc3dcSJack F Vogel #include <netinet/in_rss.h> 46758cc3dcSJack F Vogel #endif 47758cc3dcSJack F Vogel 48758cc3dcSJack F Vogel /* 49758cc3dcSJack F Vogel ** HW RSC control: 50758cc3dcSJack F Vogel ** this feature only works with 51758cc3dcSJack F Vogel ** IPv4, and only on 82599 and later. 52758cc3dcSJack F Vogel ** Also this will cause IP forwarding to 53758cc3dcSJack F Vogel ** fail and that can't be controlled by 54758cc3dcSJack F Vogel ** the stack as LRO can. For all these 55758cc3dcSJack F Vogel ** reasons I've deemed it best to leave 56758cc3dcSJack F Vogel ** this off and not bother with a tuneable 57758cc3dcSJack F Vogel ** interface, this would need to be compiled 58758cc3dcSJack F Vogel ** to enable. 59758cc3dcSJack F Vogel */ 60758cc3dcSJack F Vogel static bool ixgbe_rsc_enable = FALSE; 61758cc3dcSJack F Vogel 62758cc3dcSJack F Vogel #ifdef IXGBE_FDIR 63758cc3dcSJack F Vogel /* 64758cc3dcSJack F Vogel ** For Flow Director: this is the 65758cc3dcSJack F Vogel ** number of TX packets we sample 66758cc3dcSJack F Vogel ** for the filter pool, this means 67758cc3dcSJack F Vogel ** every 20th packet will be probed. 68758cc3dcSJack F Vogel ** 69758cc3dcSJack F Vogel ** This feature can be disabled by 70758cc3dcSJack F Vogel ** setting this to 0. 71758cc3dcSJack F Vogel */ 72758cc3dcSJack F Vogel static int atr_sample_rate = 20; 73758cc3dcSJack F Vogel #endif 74758cc3dcSJack F Vogel 75758cc3dcSJack F Vogel /* Shared PCI config read/write */ 76758cc3dcSJack F Vogel inline u16 77758cc3dcSJack F Vogel ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg) 78758cc3dcSJack F Vogel { 79758cc3dcSJack F Vogel u16 value; 80758cc3dcSJack F Vogel 81758cc3dcSJack F Vogel value = pci_read_config(((struct ixgbe_osdep *)hw->back)->dev, 82758cc3dcSJack F Vogel reg, 2); 83758cc3dcSJack F Vogel 84758cc3dcSJack F Vogel return (value); 85758cc3dcSJack F Vogel } 86758cc3dcSJack F Vogel 87758cc3dcSJack F Vogel inline void 88758cc3dcSJack F Vogel ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value) 89758cc3dcSJack F Vogel { 90758cc3dcSJack F Vogel pci_write_config(((struct ixgbe_osdep *)hw->back)->dev, 91758cc3dcSJack F Vogel reg, value, 2); 92758cc3dcSJack F Vogel 93758cc3dcSJack F Vogel return; 94758cc3dcSJack F Vogel } 95758cc3dcSJack F Vogel 96758cc3dcSJack F Vogel /********************************************************************* 97758cc3dcSJack F Vogel * Local Function prototypes 98758cc3dcSJack F Vogel *********************************************************************/ 99758cc3dcSJack F Vogel static void ixgbe_setup_transmit_ring(struct tx_ring *); 100758cc3dcSJack F Vogel static void ixgbe_free_transmit_buffers(struct tx_ring *); 101758cc3dcSJack F Vogel static int ixgbe_setup_receive_ring(struct rx_ring *); 102758cc3dcSJack F Vogel static void ixgbe_free_receive_buffers(struct rx_ring *); 103758cc3dcSJack F Vogel 104758cc3dcSJack F Vogel static void ixgbe_rx_checksum(u32, struct mbuf *, u32); 105758cc3dcSJack F Vogel static void ixgbe_refresh_mbufs(struct rx_ring *, int); 106758cc3dcSJack F Vogel static int ixgbe_xmit(struct tx_ring *, struct mbuf **); 107758cc3dcSJack F Vogel static int ixgbe_tx_ctx_setup(struct tx_ring *, 108758cc3dcSJack F Vogel struct mbuf *, u32 *, u32 *); 109758cc3dcSJack F Vogel static int ixgbe_tso_setup(struct tx_ring *, 110758cc3dcSJack F Vogel struct mbuf *, u32 *, u32 *); 111758cc3dcSJack F Vogel #ifdef IXGBE_FDIR 112758cc3dcSJack F Vogel static void ixgbe_atr(struct tx_ring *, struct mbuf *); 113758cc3dcSJack F Vogel #endif 114758cc3dcSJack F Vogel static __inline void ixgbe_rx_discard(struct rx_ring *, int); 115758cc3dcSJack F Vogel static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *, 116758cc3dcSJack F Vogel struct mbuf *, u32); 117758cc3dcSJack F Vogel 118758cc3dcSJack F Vogel #ifdef IXGBE_LEGACY_TX 119758cc3dcSJack F Vogel /********************************************************************* 120758cc3dcSJack F Vogel * Transmit entry point 121758cc3dcSJack F Vogel * 122758cc3dcSJack F Vogel * ixgbe_start is called by the stack to initiate a transmit. 123758cc3dcSJack F Vogel * The driver will remain in this routine as long as there are 124758cc3dcSJack F Vogel * packets to transmit and transmit resources are available. 125758cc3dcSJack F Vogel * In case resources are not available stack is notified and 126758cc3dcSJack F Vogel * the packet is requeued. 127758cc3dcSJack F Vogel **********************************************************************/ 128758cc3dcSJack F Vogel 129758cc3dcSJack F Vogel void 130758cc3dcSJack F Vogel ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp) 131758cc3dcSJack F Vogel { 132758cc3dcSJack F Vogel struct mbuf *m_head; 133758cc3dcSJack F Vogel struct adapter *adapter = txr->adapter; 134758cc3dcSJack F Vogel 135758cc3dcSJack F Vogel IXGBE_TX_LOCK_ASSERT(txr); 136758cc3dcSJack F Vogel 137758cc3dcSJack F Vogel if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 138758cc3dcSJack F Vogel return; 139758cc3dcSJack F Vogel if (!adapter->link_active) 140758cc3dcSJack F Vogel return; 141758cc3dcSJack F Vogel 142758cc3dcSJack F Vogel while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 143758cc3dcSJack F Vogel if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) 144758cc3dcSJack F Vogel break; 145758cc3dcSJack F Vogel 146758cc3dcSJack F Vogel IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); 147758cc3dcSJack F Vogel if (m_head == NULL) 148758cc3dcSJack F Vogel break; 149758cc3dcSJack F Vogel 150758cc3dcSJack F Vogel if (ixgbe_xmit(txr, &m_head)) { 151758cc3dcSJack F Vogel if (m_head != NULL) 152758cc3dcSJack F Vogel IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 153758cc3dcSJack F Vogel break; 154758cc3dcSJack F Vogel } 155758cc3dcSJack F Vogel /* Send a copy of the frame to the BPF listener */ 156758cc3dcSJack F Vogel ETHER_BPF_MTAP(ifp, m_head); 157758cc3dcSJack F Vogel } 158758cc3dcSJack F Vogel return; 159758cc3dcSJack F Vogel } 160758cc3dcSJack F Vogel 161758cc3dcSJack F Vogel /* 162758cc3dcSJack F Vogel * Legacy TX start - called by the stack, this 163758cc3dcSJack F Vogel * always uses the first tx ring, and should 164758cc3dcSJack F Vogel * not be used with multiqueue tx enabled. 165758cc3dcSJack F Vogel */ 166758cc3dcSJack F Vogel void 167758cc3dcSJack F Vogel ixgbe_start(struct ifnet *ifp) 168758cc3dcSJack F Vogel { 169758cc3dcSJack F Vogel struct adapter *adapter = ifp->if_softc; 170758cc3dcSJack F Vogel struct tx_ring *txr = adapter->tx_rings; 171758cc3dcSJack F Vogel 172758cc3dcSJack F Vogel if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 173758cc3dcSJack F Vogel IXGBE_TX_LOCK(txr); 174758cc3dcSJack F Vogel ixgbe_start_locked(txr, ifp); 175758cc3dcSJack F Vogel IXGBE_TX_UNLOCK(txr); 176758cc3dcSJack F Vogel } 177758cc3dcSJack F Vogel return; 178758cc3dcSJack F Vogel } 179758cc3dcSJack F Vogel 180758cc3dcSJack F Vogel #else /* ! IXGBE_LEGACY_TX */ 181758cc3dcSJack F Vogel 182758cc3dcSJack F Vogel /* 183758cc3dcSJack F Vogel ** Multiqueue Transmit driver 184758cc3dcSJack F Vogel ** 185758cc3dcSJack F Vogel */ 186758cc3dcSJack F Vogel int 187758cc3dcSJack F Vogel ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m) 188758cc3dcSJack F Vogel { 189758cc3dcSJack F Vogel struct adapter *adapter = ifp->if_softc; 190758cc3dcSJack F Vogel struct ix_queue *que; 191758cc3dcSJack F Vogel struct tx_ring *txr; 192758cc3dcSJack F Vogel int i, err = 0; 193758cc3dcSJack F Vogel #ifdef RSS 194758cc3dcSJack F Vogel uint32_t bucket_id; 195758cc3dcSJack F Vogel #endif 196758cc3dcSJack F Vogel 197758cc3dcSJack F Vogel /* 198758cc3dcSJack F Vogel * When doing RSS, map it to the same outbound queue 199758cc3dcSJack F Vogel * as the incoming flow would be mapped to. 200758cc3dcSJack F Vogel * 201758cc3dcSJack F Vogel * If everything is setup correctly, it should be the 202758cc3dcSJack F Vogel * same bucket that the current CPU we're on is. 203758cc3dcSJack F Vogel */ 204758cc3dcSJack F Vogel if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) { 205758cc3dcSJack F Vogel #ifdef RSS 206758cc3dcSJack F Vogel if (rss_hash2bucket(m->m_pkthdr.flowid, 207758cc3dcSJack F Vogel M_HASHTYPE_GET(m), &bucket_id) == 0) 208758cc3dcSJack F Vogel /* TODO: spit out something if bucket_id > num_queues? */ 209758cc3dcSJack F Vogel i = bucket_id % adapter->num_queues; 210758cc3dcSJack F Vogel else 211758cc3dcSJack F Vogel #endif 212758cc3dcSJack F Vogel i = m->m_pkthdr.flowid % adapter->num_queues; 213758cc3dcSJack F Vogel } else 214758cc3dcSJack F Vogel i = curcpu % adapter->num_queues; 215758cc3dcSJack F Vogel 216758cc3dcSJack F Vogel /* Check for a hung queue and pick alternative */ 217758cc3dcSJack F Vogel if (((1 << i) & adapter->active_queues) == 0) 218758cc3dcSJack F Vogel i = ffsl(adapter->active_queues); 219758cc3dcSJack F Vogel 220758cc3dcSJack F Vogel txr = &adapter->tx_rings[i]; 221758cc3dcSJack F Vogel que = &adapter->queues[i]; 222758cc3dcSJack F Vogel 223758cc3dcSJack F Vogel err = drbr_enqueue(ifp, txr->br, m); 224758cc3dcSJack F Vogel if (err) 225758cc3dcSJack F Vogel return (err); 226758cc3dcSJack F Vogel if (IXGBE_TX_TRYLOCK(txr)) { 227758cc3dcSJack F Vogel ixgbe_mq_start_locked(ifp, txr); 228758cc3dcSJack F Vogel IXGBE_TX_UNLOCK(txr); 229758cc3dcSJack F Vogel } else 230758cc3dcSJack F Vogel taskqueue_enqueue(que->tq, &txr->txq_task); 231758cc3dcSJack F Vogel 232758cc3dcSJack F Vogel return (0); 233758cc3dcSJack F Vogel } 234758cc3dcSJack F Vogel 235758cc3dcSJack F Vogel int 236758cc3dcSJack F Vogel ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr) 237758cc3dcSJack F Vogel { 238758cc3dcSJack F Vogel struct adapter *adapter = txr->adapter; 239758cc3dcSJack F Vogel struct mbuf *next; 240758cc3dcSJack F Vogel int enqueued = 0, err = 0; 241758cc3dcSJack F Vogel 242758cc3dcSJack F Vogel if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) || 243758cc3dcSJack F Vogel adapter->link_active == 0) 244758cc3dcSJack F Vogel return (ENETDOWN); 245758cc3dcSJack F Vogel 246758cc3dcSJack F Vogel /* Process the queue */ 247758cc3dcSJack F Vogel #if __FreeBSD_version < 901504 248758cc3dcSJack F Vogel next = drbr_dequeue(ifp, txr->br); 249758cc3dcSJack F Vogel while (next != NULL) { 250758cc3dcSJack F Vogel if ((err = ixgbe_xmit(txr, &next)) != 0) { 251758cc3dcSJack F Vogel if (next != NULL) 252758cc3dcSJack F Vogel err = drbr_enqueue(ifp, txr->br, next); 253758cc3dcSJack F Vogel #else 254758cc3dcSJack F Vogel while ((next = drbr_peek(ifp, txr->br)) != NULL) { 255758cc3dcSJack F Vogel if ((err = ixgbe_xmit(txr, &next)) != 0) { 256758cc3dcSJack F Vogel if (next == NULL) { 257758cc3dcSJack F Vogel drbr_advance(ifp, txr->br); 258758cc3dcSJack F Vogel } else { 259758cc3dcSJack F Vogel drbr_putback(ifp, txr->br, next); 260758cc3dcSJack F Vogel } 261758cc3dcSJack F Vogel #endif 262758cc3dcSJack F Vogel break; 263758cc3dcSJack F Vogel } 264758cc3dcSJack F Vogel #if __FreeBSD_version >= 901504 265758cc3dcSJack F Vogel drbr_advance(ifp, txr->br); 266758cc3dcSJack F Vogel #endif 267758cc3dcSJack F Vogel enqueued++; 268758cc3dcSJack F Vogel #if 0 // this is VF-only 269758cc3dcSJack F Vogel #if __FreeBSD_version >= 1100036 270758cc3dcSJack F Vogel if (next->m_flags & M_MCAST) 271758cc3dcSJack F Vogel if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); 272758cc3dcSJack F Vogel #endif 273758cc3dcSJack F Vogel #endif 274758cc3dcSJack F Vogel /* Send a copy of the frame to the BPF listener */ 275758cc3dcSJack F Vogel ETHER_BPF_MTAP(ifp, next); 276758cc3dcSJack F Vogel if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 277758cc3dcSJack F Vogel break; 278758cc3dcSJack F Vogel #if __FreeBSD_version < 901504 279758cc3dcSJack F Vogel next = drbr_dequeue(ifp, txr->br); 280758cc3dcSJack F Vogel #endif 281758cc3dcSJack F Vogel } 282758cc3dcSJack F Vogel 283758cc3dcSJack F Vogel if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD) 284758cc3dcSJack F Vogel ixgbe_txeof(txr); 285758cc3dcSJack F Vogel 286758cc3dcSJack F Vogel return (err); 287758cc3dcSJack F Vogel } 288758cc3dcSJack F Vogel 289758cc3dcSJack F Vogel /* 290758cc3dcSJack F Vogel * Called from a taskqueue to drain queued transmit packets. 291758cc3dcSJack F Vogel */ 292758cc3dcSJack F Vogel void 293758cc3dcSJack F Vogel ixgbe_deferred_mq_start(void *arg, int pending) 294758cc3dcSJack F Vogel { 295758cc3dcSJack F Vogel struct tx_ring *txr = arg; 296758cc3dcSJack F Vogel struct adapter *adapter = txr->adapter; 297758cc3dcSJack F Vogel struct ifnet *ifp = adapter->ifp; 298758cc3dcSJack F Vogel 299758cc3dcSJack F Vogel IXGBE_TX_LOCK(txr); 300758cc3dcSJack F Vogel if (!drbr_empty(ifp, txr->br)) 301758cc3dcSJack F Vogel ixgbe_mq_start_locked(ifp, txr); 302758cc3dcSJack F Vogel IXGBE_TX_UNLOCK(txr); 303758cc3dcSJack F Vogel } 304758cc3dcSJack F Vogel 305758cc3dcSJack F Vogel /* 306758cc3dcSJack F Vogel ** Flush all ring buffers 307758cc3dcSJack F Vogel */ 308758cc3dcSJack F Vogel void 309758cc3dcSJack F Vogel ixgbe_qflush(struct ifnet *ifp) 310758cc3dcSJack F Vogel { 311758cc3dcSJack F Vogel struct adapter *adapter = ifp->if_softc; 312758cc3dcSJack F Vogel struct tx_ring *txr = adapter->tx_rings; 313758cc3dcSJack F Vogel struct mbuf *m; 314758cc3dcSJack F Vogel 315758cc3dcSJack F Vogel for (int i = 0; i < adapter->num_queues; i++, txr++) { 316758cc3dcSJack F Vogel IXGBE_TX_LOCK(txr); 317758cc3dcSJack F Vogel while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) 318758cc3dcSJack F Vogel m_freem(m); 319758cc3dcSJack F Vogel IXGBE_TX_UNLOCK(txr); 320758cc3dcSJack F Vogel } 321758cc3dcSJack F Vogel if_qflush(ifp); 322758cc3dcSJack F Vogel } 323758cc3dcSJack F Vogel #endif /* IXGBE_LEGACY_TX */ 324758cc3dcSJack F Vogel 325758cc3dcSJack F Vogel 326758cc3dcSJack F Vogel /********************************************************************* 327758cc3dcSJack F Vogel * 328758cc3dcSJack F Vogel * This routine maps the mbufs to tx descriptors, allowing the 329758cc3dcSJack F Vogel * TX engine to transmit the packets. 330758cc3dcSJack F Vogel * - return 0 on success, positive on failure 331758cc3dcSJack F Vogel * 332758cc3dcSJack F Vogel **********************************************************************/ 333758cc3dcSJack F Vogel 334758cc3dcSJack F Vogel static int 335758cc3dcSJack F Vogel ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp) 336758cc3dcSJack F Vogel { 337758cc3dcSJack F Vogel struct adapter *adapter = txr->adapter; 338758cc3dcSJack F Vogel u32 olinfo_status = 0, cmd_type_len; 339758cc3dcSJack F Vogel int i, j, error, nsegs; 340758cc3dcSJack F Vogel int first; 341758cc3dcSJack F Vogel bool remap = TRUE; 342758cc3dcSJack F Vogel struct mbuf *m_head; 343758cc3dcSJack F Vogel bus_dma_segment_t segs[adapter->num_segs]; 344758cc3dcSJack F Vogel bus_dmamap_t map; 345758cc3dcSJack F Vogel struct ixgbe_tx_buf *txbuf; 346758cc3dcSJack F Vogel union ixgbe_adv_tx_desc *txd = NULL; 347758cc3dcSJack F Vogel 348758cc3dcSJack F Vogel m_head = *m_headp; 349758cc3dcSJack F Vogel 350758cc3dcSJack F Vogel /* Basic descriptor defines */ 351758cc3dcSJack F Vogel cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA | 352758cc3dcSJack F Vogel IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT); 353758cc3dcSJack F Vogel 354758cc3dcSJack F Vogel if (m_head->m_flags & M_VLANTAG) 355758cc3dcSJack F Vogel cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 356758cc3dcSJack F Vogel 357758cc3dcSJack F Vogel /* 358758cc3dcSJack F Vogel * Important to capture the first descriptor 359758cc3dcSJack F Vogel * used because it will contain the index of 360758cc3dcSJack F Vogel * the one we tell the hardware to report back 361758cc3dcSJack F Vogel */ 362758cc3dcSJack F Vogel first = txr->next_avail_desc; 363758cc3dcSJack F Vogel txbuf = &txr->tx_buffers[first]; 364758cc3dcSJack F Vogel map = txbuf->map; 365758cc3dcSJack F Vogel 366758cc3dcSJack F Vogel /* 367758cc3dcSJack F Vogel * Map the packet for DMA. 368758cc3dcSJack F Vogel */ 369758cc3dcSJack F Vogel retry: 370758cc3dcSJack F Vogel error = bus_dmamap_load_mbuf_sg(txr->txtag, map, 371758cc3dcSJack F Vogel *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 372758cc3dcSJack F Vogel 373758cc3dcSJack F Vogel if (__predict_false(error)) { 374758cc3dcSJack F Vogel struct mbuf *m; 375758cc3dcSJack F Vogel 376758cc3dcSJack F Vogel switch (error) { 377758cc3dcSJack F Vogel case EFBIG: 378758cc3dcSJack F Vogel /* Try it again? - one try */ 379758cc3dcSJack F Vogel if (remap == TRUE) { 380758cc3dcSJack F Vogel remap = FALSE; 381758cc3dcSJack F Vogel m = m_defrag(*m_headp, M_NOWAIT); 382758cc3dcSJack F Vogel if (m == NULL) { 383758cc3dcSJack F Vogel adapter->mbuf_defrag_failed++; 384758cc3dcSJack F Vogel m_freem(*m_headp); 385758cc3dcSJack F Vogel *m_headp = NULL; 386758cc3dcSJack F Vogel return (ENOBUFS); 387758cc3dcSJack F Vogel } 388758cc3dcSJack F Vogel *m_headp = m; 389758cc3dcSJack F Vogel goto retry; 390758cc3dcSJack F Vogel } else 391758cc3dcSJack F Vogel return (error); 392758cc3dcSJack F Vogel case ENOMEM: 393758cc3dcSJack F Vogel txr->no_tx_dma_setup++; 394758cc3dcSJack F Vogel return (error); 395758cc3dcSJack F Vogel default: 396758cc3dcSJack F Vogel txr->no_tx_dma_setup++; 397758cc3dcSJack F Vogel m_freem(*m_headp); 398758cc3dcSJack F Vogel *m_headp = NULL; 399758cc3dcSJack F Vogel return (error); 400758cc3dcSJack F Vogel } 401758cc3dcSJack F Vogel } 402758cc3dcSJack F Vogel 403758cc3dcSJack F Vogel /* Make certain there are enough descriptors */ 404758cc3dcSJack F Vogel if (nsegs > txr->tx_avail - 2) { 405758cc3dcSJack F Vogel txr->no_desc_avail++; 406758cc3dcSJack F Vogel bus_dmamap_unload(txr->txtag, map); 407758cc3dcSJack F Vogel return (ENOBUFS); 408758cc3dcSJack F Vogel } 409758cc3dcSJack F Vogel m_head = *m_headp; 410758cc3dcSJack F Vogel 411758cc3dcSJack F Vogel /* 412758cc3dcSJack F Vogel ** Set up the appropriate offload context 413758cc3dcSJack F Vogel ** this will consume the first descriptor 414758cc3dcSJack F Vogel */ 415758cc3dcSJack F Vogel error = ixgbe_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status); 416758cc3dcSJack F Vogel if (__predict_false(error)) { 417758cc3dcSJack F Vogel if (error == ENOBUFS) 418758cc3dcSJack F Vogel *m_headp = NULL; 419758cc3dcSJack F Vogel return (error); 420758cc3dcSJack F Vogel } 421758cc3dcSJack F Vogel 422758cc3dcSJack F Vogel #ifdef IXGBE_FDIR 423758cc3dcSJack F Vogel /* Do the flow director magic */ 424758cc3dcSJack F Vogel if ((txr->atr_sample) && (!adapter->fdir_reinit)) { 425758cc3dcSJack F Vogel ++txr->atr_count; 426758cc3dcSJack F Vogel if (txr->atr_count >= atr_sample_rate) { 427758cc3dcSJack F Vogel ixgbe_atr(txr, m_head); 428758cc3dcSJack F Vogel txr->atr_count = 0; 429758cc3dcSJack F Vogel } 430758cc3dcSJack F Vogel } 431758cc3dcSJack F Vogel #endif 432758cc3dcSJack F Vogel 433758cc3dcSJack F Vogel olinfo_status |= IXGBE_ADVTXD_CC; 434758cc3dcSJack F Vogel i = txr->next_avail_desc; 435758cc3dcSJack F Vogel for (j = 0; j < nsegs; j++) { 436758cc3dcSJack F Vogel bus_size_t seglen; 437758cc3dcSJack F Vogel bus_addr_t segaddr; 438758cc3dcSJack F Vogel 439758cc3dcSJack F Vogel txbuf = &txr->tx_buffers[i]; 440758cc3dcSJack F Vogel txd = &txr->tx_base[i]; 441758cc3dcSJack F Vogel seglen = segs[j].ds_len; 442758cc3dcSJack F Vogel segaddr = htole64(segs[j].ds_addr); 443758cc3dcSJack F Vogel 444758cc3dcSJack F Vogel txd->read.buffer_addr = segaddr; 445758cc3dcSJack F Vogel txd->read.cmd_type_len = htole32(txr->txd_cmd | 446758cc3dcSJack F Vogel cmd_type_len |seglen); 447758cc3dcSJack F Vogel txd->read.olinfo_status = htole32(olinfo_status); 448758cc3dcSJack F Vogel 449758cc3dcSJack F Vogel if (++i == txr->num_desc) 450758cc3dcSJack F Vogel i = 0; 451758cc3dcSJack F Vogel } 452758cc3dcSJack F Vogel 453758cc3dcSJack F Vogel txd->read.cmd_type_len |= 454758cc3dcSJack F Vogel htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS); 455758cc3dcSJack F Vogel txr->tx_avail -= nsegs; 456758cc3dcSJack F Vogel txr->next_avail_desc = i; 457758cc3dcSJack F Vogel 458758cc3dcSJack F Vogel txbuf->m_head = m_head; 459758cc3dcSJack F Vogel /* 460758cc3dcSJack F Vogel ** Here we swap the map so the last descriptor, 461758cc3dcSJack F Vogel ** which gets the completion interrupt has the 462758cc3dcSJack F Vogel ** real map, and the first descriptor gets the 463758cc3dcSJack F Vogel ** unused map from this descriptor. 464758cc3dcSJack F Vogel */ 465758cc3dcSJack F Vogel txr->tx_buffers[first].map = txbuf->map; 466758cc3dcSJack F Vogel txbuf->map = map; 467758cc3dcSJack F Vogel bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE); 468758cc3dcSJack F Vogel 469758cc3dcSJack F Vogel /* Set the EOP descriptor that will be marked done */ 470758cc3dcSJack F Vogel txbuf = &txr->tx_buffers[first]; 471758cc3dcSJack F Vogel txbuf->eop = txd; 472758cc3dcSJack F Vogel 473758cc3dcSJack F Vogel bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 474758cc3dcSJack F Vogel BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 475758cc3dcSJack F Vogel /* 476758cc3dcSJack F Vogel * Advance the Transmit Descriptor Tail (Tdt), this tells the 477758cc3dcSJack F Vogel * hardware that this frame is available to transmit. 478758cc3dcSJack F Vogel */ 479758cc3dcSJack F Vogel ++txr->total_packets; 480758cc3dcSJack F Vogel IXGBE_WRITE_REG(&adapter->hw, txr->tail, i); 481758cc3dcSJack F Vogel 482758cc3dcSJack F Vogel /* Mark queue as having work */ 483758cc3dcSJack F Vogel if (txr->busy == 0) 484758cc3dcSJack F Vogel txr->busy = 1; 485758cc3dcSJack F Vogel 486758cc3dcSJack F Vogel return (0); 487758cc3dcSJack F Vogel 488758cc3dcSJack F Vogel } 489758cc3dcSJack F Vogel 490758cc3dcSJack F Vogel 491758cc3dcSJack F Vogel /********************************************************************* 492758cc3dcSJack F Vogel * 493758cc3dcSJack F Vogel * Allocate memory for tx_buffer structures. The tx_buffer stores all 494758cc3dcSJack F Vogel * the information needed to transmit a packet on the wire. This is 495758cc3dcSJack F Vogel * called only once at attach, setup is done every reset. 496758cc3dcSJack F Vogel * 497758cc3dcSJack F Vogel **********************************************************************/ 498758cc3dcSJack F Vogel int 499758cc3dcSJack F Vogel ixgbe_allocate_transmit_buffers(struct tx_ring *txr) 500758cc3dcSJack F Vogel { 501758cc3dcSJack F Vogel struct adapter *adapter = txr->adapter; 502758cc3dcSJack F Vogel device_t dev = adapter->dev; 503758cc3dcSJack F Vogel struct ixgbe_tx_buf *txbuf; 504758cc3dcSJack F Vogel int error, i; 505758cc3dcSJack F Vogel 506758cc3dcSJack F Vogel /* 507758cc3dcSJack F Vogel * Setup DMA descriptor areas. 508758cc3dcSJack F Vogel */ 509758cc3dcSJack F Vogel if ((error = bus_dma_tag_create( 510758cc3dcSJack F Vogel bus_get_dma_tag(adapter->dev), /* parent */ 511758cc3dcSJack F Vogel 1, 0, /* alignment, bounds */ 512758cc3dcSJack F Vogel BUS_SPACE_MAXADDR, /* lowaddr */ 513758cc3dcSJack F Vogel BUS_SPACE_MAXADDR, /* highaddr */ 514758cc3dcSJack F Vogel NULL, NULL, /* filter, filterarg */ 515758cc3dcSJack F Vogel IXGBE_TSO_SIZE, /* maxsize */ 516758cc3dcSJack F Vogel adapter->num_segs, /* nsegments */ 517758cc3dcSJack F Vogel PAGE_SIZE, /* maxsegsize */ 518758cc3dcSJack F Vogel 0, /* flags */ 519758cc3dcSJack F Vogel NULL, /* lockfunc */ 520758cc3dcSJack F Vogel NULL, /* lockfuncarg */ 521758cc3dcSJack F Vogel &txr->txtag))) { 522758cc3dcSJack F Vogel device_printf(dev,"Unable to allocate TX DMA tag\n"); 523758cc3dcSJack F Vogel goto fail; 524758cc3dcSJack F Vogel } 525758cc3dcSJack F Vogel 526758cc3dcSJack F Vogel if (!(txr->tx_buffers = 527758cc3dcSJack F Vogel (struct ixgbe_tx_buf *) malloc(sizeof(struct ixgbe_tx_buf) * 528758cc3dcSJack F Vogel adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { 529758cc3dcSJack F Vogel device_printf(dev, "Unable to allocate tx_buffer memory\n"); 530758cc3dcSJack F Vogel error = ENOMEM; 531758cc3dcSJack F Vogel goto fail; 532758cc3dcSJack F Vogel } 533758cc3dcSJack F Vogel 534758cc3dcSJack F Vogel /* Create the descriptor buffer dma maps */ 535758cc3dcSJack F Vogel txbuf = txr->tx_buffers; 536758cc3dcSJack F Vogel for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 537758cc3dcSJack F Vogel error = bus_dmamap_create(txr->txtag, 0, &txbuf->map); 538758cc3dcSJack F Vogel if (error != 0) { 539758cc3dcSJack F Vogel device_printf(dev, "Unable to create TX DMA map\n"); 540758cc3dcSJack F Vogel goto fail; 541758cc3dcSJack F Vogel } 542758cc3dcSJack F Vogel } 543758cc3dcSJack F Vogel 544758cc3dcSJack F Vogel return 0; 545758cc3dcSJack F Vogel fail: 546758cc3dcSJack F Vogel /* We free all, it handles case where we are in the middle */ 547758cc3dcSJack F Vogel ixgbe_free_transmit_structures(adapter); 548758cc3dcSJack F Vogel return (error); 549758cc3dcSJack F Vogel } 550758cc3dcSJack F Vogel 551758cc3dcSJack F Vogel /********************************************************************* 552758cc3dcSJack F Vogel * 553758cc3dcSJack F Vogel * Initialize a transmit ring. 554758cc3dcSJack F Vogel * 555758cc3dcSJack F Vogel **********************************************************************/ 556758cc3dcSJack F Vogel static void 557758cc3dcSJack F Vogel ixgbe_setup_transmit_ring(struct tx_ring *txr) 558758cc3dcSJack F Vogel { 559758cc3dcSJack F Vogel struct adapter *adapter = txr->adapter; 560758cc3dcSJack F Vogel struct ixgbe_tx_buf *txbuf; 561758cc3dcSJack F Vogel int i; 562758cc3dcSJack F Vogel #ifdef DEV_NETMAP 563758cc3dcSJack F Vogel struct netmap_adapter *na = NA(adapter->ifp); 564758cc3dcSJack F Vogel struct netmap_slot *slot; 565758cc3dcSJack F Vogel #endif /* DEV_NETMAP */ 566758cc3dcSJack F Vogel 567758cc3dcSJack F Vogel /* Clear the old ring contents */ 568758cc3dcSJack F Vogel IXGBE_TX_LOCK(txr); 569758cc3dcSJack F Vogel #ifdef DEV_NETMAP 570758cc3dcSJack F Vogel /* 571758cc3dcSJack F Vogel * (under lock): if in netmap mode, do some consistency 572758cc3dcSJack F Vogel * checks and set slot to entry 0 of the netmap ring. 573758cc3dcSJack F Vogel */ 574758cc3dcSJack F Vogel slot = netmap_reset(na, NR_TX, txr->me, 0); 575758cc3dcSJack F Vogel #endif /* DEV_NETMAP */ 576758cc3dcSJack F Vogel bzero((void *)txr->tx_base, 577758cc3dcSJack F Vogel (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc); 578758cc3dcSJack F Vogel /* Reset indices */ 579758cc3dcSJack F Vogel txr->next_avail_desc = 0; 580758cc3dcSJack F Vogel txr->next_to_clean = 0; 581758cc3dcSJack F Vogel 582758cc3dcSJack F Vogel /* Free any existing tx buffers. */ 583758cc3dcSJack F Vogel txbuf = txr->tx_buffers; 584758cc3dcSJack F Vogel for (i = 0; i < txr->num_desc; i++, txbuf++) { 585758cc3dcSJack F Vogel if (txbuf->m_head != NULL) { 586758cc3dcSJack F Vogel bus_dmamap_sync(txr->txtag, txbuf->map, 587758cc3dcSJack F Vogel BUS_DMASYNC_POSTWRITE); 588758cc3dcSJack F Vogel bus_dmamap_unload(txr->txtag, txbuf->map); 589758cc3dcSJack F Vogel m_freem(txbuf->m_head); 590758cc3dcSJack F Vogel txbuf->m_head = NULL; 591758cc3dcSJack F Vogel } 592758cc3dcSJack F Vogel #ifdef DEV_NETMAP 593758cc3dcSJack F Vogel /* 594758cc3dcSJack F Vogel * In netmap mode, set the map for the packet buffer. 595758cc3dcSJack F Vogel * NOTE: Some drivers (not this one) also need to set 596758cc3dcSJack F Vogel * the physical buffer address in the NIC ring. 597758cc3dcSJack F Vogel * Slots in the netmap ring (indexed by "si") are 598758cc3dcSJack F Vogel * kring->nkr_hwofs positions "ahead" wrt the 599758cc3dcSJack F Vogel * corresponding slot in the NIC ring. In some drivers 600758cc3dcSJack F Vogel * (not here) nkr_hwofs can be negative. Function 601758cc3dcSJack F Vogel * netmap_idx_n2k() handles wraparounds properly. 602758cc3dcSJack F Vogel */ 603758cc3dcSJack F Vogel if (slot) { 604758cc3dcSJack F Vogel int si = netmap_idx_n2k(&na->tx_rings[txr->me], i); 605758cc3dcSJack F Vogel netmap_load_map(na, txr->txtag, txbuf->map, NMB(na, slot + si)); 606758cc3dcSJack F Vogel } 607758cc3dcSJack F Vogel #endif /* DEV_NETMAP */ 608758cc3dcSJack F Vogel /* Clear the EOP descriptor pointer */ 609758cc3dcSJack F Vogel txbuf->eop = NULL; 610758cc3dcSJack F Vogel } 611758cc3dcSJack F Vogel 612758cc3dcSJack F Vogel #ifdef IXGBE_FDIR 613758cc3dcSJack F Vogel /* Set the rate at which we sample packets */ 614758cc3dcSJack F Vogel if (adapter->hw.mac.type != ixgbe_mac_82598EB) 615758cc3dcSJack F Vogel txr->atr_sample = atr_sample_rate; 616758cc3dcSJack F Vogel #endif 617758cc3dcSJack F Vogel 618758cc3dcSJack F Vogel /* Set number of descriptors available */ 619758cc3dcSJack F Vogel txr->tx_avail = adapter->num_tx_desc; 620758cc3dcSJack F Vogel 621758cc3dcSJack F Vogel bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 622758cc3dcSJack F Vogel BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 623758cc3dcSJack F Vogel IXGBE_TX_UNLOCK(txr); 624758cc3dcSJack F Vogel } 625758cc3dcSJack F Vogel 626758cc3dcSJack F Vogel /********************************************************************* 627758cc3dcSJack F Vogel * 628758cc3dcSJack F Vogel * Initialize all transmit rings. 629758cc3dcSJack F Vogel * 630758cc3dcSJack F Vogel **********************************************************************/ 631758cc3dcSJack F Vogel int 632758cc3dcSJack F Vogel ixgbe_setup_transmit_structures(struct adapter *adapter) 633758cc3dcSJack F Vogel { 634758cc3dcSJack F Vogel struct tx_ring *txr = adapter->tx_rings; 635758cc3dcSJack F Vogel 636758cc3dcSJack F Vogel for (int i = 0; i < adapter->num_queues; i++, txr++) 637758cc3dcSJack F Vogel ixgbe_setup_transmit_ring(txr); 638758cc3dcSJack F Vogel 639758cc3dcSJack F Vogel return (0); 640758cc3dcSJack F Vogel } 641758cc3dcSJack F Vogel 642758cc3dcSJack F Vogel /********************************************************************* 643758cc3dcSJack F Vogel * 644758cc3dcSJack F Vogel * Free all transmit rings. 645758cc3dcSJack F Vogel * 646758cc3dcSJack F Vogel **********************************************************************/ 647758cc3dcSJack F Vogel void 648758cc3dcSJack F Vogel ixgbe_free_transmit_structures(struct adapter *adapter) 649758cc3dcSJack F Vogel { 650758cc3dcSJack F Vogel struct tx_ring *txr = adapter->tx_rings; 651758cc3dcSJack F Vogel 652758cc3dcSJack F Vogel for (int i = 0; i < adapter->num_queues; i++, txr++) { 653758cc3dcSJack F Vogel IXGBE_TX_LOCK(txr); 654758cc3dcSJack F Vogel ixgbe_free_transmit_buffers(txr); 655758cc3dcSJack F Vogel ixgbe_dma_free(adapter, &txr->txdma); 656758cc3dcSJack F Vogel IXGBE_TX_UNLOCK(txr); 657758cc3dcSJack F Vogel IXGBE_TX_LOCK_DESTROY(txr); 658758cc3dcSJack F Vogel } 659758cc3dcSJack F Vogel free(adapter->tx_rings, M_DEVBUF); 660758cc3dcSJack F Vogel } 661758cc3dcSJack F Vogel 662758cc3dcSJack F Vogel /********************************************************************* 663758cc3dcSJack F Vogel * 664758cc3dcSJack F Vogel * Free transmit ring related data structures. 665758cc3dcSJack F Vogel * 666758cc3dcSJack F Vogel **********************************************************************/ 667758cc3dcSJack F Vogel static void 668758cc3dcSJack F Vogel ixgbe_free_transmit_buffers(struct tx_ring *txr) 669758cc3dcSJack F Vogel { 670758cc3dcSJack F Vogel struct adapter *adapter = txr->adapter; 671758cc3dcSJack F Vogel struct ixgbe_tx_buf *tx_buffer; 672758cc3dcSJack F Vogel int i; 673758cc3dcSJack F Vogel 674758cc3dcSJack F Vogel INIT_DEBUGOUT("ixgbe_free_transmit_ring: begin"); 675758cc3dcSJack F Vogel 676758cc3dcSJack F Vogel if (txr->tx_buffers == NULL) 677758cc3dcSJack F Vogel return; 678758cc3dcSJack F Vogel 679758cc3dcSJack F Vogel tx_buffer = txr->tx_buffers; 680758cc3dcSJack F Vogel for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { 681758cc3dcSJack F Vogel if (tx_buffer->m_head != NULL) { 682758cc3dcSJack F Vogel bus_dmamap_sync(txr->txtag, tx_buffer->map, 683758cc3dcSJack F Vogel BUS_DMASYNC_POSTWRITE); 684758cc3dcSJack F Vogel bus_dmamap_unload(txr->txtag, 685758cc3dcSJack F Vogel tx_buffer->map); 686758cc3dcSJack F Vogel m_freem(tx_buffer->m_head); 687758cc3dcSJack F Vogel tx_buffer->m_head = NULL; 688758cc3dcSJack F Vogel if (tx_buffer->map != NULL) { 689758cc3dcSJack F Vogel bus_dmamap_destroy(txr->txtag, 690758cc3dcSJack F Vogel tx_buffer->map); 691758cc3dcSJack F Vogel tx_buffer->map = NULL; 692758cc3dcSJack F Vogel } 693758cc3dcSJack F Vogel } else if (tx_buffer->map != NULL) { 694758cc3dcSJack F Vogel bus_dmamap_unload(txr->txtag, 695758cc3dcSJack F Vogel tx_buffer->map); 696758cc3dcSJack F Vogel bus_dmamap_destroy(txr->txtag, 697758cc3dcSJack F Vogel tx_buffer->map); 698758cc3dcSJack F Vogel tx_buffer->map = NULL; 699758cc3dcSJack F Vogel } 700758cc3dcSJack F Vogel } 701758cc3dcSJack F Vogel #ifdef IXGBE_LEGACY_TX 702758cc3dcSJack F Vogel if (txr->br != NULL) 703758cc3dcSJack F Vogel buf_ring_free(txr->br, M_DEVBUF); 704758cc3dcSJack F Vogel #endif 705758cc3dcSJack F Vogel if (txr->tx_buffers != NULL) { 706758cc3dcSJack F Vogel free(txr->tx_buffers, M_DEVBUF); 707758cc3dcSJack F Vogel txr->tx_buffers = NULL; 708758cc3dcSJack F Vogel } 709758cc3dcSJack F Vogel if (txr->txtag != NULL) { 710758cc3dcSJack F Vogel bus_dma_tag_destroy(txr->txtag); 711758cc3dcSJack F Vogel txr->txtag = NULL; 712758cc3dcSJack F Vogel } 713758cc3dcSJack F Vogel return; 714758cc3dcSJack F Vogel } 715758cc3dcSJack F Vogel 716758cc3dcSJack F Vogel /********************************************************************* 717758cc3dcSJack F Vogel * 718758cc3dcSJack F Vogel * Advanced Context Descriptor setup for VLAN, CSUM or TSO 719758cc3dcSJack F Vogel * 720758cc3dcSJack F Vogel **********************************************************************/ 721758cc3dcSJack F Vogel 722758cc3dcSJack F Vogel static int 723758cc3dcSJack F Vogel ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp, 724758cc3dcSJack F Vogel u32 *cmd_type_len, u32 *olinfo_status) 725758cc3dcSJack F Vogel { 726758cc3dcSJack F Vogel struct ixgbe_adv_tx_context_desc *TXD; 727758cc3dcSJack F Vogel struct ether_vlan_header *eh; 728758cc3dcSJack F Vogel struct ip *ip; 729758cc3dcSJack F Vogel struct ip6_hdr *ip6; 730758cc3dcSJack F Vogel u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; 731758cc3dcSJack F Vogel int ehdrlen, ip_hlen = 0; 732758cc3dcSJack F Vogel u16 etype; 733758cc3dcSJack F Vogel u8 ipproto = 0; 734758cc3dcSJack F Vogel int offload = TRUE; 735758cc3dcSJack F Vogel int ctxd = txr->next_avail_desc; 736758cc3dcSJack F Vogel u16 vtag = 0; 737758cc3dcSJack F Vogel 738758cc3dcSJack F Vogel /* First check if TSO is to be used */ 739758cc3dcSJack F Vogel if (mp->m_pkthdr.csum_flags & CSUM_TSO) 740758cc3dcSJack F Vogel return (ixgbe_tso_setup(txr, mp, cmd_type_len, olinfo_status)); 741758cc3dcSJack F Vogel 742758cc3dcSJack F Vogel if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0) 743758cc3dcSJack F Vogel offload = FALSE; 744758cc3dcSJack F Vogel 745758cc3dcSJack F Vogel /* Indicate the whole packet as payload when not doing TSO */ 746758cc3dcSJack F Vogel *olinfo_status |= mp->m_pkthdr.len << IXGBE_ADVTXD_PAYLEN_SHIFT; 747758cc3dcSJack F Vogel 748758cc3dcSJack F Vogel /* Now ready a context descriptor */ 749758cc3dcSJack F Vogel TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 750758cc3dcSJack F Vogel 751758cc3dcSJack F Vogel /* 752758cc3dcSJack F Vogel ** In advanced descriptors the vlan tag must 753758cc3dcSJack F Vogel ** be placed into the context descriptor. Hence 754758cc3dcSJack F Vogel ** we need to make one even if not doing offloads. 755758cc3dcSJack F Vogel */ 756758cc3dcSJack F Vogel if (mp->m_flags & M_VLANTAG) { 757758cc3dcSJack F Vogel vtag = htole16(mp->m_pkthdr.ether_vtag); 758758cc3dcSJack F Vogel vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 759758cc3dcSJack F Vogel } 760758cc3dcSJack F Vogel 761758cc3dcSJack F Vogel /* 762758cc3dcSJack F Vogel * Determine where frame payload starts. 763758cc3dcSJack F Vogel * Jump over vlan headers if already present, 764758cc3dcSJack F Vogel * helpful for QinQ too. 765758cc3dcSJack F Vogel */ 766758cc3dcSJack F Vogel eh = mtod(mp, struct ether_vlan_header *); 767758cc3dcSJack F Vogel if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 768758cc3dcSJack F Vogel etype = ntohs(eh->evl_proto); 769758cc3dcSJack F Vogel ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 770758cc3dcSJack F Vogel } else { 771758cc3dcSJack F Vogel etype = ntohs(eh->evl_encap_proto); 772758cc3dcSJack F Vogel ehdrlen = ETHER_HDR_LEN; 773758cc3dcSJack F Vogel } 774758cc3dcSJack F Vogel 775758cc3dcSJack F Vogel /* Set the ether header length */ 776758cc3dcSJack F Vogel vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 777758cc3dcSJack F Vogel 778758cc3dcSJack F Vogel if (offload == FALSE) 779758cc3dcSJack F Vogel goto no_offloads; 780758cc3dcSJack F Vogel 781758cc3dcSJack F Vogel switch (etype) { 782758cc3dcSJack F Vogel case ETHERTYPE_IP: 783758cc3dcSJack F Vogel ip = (struct ip *)(mp->m_data + ehdrlen); 784758cc3dcSJack F Vogel ip_hlen = ip->ip_hl << 2; 785758cc3dcSJack F Vogel ipproto = ip->ip_p; 786758cc3dcSJack F Vogel type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 787758cc3dcSJack F Vogel break; 788758cc3dcSJack F Vogel case ETHERTYPE_IPV6: 789758cc3dcSJack F Vogel ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 790758cc3dcSJack F Vogel ip_hlen = sizeof(struct ip6_hdr); 791758cc3dcSJack F Vogel /* XXX-BZ this will go badly in case of ext hdrs. */ 792758cc3dcSJack F Vogel ipproto = ip6->ip6_nxt; 793758cc3dcSJack F Vogel type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; 794758cc3dcSJack F Vogel break; 795758cc3dcSJack F Vogel default: 796758cc3dcSJack F Vogel offload = FALSE; 797758cc3dcSJack F Vogel break; 798758cc3dcSJack F Vogel } 799758cc3dcSJack F Vogel 800758cc3dcSJack F Vogel vlan_macip_lens |= ip_hlen; 801758cc3dcSJack F Vogel 802758cc3dcSJack F Vogel switch (ipproto) { 803758cc3dcSJack F Vogel case IPPROTO_TCP: 804758cc3dcSJack F Vogel if (mp->m_pkthdr.csum_flags & CSUM_TCP) 805758cc3dcSJack F Vogel type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 806758cc3dcSJack F Vogel break; 807758cc3dcSJack F Vogel 808758cc3dcSJack F Vogel case IPPROTO_UDP: 809758cc3dcSJack F Vogel if (mp->m_pkthdr.csum_flags & CSUM_UDP) 810758cc3dcSJack F Vogel type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP; 811758cc3dcSJack F Vogel break; 812758cc3dcSJack F Vogel 813758cc3dcSJack F Vogel #if __FreeBSD_version >= 800000 814758cc3dcSJack F Vogel case IPPROTO_SCTP: 815758cc3dcSJack F Vogel if (mp->m_pkthdr.csum_flags & CSUM_SCTP) 816758cc3dcSJack F Vogel type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP; 817758cc3dcSJack F Vogel break; 818758cc3dcSJack F Vogel #endif 819758cc3dcSJack F Vogel default: 820758cc3dcSJack F Vogel offload = FALSE; 821758cc3dcSJack F Vogel break; 822758cc3dcSJack F Vogel } 823758cc3dcSJack F Vogel 824758cc3dcSJack F Vogel if (offload) /* For the TX descriptor setup */ 825758cc3dcSJack F Vogel *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 826758cc3dcSJack F Vogel 827758cc3dcSJack F Vogel no_offloads: 828758cc3dcSJack F Vogel type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 829758cc3dcSJack F Vogel 830758cc3dcSJack F Vogel /* Now copy bits into descriptor */ 831758cc3dcSJack F Vogel TXD->vlan_macip_lens = htole32(vlan_macip_lens); 832758cc3dcSJack F Vogel TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); 833758cc3dcSJack F Vogel TXD->seqnum_seed = htole32(0); 834758cc3dcSJack F Vogel TXD->mss_l4len_idx = htole32(0); 835758cc3dcSJack F Vogel 836758cc3dcSJack F Vogel /* We've consumed the first desc, adjust counters */ 837758cc3dcSJack F Vogel if (++ctxd == txr->num_desc) 838758cc3dcSJack F Vogel ctxd = 0; 839758cc3dcSJack F Vogel txr->next_avail_desc = ctxd; 840758cc3dcSJack F Vogel --txr->tx_avail; 841758cc3dcSJack F Vogel 842758cc3dcSJack F Vogel return (0); 843758cc3dcSJack F Vogel } 844758cc3dcSJack F Vogel 845758cc3dcSJack F Vogel /********************************************************************** 846758cc3dcSJack F Vogel * 847758cc3dcSJack F Vogel * Setup work for hardware segmentation offload (TSO) on 848758cc3dcSJack F Vogel * adapters using advanced tx descriptors 849758cc3dcSJack F Vogel * 850758cc3dcSJack F Vogel **********************************************************************/ 851758cc3dcSJack F Vogel static int 852758cc3dcSJack F Vogel ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, 853758cc3dcSJack F Vogel u32 *cmd_type_len, u32 *olinfo_status) 854758cc3dcSJack F Vogel { 855758cc3dcSJack F Vogel struct ixgbe_adv_tx_context_desc *TXD; 856758cc3dcSJack F Vogel u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; 857758cc3dcSJack F Vogel u32 mss_l4len_idx = 0, paylen; 858758cc3dcSJack F Vogel u16 vtag = 0, eh_type; 859758cc3dcSJack F Vogel int ctxd, ehdrlen, ip_hlen, tcp_hlen; 860758cc3dcSJack F Vogel struct ether_vlan_header *eh; 861758cc3dcSJack F Vogel #ifdef INET6 862758cc3dcSJack F Vogel struct ip6_hdr *ip6; 863758cc3dcSJack F Vogel #endif 864758cc3dcSJack F Vogel #ifdef INET 865758cc3dcSJack F Vogel struct ip *ip; 866758cc3dcSJack F Vogel #endif 867758cc3dcSJack F Vogel struct tcphdr *th; 868758cc3dcSJack F Vogel 869758cc3dcSJack F Vogel 870758cc3dcSJack F Vogel /* 871758cc3dcSJack F Vogel * Determine where frame payload starts. 872758cc3dcSJack F Vogel * Jump over vlan headers if already present 873758cc3dcSJack F Vogel */ 874758cc3dcSJack F Vogel eh = mtod(mp, struct ether_vlan_header *); 875758cc3dcSJack F Vogel if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 876758cc3dcSJack F Vogel ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 877758cc3dcSJack F Vogel eh_type = eh->evl_proto; 878758cc3dcSJack F Vogel } else { 879758cc3dcSJack F Vogel ehdrlen = ETHER_HDR_LEN; 880758cc3dcSJack F Vogel eh_type = eh->evl_encap_proto; 881758cc3dcSJack F Vogel } 882758cc3dcSJack F Vogel 883758cc3dcSJack F Vogel switch (ntohs(eh_type)) { 884758cc3dcSJack F Vogel #ifdef INET6 885758cc3dcSJack F Vogel case ETHERTYPE_IPV6: 886758cc3dcSJack F Vogel ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 887758cc3dcSJack F Vogel /* XXX-BZ For now we do not pretend to support ext. hdrs. */ 888758cc3dcSJack F Vogel if (ip6->ip6_nxt != IPPROTO_TCP) 889758cc3dcSJack F Vogel return (ENXIO); 890758cc3dcSJack F Vogel ip_hlen = sizeof(struct ip6_hdr); 891758cc3dcSJack F Vogel ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 892758cc3dcSJack F Vogel th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen); 893758cc3dcSJack F Vogel th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0); 894758cc3dcSJack F Vogel type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; 895758cc3dcSJack F Vogel break; 896758cc3dcSJack F Vogel #endif 897758cc3dcSJack F Vogel #ifdef INET 898758cc3dcSJack F Vogel case ETHERTYPE_IP: 899758cc3dcSJack F Vogel ip = (struct ip *)(mp->m_data + ehdrlen); 900758cc3dcSJack F Vogel if (ip->ip_p != IPPROTO_TCP) 901758cc3dcSJack F Vogel return (ENXIO); 902758cc3dcSJack F Vogel ip->ip_sum = 0; 903758cc3dcSJack F Vogel ip_hlen = ip->ip_hl << 2; 904758cc3dcSJack F Vogel th = (struct tcphdr *)((caddr_t)ip + ip_hlen); 905758cc3dcSJack F Vogel th->th_sum = in_pseudo(ip->ip_src.s_addr, 906758cc3dcSJack F Vogel ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 907758cc3dcSJack F Vogel type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 908758cc3dcSJack F Vogel /* Tell transmit desc to also do IPv4 checksum. */ 909758cc3dcSJack F Vogel *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; 910758cc3dcSJack F Vogel break; 911758cc3dcSJack F Vogel #endif 912758cc3dcSJack F Vogel default: 913758cc3dcSJack F Vogel panic("%s: CSUM_TSO but no supported IP version (0x%04x)", 914758cc3dcSJack F Vogel __func__, ntohs(eh_type)); 915758cc3dcSJack F Vogel break; 916758cc3dcSJack F Vogel } 917758cc3dcSJack F Vogel 918758cc3dcSJack F Vogel ctxd = txr->next_avail_desc; 919758cc3dcSJack F Vogel TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 920758cc3dcSJack F Vogel 921758cc3dcSJack F Vogel tcp_hlen = th->th_off << 2; 922758cc3dcSJack F Vogel 923758cc3dcSJack F Vogel /* This is used in the transmit desc in encap */ 924758cc3dcSJack F Vogel paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen; 925758cc3dcSJack F Vogel 926758cc3dcSJack F Vogel /* VLAN MACLEN IPLEN */ 927758cc3dcSJack F Vogel if (mp->m_flags & M_VLANTAG) { 928758cc3dcSJack F Vogel vtag = htole16(mp->m_pkthdr.ether_vtag); 929758cc3dcSJack F Vogel vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 930758cc3dcSJack F Vogel } 931758cc3dcSJack F Vogel 932758cc3dcSJack F Vogel vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 933758cc3dcSJack F Vogel vlan_macip_lens |= ip_hlen; 934758cc3dcSJack F Vogel TXD->vlan_macip_lens = htole32(vlan_macip_lens); 935758cc3dcSJack F Vogel 936758cc3dcSJack F Vogel /* ADV DTYPE TUCMD */ 937758cc3dcSJack F Vogel type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 938758cc3dcSJack F Vogel type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 939758cc3dcSJack F Vogel TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); 940758cc3dcSJack F Vogel 941758cc3dcSJack F Vogel /* MSS L4LEN IDX */ 942758cc3dcSJack F Vogel mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT); 943758cc3dcSJack F Vogel mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT); 944758cc3dcSJack F Vogel TXD->mss_l4len_idx = htole32(mss_l4len_idx); 945758cc3dcSJack F Vogel 946758cc3dcSJack F Vogel TXD->seqnum_seed = htole32(0); 947758cc3dcSJack F Vogel 948758cc3dcSJack F Vogel if (++ctxd == txr->num_desc) 949758cc3dcSJack F Vogel ctxd = 0; 950758cc3dcSJack F Vogel 951758cc3dcSJack F Vogel txr->tx_avail--; 952758cc3dcSJack F Vogel txr->next_avail_desc = ctxd; 953758cc3dcSJack F Vogel *cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; 954758cc3dcSJack F Vogel *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 955758cc3dcSJack F Vogel *olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; 956758cc3dcSJack F Vogel ++txr->tso_tx; 957758cc3dcSJack F Vogel return (0); 958758cc3dcSJack F Vogel } 959758cc3dcSJack F Vogel 960758cc3dcSJack F Vogel 961758cc3dcSJack F Vogel /********************************************************************** 962758cc3dcSJack F Vogel * 963758cc3dcSJack F Vogel * Examine each tx_buffer in the used queue. If the hardware is done 964758cc3dcSJack F Vogel * processing the packet then free associated resources. The 965758cc3dcSJack F Vogel * tx_buffer is put back on the free queue. 966758cc3dcSJack F Vogel * 967758cc3dcSJack F Vogel **********************************************************************/ 968758cc3dcSJack F Vogel void 969758cc3dcSJack F Vogel ixgbe_txeof(struct tx_ring *txr) 970758cc3dcSJack F Vogel { 971758cc3dcSJack F Vogel #ifdef DEV_NETMAP 972758cc3dcSJack F Vogel struct adapter *adapter = txr->adapter; 973758cc3dcSJack F Vogel struct ifnet *ifp = adapter->ifp; 974758cc3dcSJack F Vogel #endif 975758cc3dcSJack F Vogel u32 work, processed = 0; 976758cc3dcSJack F Vogel u16 limit = txr->process_limit; 977758cc3dcSJack F Vogel struct ixgbe_tx_buf *buf; 978758cc3dcSJack F Vogel union ixgbe_adv_tx_desc *txd; 979758cc3dcSJack F Vogel 980758cc3dcSJack F Vogel mtx_assert(&txr->tx_mtx, MA_OWNED); 981758cc3dcSJack F Vogel 982758cc3dcSJack F Vogel #ifdef DEV_NETMAP 983758cc3dcSJack F Vogel if (ifp->if_capenable & IFCAP_NETMAP) { 984758cc3dcSJack F Vogel struct netmap_adapter *na = NA(ifp); 985758cc3dcSJack F Vogel struct netmap_kring *kring = &na->tx_rings[txr->me]; 986758cc3dcSJack F Vogel txd = txr->tx_base; 987758cc3dcSJack F Vogel bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 988758cc3dcSJack F Vogel BUS_DMASYNC_POSTREAD); 989758cc3dcSJack F Vogel /* 990758cc3dcSJack F Vogel * In netmap mode, all the work is done in the context 991758cc3dcSJack F Vogel * of the client thread. Interrupt handlers only wake up 992758cc3dcSJack F Vogel * clients, which may be sleeping on individual rings 993758cc3dcSJack F Vogel * or on a global resource for all rings. 994758cc3dcSJack F Vogel * To implement tx interrupt mitigation, we wake up the client 995758cc3dcSJack F Vogel * thread roughly every half ring, even if the NIC interrupts 996758cc3dcSJack F Vogel * more frequently. This is implemented as follows: 997758cc3dcSJack F Vogel * - ixgbe_txsync() sets kring->nr_kflags with the index of 998758cc3dcSJack F Vogel * the slot that should wake up the thread (nkr_num_slots 999758cc3dcSJack F Vogel * means the user thread should not be woken up); 1000758cc3dcSJack F Vogel * - the driver ignores tx interrupts unless netmap_mitigate=0 1001758cc3dcSJack F Vogel * or the slot has the DD bit set. 1002758cc3dcSJack F Vogel */ 1003758cc3dcSJack F Vogel if (!netmap_mitigate || 1004758cc3dcSJack F Vogel (kring->nr_kflags < kring->nkr_num_slots && 1005758cc3dcSJack F Vogel txd[kring->nr_kflags].wb.status & IXGBE_TXD_STAT_DD)) { 1006758cc3dcSJack F Vogel netmap_tx_irq(ifp, txr->me); 1007758cc3dcSJack F Vogel } 1008758cc3dcSJack F Vogel return; 1009758cc3dcSJack F Vogel } 1010758cc3dcSJack F Vogel #endif /* DEV_NETMAP */ 1011758cc3dcSJack F Vogel 1012758cc3dcSJack F Vogel if (txr->tx_avail == txr->num_desc) { 1013758cc3dcSJack F Vogel txr->busy = 0; 1014758cc3dcSJack F Vogel return; 1015758cc3dcSJack F Vogel } 1016758cc3dcSJack F Vogel 1017758cc3dcSJack F Vogel /* Get work starting point */ 1018758cc3dcSJack F Vogel work = txr->next_to_clean; 1019758cc3dcSJack F Vogel buf = &txr->tx_buffers[work]; 1020758cc3dcSJack F Vogel txd = &txr->tx_base[work]; 1021758cc3dcSJack F Vogel work -= txr->num_desc; /* The distance to ring end */ 1022758cc3dcSJack F Vogel bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 1023758cc3dcSJack F Vogel BUS_DMASYNC_POSTREAD); 1024758cc3dcSJack F Vogel 1025758cc3dcSJack F Vogel do { 1026758cc3dcSJack F Vogel union ixgbe_adv_tx_desc *eop= buf->eop; 1027758cc3dcSJack F Vogel if (eop == NULL) /* No work */ 1028758cc3dcSJack F Vogel break; 1029758cc3dcSJack F Vogel 1030758cc3dcSJack F Vogel if ((eop->wb.status & IXGBE_TXD_STAT_DD) == 0) 1031758cc3dcSJack F Vogel break; /* I/O not complete */ 1032758cc3dcSJack F Vogel 1033758cc3dcSJack F Vogel if (buf->m_head) { 1034758cc3dcSJack F Vogel txr->bytes += 1035758cc3dcSJack F Vogel buf->m_head->m_pkthdr.len; 1036758cc3dcSJack F Vogel bus_dmamap_sync(txr->txtag, 1037758cc3dcSJack F Vogel buf->map, 1038758cc3dcSJack F Vogel BUS_DMASYNC_POSTWRITE); 1039758cc3dcSJack F Vogel bus_dmamap_unload(txr->txtag, 1040758cc3dcSJack F Vogel buf->map); 1041758cc3dcSJack F Vogel m_freem(buf->m_head); 1042758cc3dcSJack F Vogel buf->m_head = NULL; 1043758cc3dcSJack F Vogel buf->map = NULL; 1044758cc3dcSJack F Vogel } 1045758cc3dcSJack F Vogel buf->eop = NULL; 1046758cc3dcSJack F Vogel ++txr->tx_avail; 1047758cc3dcSJack F Vogel 1048758cc3dcSJack F Vogel /* We clean the range if multi segment */ 1049758cc3dcSJack F Vogel while (txd != eop) { 1050758cc3dcSJack F Vogel ++txd; 1051758cc3dcSJack F Vogel ++buf; 1052758cc3dcSJack F Vogel ++work; 1053758cc3dcSJack F Vogel /* wrap the ring? */ 1054758cc3dcSJack F Vogel if (__predict_false(!work)) { 1055758cc3dcSJack F Vogel work -= txr->num_desc; 1056758cc3dcSJack F Vogel buf = txr->tx_buffers; 1057758cc3dcSJack F Vogel txd = txr->tx_base; 1058758cc3dcSJack F Vogel } 1059758cc3dcSJack F Vogel if (buf->m_head) { 1060758cc3dcSJack F Vogel txr->bytes += 1061758cc3dcSJack F Vogel buf->m_head->m_pkthdr.len; 1062758cc3dcSJack F Vogel bus_dmamap_sync(txr->txtag, 1063758cc3dcSJack F Vogel buf->map, 1064758cc3dcSJack F Vogel BUS_DMASYNC_POSTWRITE); 1065758cc3dcSJack F Vogel bus_dmamap_unload(txr->txtag, 1066758cc3dcSJack F Vogel buf->map); 1067758cc3dcSJack F Vogel m_freem(buf->m_head); 1068758cc3dcSJack F Vogel buf->m_head = NULL; 1069758cc3dcSJack F Vogel buf->map = NULL; 1070758cc3dcSJack F Vogel } 1071758cc3dcSJack F Vogel ++txr->tx_avail; 1072758cc3dcSJack F Vogel buf->eop = NULL; 1073758cc3dcSJack F Vogel 1074758cc3dcSJack F Vogel } 1075758cc3dcSJack F Vogel ++txr->packets; 1076758cc3dcSJack F Vogel ++processed; 1077758cc3dcSJack F Vogel 1078758cc3dcSJack F Vogel /* Try the next packet */ 1079758cc3dcSJack F Vogel ++txd; 1080758cc3dcSJack F Vogel ++buf; 1081758cc3dcSJack F Vogel ++work; 1082758cc3dcSJack F Vogel /* reset with a wrap */ 1083758cc3dcSJack F Vogel if (__predict_false(!work)) { 1084758cc3dcSJack F Vogel work -= txr->num_desc; 1085758cc3dcSJack F Vogel buf = txr->tx_buffers; 1086758cc3dcSJack F Vogel txd = txr->tx_base; 1087758cc3dcSJack F Vogel } 1088758cc3dcSJack F Vogel prefetch(txd); 1089758cc3dcSJack F Vogel } while (__predict_true(--limit)); 1090758cc3dcSJack F Vogel 1091758cc3dcSJack F Vogel bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 1092758cc3dcSJack F Vogel BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1093758cc3dcSJack F Vogel 1094758cc3dcSJack F Vogel work += txr->num_desc; 1095758cc3dcSJack F Vogel txr->next_to_clean = work; 1096758cc3dcSJack F Vogel 1097758cc3dcSJack F Vogel /* 1098758cc3dcSJack F Vogel ** Queue Hang detection, we know there's 1099758cc3dcSJack F Vogel ** work outstanding or the first return 1100758cc3dcSJack F Vogel ** would have been taken, so increment busy 1101758cc3dcSJack F Vogel ** if nothing managed to get cleaned, then 1102758cc3dcSJack F Vogel ** in local_timer it will be checked and 1103758cc3dcSJack F Vogel ** marked as HUNG if it exceeds a MAX attempt. 1104758cc3dcSJack F Vogel */ 1105758cc3dcSJack F Vogel if ((processed == 0) && (txr->busy != IXGBE_QUEUE_HUNG)) 1106758cc3dcSJack F Vogel ++txr->busy; 1107758cc3dcSJack F Vogel /* 1108758cc3dcSJack F Vogel ** If anything gets cleaned we reset state to 1, 1109758cc3dcSJack F Vogel ** note this will turn off HUNG if its set. 1110758cc3dcSJack F Vogel */ 1111758cc3dcSJack F Vogel if (processed) 1112758cc3dcSJack F Vogel txr->busy = 1; 1113758cc3dcSJack F Vogel 1114758cc3dcSJack F Vogel if (txr->tx_avail == txr->num_desc) 1115758cc3dcSJack F Vogel txr->busy = 0; 1116758cc3dcSJack F Vogel 1117758cc3dcSJack F Vogel return; 1118758cc3dcSJack F Vogel } 1119758cc3dcSJack F Vogel 1120758cc3dcSJack F Vogel 1121758cc3dcSJack F Vogel #ifdef IXGBE_FDIR 1122758cc3dcSJack F Vogel /* 1123758cc3dcSJack F Vogel ** This routine parses packet headers so that Flow 1124758cc3dcSJack F Vogel ** Director can make a hashed filter table entry 1125758cc3dcSJack F Vogel ** allowing traffic flows to be identified and kept 1126758cc3dcSJack F Vogel ** on the same cpu. This would be a performance 1127758cc3dcSJack F Vogel ** hit, but we only do it at IXGBE_FDIR_RATE of 1128758cc3dcSJack F Vogel ** packets. 1129758cc3dcSJack F Vogel */ 1130758cc3dcSJack F Vogel static void 1131758cc3dcSJack F Vogel ixgbe_atr(struct tx_ring *txr, struct mbuf *mp) 1132758cc3dcSJack F Vogel { 1133758cc3dcSJack F Vogel struct adapter *adapter = txr->adapter; 1134758cc3dcSJack F Vogel struct ix_queue *que; 1135758cc3dcSJack F Vogel struct ip *ip; 1136758cc3dcSJack F Vogel struct tcphdr *th; 1137758cc3dcSJack F Vogel struct udphdr *uh; 1138758cc3dcSJack F Vogel struct ether_vlan_header *eh; 1139758cc3dcSJack F Vogel union ixgbe_atr_hash_dword input = {.dword = 0}; 1140758cc3dcSJack F Vogel union ixgbe_atr_hash_dword common = {.dword = 0}; 1141758cc3dcSJack F Vogel int ehdrlen, ip_hlen; 1142758cc3dcSJack F Vogel u16 etype; 1143758cc3dcSJack F Vogel 1144758cc3dcSJack F Vogel eh = mtod(mp, struct ether_vlan_header *); 1145758cc3dcSJack F Vogel if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 1146758cc3dcSJack F Vogel ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 1147758cc3dcSJack F Vogel etype = eh->evl_proto; 1148758cc3dcSJack F Vogel } else { 1149758cc3dcSJack F Vogel ehdrlen = ETHER_HDR_LEN; 1150758cc3dcSJack F Vogel etype = eh->evl_encap_proto; 1151758cc3dcSJack F Vogel } 1152758cc3dcSJack F Vogel 1153758cc3dcSJack F Vogel /* Only handling IPv4 */ 1154758cc3dcSJack F Vogel if (etype != htons(ETHERTYPE_IP)) 1155758cc3dcSJack F Vogel return; 1156758cc3dcSJack F Vogel 1157758cc3dcSJack F Vogel ip = (struct ip *)(mp->m_data + ehdrlen); 1158758cc3dcSJack F Vogel ip_hlen = ip->ip_hl << 2; 1159758cc3dcSJack F Vogel 1160758cc3dcSJack F Vogel /* check if we're UDP or TCP */ 1161758cc3dcSJack F Vogel switch (ip->ip_p) { 1162758cc3dcSJack F Vogel case IPPROTO_TCP: 1163758cc3dcSJack F Vogel th = (struct tcphdr *)((caddr_t)ip + ip_hlen); 1164758cc3dcSJack F Vogel /* src and dst are inverted */ 1165758cc3dcSJack F Vogel common.port.dst ^= th->th_sport; 1166758cc3dcSJack F Vogel common.port.src ^= th->th_dport; 1167758cc3dcSJack F Vogel input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4; 1168758cc3dcSJack F Vogel break; 1169758cc3dcSJack F Vogel case IPPROTO_UDP: 1170758cc3dcSJack F Vogel uh = (struct udphdr *)((caddr_t)ip + ip_hlen); 1171758cc3dcSJack F Vogel /* src and dst are inverted */ 1172758cc3dcSJack F Vogel common.port.dst ^= uh->uh_sport; 1173758cc3dcSJack F Vogel common.port.src ^= uh->uh_dport; 1174758cc3dcSJack F Vogel input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4; 1175758cc3dcSJack F Vogel break; 1176758cc3dcSJack F Vogel default: 1177758cc3dcSJack F Vogel return; 1178758cc3dcSJack F Vogel } 1179758cc3dcSJack F Vogel 1180758cc3dcSJack F Vogel input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag); 1181758cc3dcSJack F Vogel if (mp->m_pkthdr.ether_vtag) 1182758cc3dcSJack F Vogel common.flex_bytes ^= htons(ETHERTYPE_VLAN); 1183758cc3dcSJack F Vogel else 1184758cc3dcSJack F Vogel common.flex_bytes ^= etype; 1185758cc3dcSJack F Vogel common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr; 1186758cc3dcSJack F Vogel 1187758cc3dcSJack F Vogel que = &adapter->queues[txr->me]; 1188758cc3dcSJack F Vogel /* 1189758cc3dcSJack F Vogel ** This assumes the Rx queue and Tx 1190758cc3dcSJack F Vogel ** queue are bound to the same CPU 1191758cc3dcSJack F Vogel */ 1192758cc3dcSJack F Vogel ixgbe_fdir_add_signature_filter_82599(&adapter->hw, 1193758cc3dcSJack F Vogel input, common, que->msix); 1194758cc3dcSJack F Vogel } 1195758cc3dcSJack F Vogel #endif /* IXGBE_FDIR */ 1196758cc3dcSJack F Vogel 1197758cc3dcSJack F Vogel /* 1198758cc3dcSJack F Vogel ** Used to detect a descriptor that has 1199758cc3dcSJack F Vogel ** been merged by Hardware RSC. 1200758cc3dcSJack F Vogel */ 1201758cc3dcSJack F Vogel static inline u32 1202758cc3dcSJack F Vogel ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx) 1203758cc3dcSJack F Vogel { 1204758cc3dcSJack F Vogel return (le32toh(rx->wb.lower.lo_dword.data) & 1205758cc3dcSJack F Vogel IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT; 1206758cc3dcSJack F Vogel } 1207758cc3dcSJack F Vogel 1208758cc3dcSJack F Vogel /********************************************************************* 1209758cc3dcSJack F Vogel * 1210758cc3dcSJack F Vogel * Initialize Hardware RSC (LRO) feature on 82599 1211758cc3dcSJack F Vogel * for an RX ring, this is toggled by the LRO capability 1212758cc3dcSJack F Vogel * even though it is transparent to the stack. 1213758cc3dcSJack F Vogel * 1214758cc3dcSJack F Vogel * NOTE: since this HW feature only works with IPV4 and 1215758cc3dcSJack F Vogel * our testing has shown soft LRO to be as effective 1216758cc3dcSJack F Vogel * I have decided to disable this by default. 1217758cc3dcSJack F Vogel * 1218758cc3dcSJack F Vogel **********************************************************************/ 1219758cc3dcSJack F Vogel static void 1220758cc3dcSJack F Vogel ixgbe_setup_hw_rsc(struct rx_ring *rxr) 1221758cc3dcSJack F Vogel { 1222758cc3dcSJack F Vogel struct adapter *adapter = rxr->adapter; 1223758cc3dcSJack F Vogel struct ixgbe_hw *hw = &adapter->hw; 1224758cc3dcSJack F Vogel u32 rscctrl, rdrxctl; 1225758cc3dcSJack F Vogel 1226758cc3dcSJack F Vogel /* If turning LRO/RSC off we need to disable it */ 1227758cc3dcSJack F Vogel if ((adapter->ifp->if_capenable & IFCAP_LRO) == 0) { 1228758cc3dcSJack F Vogel rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me)); 1229758cc3dcSJack F Vogel rscctrl &= ~IXGBE_RSCCTL_RSCEN; 1230758cc3dcSJack F Vogel return; 1231758cc3dcSJack F Vogel } 1232758cc3dcSJack F Vogel 1233758cc3dcSJack F Vogel rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); 1234758cc3dcSJack F Vogel rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE; 1235758cc3dcSJack F Vogel #ifdef DEV_NETMAP /* crcstrip is optional in netmap */ 1236758cc3dcSJack F Vogel if (adapter->ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip) 1237758cc3dcSJack F Vogel #endif /* DEV_NETMAP */ 1238758cc3dcSJack F Vogel rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP; 1239758cc3dcSJack F Vogel rdrxctl |= IXGBE_RDRXCTL_RSCACKC; 1240758cc3dcSJack F Vogel IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl); 1241758cc3dcSJack F Vogel 1242758cc3dcSJack F Vogel rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me)); 1243758cc3dcSJack F Vogel rscctrl |= IXGBE_RSCCTL_RSCEN; 1244758cc3dcSJack F Vogel /* 1245758cc3dcSJack F Vogel ** Limit the total number of descriptors that 1246758cc3dcSJack F Vogel ** can be combined, so it does not exceed 64K 1247758cc3dcSJack F Vogel */ 1248758cc3dcSJack F Vogel if (rxr->mbuf_sz == MCLBYTES) 1249758cc3dcSJack F Vogel rscctrl |= IXGBE_RSCCTL_MAXDESC_16; 1250758cc3dcSJack F Vogel else if (rxr->mbuf_sz == MJUMPAGESIZE) 1251758cc3dcSJack F Vogel rscctrl |= IXGBE_RSCCTL_MAXDESC_8; 1252758cc3dcSJack F Vogel else if (rxr->mbuf_sz == MJUM9BYTES) 1253758cc3dcSJack F Vogel rscctrl |= IXGBE_RSCCTL_MAXDESC_4; 1254758cc3dcSJack F Vogel else /* Using 16K cluster */ 1255758cc3dcSJack F Vogel rscctrl |= IXGBE_RSCCTL_MAXDESC_1; 1256758cc3dcSJack F Vogel 1257758cc3dcSJack F Vogel IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl); 1258758cc3dcSJack F Vogel 1259758cc3dcSJack F Vogel /* Enable TCP header recognition */ 1260758cc3dcSJack F Vogel IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), 1261758cc3dcSJack F Vogel (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) | 1262758cc3dcSJack F Vogel IXGBE_PSRTYPE_TCPHDR)); 1263758cc3dcSJack F Vogel 1264758cc3dcSJack F Vogel /* Disable RSC for ACK packets */ 1265758cc3dcSJack F Vogel IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, 1266758cc3dcSJack F Vogel (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU))); 1267758cc3dcSJack F Vogel 1268758cc3dcSJack F Vogel rxr->hw_rsc = TRUE; 1269758cc3dcSJack F Vogel } 1270758cc3dcSJack F Vogel /********************************************************************* 1271758cc3dcSJack F Vogel * 1272758cc3dcSJack F Vogel * Refresh mbuf buffers for RX descriptor rings 1273758cc3dcSJack F Vogel * - now keeps its own state so discards due to resource 1274758cc3dcSJack F Vogel * exhaustion are unnecessary, if an mbuf cannot be obtained 1275758cc3dcSJack F Vogel * it just returns, keeping its placeholder, thus it can simply 1276758cc3dcSJack F Vogel * be recalled to try again. 1277758cc3dcSJack F Vogel * 1278758cc3dcSJack F Vogel **********************************************************************/ 1279758cc3dcSJack F Vogel static void 1280758cc3dcSJack F Vogel ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit) 1281758cc3dcSJack F Vogel { 1282758cc3dcSJack F Vogel struct adapter *adapter = rxr->adapter; 1283758cc3dcSJack F Vogel bus_dma_segment_t seg[1]; 1284758cc3dcSJack F Vogel struct ixgbe_rx_buf *rxbuf; 1285758cc3dcSJack F Vogel struct mbuf *mp; 1286758cc3dcSJack F Vogel int i, j, nsegs, error; 1287758cc3dcSJack F Vogel bool refreshed = FALSE; 1288758cc3dcSJack F Vogel 1289758cc3dcSJack F Vogel i = j = rxr->next_to_refresh; 1290758cc3dcSJack F Vogel /* Control the loop with one beyond */ 1291758cc3dcSJack F Vogel if (++j == rxr->num_desc) 1292758cc3dcSJack F Vogel j = 0; 1293758cc3dcSJack F Vogel 1294758cc3dcSJack F Vogel while (j != limit) { 1295758cc3dcSJack F Vogel rxbuf = &rxr->rx_buffers[i]; 1296758cc3dcSJack F Vogel if (rxbuf->buf == NULL) { 1297758cc3dcSJack F Vogel mp = m_getjcl(M_NOWAIT, MT_DATA, 1298758cc3dcSJack F Vogel M_PKTHDR, rxr->mbuf_sz); 1299758cc3dcSJack F Vogel if (mp == NULL) 1300758cc3dcSJack F Vogel goto update; 1301758cc3dcSJack F Vogel if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN)) 1302758cc3dcSJack F Vogel m_adj(mp, ETHER_ALIGN); 1303758cc3dcSJack F Vogel } else 1304758cc3dcSJack F Vogel mp = rxbuf->buf; 1305758cc3dcSJack F Vogel 1306758cc3dcSJack F Vogel mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz; 1307758cc3dcSJack F Vogel 1308758cc3dcSJack F Vogel /* If we're dealing with an mbuf that was copied rather 1309758cc3dcSJack F Vogel * than replaced, there's no need to go through busdma. 1310758cc3dcSJack F Vogel */ 1311758cc3dcSJack F Vogel if ((rxbuf->flags & IXGBE_RX_COPY) == 0) { 1312758cc3dcSJack F Vogel /* Get the memory mapping */ 1313758cc3dcSJack F Vogel error = bus_dmamap_load_mbuf_sg(rxr->ptag, 1314758cc3dcSJack F Vogel rxbuf->pmap, mp, seg, &nsegs, BUS_DMA_NOWAIT); 1315758cc3dcSJack F Vogel if (error != 0) { 1316758cc3dcSJack F Vogel printf("Refresh mbufs: payload dmamap load" 1317758cc3dcSJack F Vogel " failure - %d\n", error); 1318758cc3dcSJack F Vogel m_free(mp); 1319758cc3dcSJack F Vogel rxbuf->buf = NULL; 1320758cc3dcSJack F Vogel goto update; 1321758cc3dcSJack F Vogel } 1322758cc3dcSJack F Vogel rxbuf->buf = mp; 1323758cc3dcSJack F Vogel bus_dmamap_sync(rxr->ptag, rxbuf->pmap, 1324758cc3dcSJack F Vogel BUS_DMASYNC_PREREAD); 1325758cc3dcSJack F Vogel rxbuf->addr = rxr->rx_base[i].read.pkt_addr = 1326758cc3dcSJack F Vogel htole64(seg[0].ds_addr); 1327758cc3dcSJack F Vogel } else { 1328758cc3dcSJack F Vogel rxr->rx_base[i].read.pkt_addr = rxbuf->addr; 1329758cc3dcSJack F Vogel rxbuf->flags &= ~IXGBE_RX_COPY; 1330758cc3dcSJack F Vogel } 1331758cc3dcSJack F Vogel 1332758cc3dcSJack F Vogel refreshed = TRUE; 1333758cc3dcSJack F Vogel /* Next is precalculated */ 1334758cc3dcSJack F Vogel i = j; 1335758cc3dcSJack F Vogel rxr->next_to_refresh = i; 1336758cc3dcSJack F Vogel if (++j == rxr->num_desc) 1337758cc3dcSJack F Vogel j = 0; 1338758cc3dcSJack F Vogel } 1339758cc3dcSJack F Vogel update: 1340758cc3dcSJack F Vogel if (refreshed) /* Update hardware tail index */ 1341758cc3dcSJack F Vogel IXGBE_WRITE_REG(&adapter->hw, 1342758cc3dcSJack F Vogel rxr->tail, rxr->next_to_refresh); 1343758cc3dcSJack F Vogel return; 1344758cc3dcSJack F Vogel } 1345758cc3dcSJack F Vogel 1346758cc3dcSJack F Vogel /********************************************************************* 1347758cc3dcSJack F Vogel * 1348758cc3dcSJack F Vogel * Allocate memory for rx_buffer structures. Since we use one 1349758cc3dcSJack F Vogel * rx_buffer per received packet, the maximum number of rx_buffer's 1350758cc3dcSJack F Vogel * that we'll need is equal to the number of receive descriptors 1351758cc3dcSJack F Vogel * that we've allocated. 1352758cc3dcSJack F Vogel * 1353758cc3dcSJack F Vogel **********************************************************************/ 1354758cc3dcSJack F Vogel int 1355758cc3dcSJack F Vogel ixgbe_allocate_receive_buffers(struct rx_ring *rxr) 1356758cc3dcSJack F Vogel { 1357758cc3dcSJack F Vogel struct adapter *adapter = rxr->adapter; 1358758cc3dcSJack F Vogel device_t dev = adapter->dev; 1359758cc3dcSJack F Vogel struct ixgbe_rx_buf *rxbuf; 1360758cc3dcSJack F Vogel int i, bsize, error; 1361758cc3dcSJack F Vogel 1362758cc3dcSJack F Vogel bsize = sizeof(struct ixgbe_rx_buf) * rxr->num_desc; 1363758cc3dcSJack F Vogel if (!(rxr->rx_buffers = 1364758cc3dcSJack F Vogel (struct ixgbe_rx_buf *) malloc(bsize, 1365758cc3dcSJack F Vogel M_DEVBUF, M_NOWAIT | M_ZERO))) { 1366758cc3dcSJack F Vogel device_printf(dev, "Unable to allocate rx_buffer memory\n"); 1367758cc3dcSJack F Vogel error = ENOMEM; 1368758cc3dcSJack F Vogel goto fail; 1369758cc3dcSJack F Vogel } 1370758cc3dcSJack F Vogel 1371758cc3dcSJack F Vogel if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 1372758cc3dcSJack F Vogel 1, 0, /* alignment, bounds */ 1373758cc3dcSJack F Vogel BUS_SPACE_MAXADDR, /* lowaddr */ 1374758cc3dcSJack F Vogel BUS_SPACE_MAXADDR, /* highaddr */ 1375758cc3dcSJack F Vogel NULL, NULL, /* filter, filterarg */ 1376758cc3dcSJack F Vogel MJUM16BYTES, /* maxsize */ 1377758cc3dcSJack F Vogel 1, /* nsegments */ 1378758cc3dcSJack F Vogel MJUM16BYTES, /* maxsegsize */ 1379758cc3dcSJack F Vogel 0, /* flags */ 1380758cc3dcSJack F Vogel NULL, /* lockfunc */ 1381758cc3dcSJack F Vogel NULL, /* lockfuncarg */ 1382758cc3dcSJack F Vogel &rxr->ptag))) { 1383758cc3dcSJack F Vogel device_printf(dev, "Unable to create RX DMA tag\n"); 1384758cc3dcSJack F Vogel goto fail; 1385758cc3dcSJack F Vogel } 1386758cc3dcSJack F Vogel 1387758cc3dcSJack F Vogel for (i = 0; i < rxr->num_desc; i++, rxbuf++) { 1388758cc3dcSJack F Vogel rxbuf = &rxr->rx_buffers[i]; 1389758cc3dcSJack F Vogel error = bus_dmamap_create(rxr->ptag, 1390758cc3dcSJack F Vogel BUS_DMA_NOWAIT, &rxbuf->pmap); 1391758cc3dcSJack F Vogel if (error) { 1392758cc3dcSJack F Vogel device_printf(dev, "Unable to create RX dma map\n"); 1393758cc3dcSJack F Vogel goto fail; 1394758cc3dcSJack F Vogel } 1395758cc3dcSJack F Vogel } 1396758cc3dcSJack F Vogel 1397758cc3dcSJack F Vogel return (0); 1398758cc3dcSJack F Vogel 1399758cc3dcSJack F Vogel fail: 1400758cc3dcSJack F Vogel /* Frees all, but can handle partial completion */ 1401758cc3dcSJack F Vogel ixgbe_free_receive_structures(adapter); 1402758cc3dcSJack F Vogel return (error); 1403758cc3dcSJack F Vogel } 1404758cc3dcSJack F Vogel 1405758cc3dcSJack F Vogel 1406758cc3dcSJack F Vogel static void 1407758cc3dcSJack F Vogel ixgbe_free_receive_ring(struct rx_ring *rxr) 1408758cc3dcSJack F Vogel { 1409758cc3dcSJack F Vogel struct ixgbe_rx_buf *rxbuf; 1410758cc3dcSJack F Vogel int i; 1411758cc3dcSJack F Vogel 1412758cc3dcSJack F Vogel for (i = 0; i < rxr->num_desc; i++) { 1413758cc3dcSJack F Vogel rxbuf = &rxr->rx_buffers[i]; 1414758cc3dcSJack F Vogel if (rxbuf->buf != NULL) { 1415758cc3dcSJack F Vogel bus_dmamap_sync(rxr->ptag, rxbuf->pmap, 1416758cc3dcSJack F Vogel BUS_DMASYNC_POSTREAD); 1417758cc3dcSJack F Vogel bus_dmamap_unload(rxr->ptag, rxbuf->pmap); 1418758cc3dcSJack F Vogel rxbuf->buf->m_flags |= M_PKTHDR; 1419758cc3dcSJack F Vogel m_freem(rxbuf->buf); 1420758cc3dcSJack F Vogel rxbuf->buf = NULL; 1421758cc3dcSJack F Vogel rxbuf->flags = 0; 1422758cc3dcSJack F Vogel } 1423758cc3dcSJack F Vogel } 1424758cc3dcSJack F Vogel } 1425758cc3dcSJack F Vogel 1426758cc3dcSJack F Vogel 1427758cc3dcSJack F Vogel /********************************************************************* 1428758cc3dcSJack F Vogel * 1429758cc3dcSJack F Vogel * Initialize a receive ring and its buffers. 1430758cc3dcSJack F Vogel * 1431758cc3dcSJack F Vogel **********************************************************************/ 1432758cc3dcSJack F Vogel static int 1433758cc3dcSJack F Vogel ixgbe_setup_receive_ring(struct rx_ring *rxr) 1434758cc3dcSJack F Vogel { 1435758cc3dcSJack F Vogel struct adapter *adapter; 1436758cc3dcSJack F Vogel struct ifnet *ifp; 1437758cc3dcSJack F Vogel device_t dev; 1438758cc3dcSJack F Vogel struct ixgbe_rx_buf *rxbuf; 1439758cc3dcSJack F Vogel bus_dma_segment_t seg[1]; 1440758cc3dcSJack F Vogel struct lro_ctrl *lro = &rxr->lro; 1441758cc3dcSJack F Vogel int rsize, nsegs, error = 0; 1442758cc3dcSJack F Vogel #ifdef DEV_NETMAP 1443758cc3dcSJack F Vogel struct netmap_adapter *na = NA(rxr->adapter->ifp); 1444758cc3dcSJack F Vogel struct netmap_slot *slot; 1445758cc3dcSJack F Vogel #endif /* DEV_NETMAP */ 1446758cc3dcSJack F Vogel 1447758cc3dcSJack F Vogel adapter = rxr->adapter; 1448758cc3dcSJack F Vogel ifp = adapter->ifp; 1449758cc3dcSJack F Vogel dev = adapter->dev; 1450758cc3dcSJack F Vogel 1451758cc3dcSJack F Vogel /* Clear the ring contents */ 1452758cc3dcSJack F Vogel IXGBE_RX_LOCK(rxr); 1453758cc3dcSJack F Vogel #ifdef DEV_NETMAP 1454758cc3dcSJack F Vogel /* same as in ixgbe_setup_transmit_ring() */ 1455758cc3dcSJack F Vogel slot = netmap_reset(na, NR_RX, rxr->me, 0); 1456758cc3dcSJack F Vogel #endif /* DEV_NETMAP */ 1457758cc3dcSJack F Vogel rsize = roundup2(adapter->num_rx_desc * 1458758cc3dcSJack F Vogel sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); 1459758cc3dcSJack F Vogel bzero((void *)rxr->rx_base, rsize); 1460758cc3dcSJack F Vogel /* Cache the size */ 1461758cc3dcSJack F Vogel rxr->mbuf_sz = adapter->rx_mbuf_sz; 1462758cc3dcSJack F Vogel 1463758cc3dcSJack F Vogel /* Free current RX buffer structs and their mbufs */ 1464758cc3dcSJack F Vogel ixgbe_free_receive_ring(rxr); 1465758cc3dcSJack F Vogel 1466758cc3dcSJack F Vogel /* Now replenish the mbufs */ 1467758cc3dcSJack F Vogel for (int j = 0; j != rxr->num_desc; ++j) { 1468758cc3dcSJack F Vogel struct mbuf *mp; 1469758cc3dcSJack F Vogel 1470758cc3dcSJack F Vogel rxbuf = &rxr->rx_buffers[j]; 1471758cc3dcSJack F Vogel #ifdef DEV_NETMAP 1472758cc3dcSJack F Vogel /* 1473758cc3dcSJack F Vogel * In netmap mode, fill the map and set the buffer 1474758cc3dcSJack F Vogel * address in the NIC ring, considering the offset 1475758cc3dcSJack F Vogel * between the netmap and NIC rings (see comment in 1476758cc3dcSJack F Vogel * ixgbe_setup_transmit_ring() ). No need to allocate 1477758cc3dcSJack F Vogel * an mbuf, so end the block with a continue; 1478758cc3dcSJack F Vogel */ 1479758cc3dcSJack F Vogel if (slot) { 1480758cc3dcSJack F Vogel int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j); 1481758cc3dcSJack F Vogel uint64_t paddr; 1482758cc3dcSJack F Vogel void *addr; 1483758cc3dcSJack F Vogel 1484758cc3dcSJack F Vogel addr = PNMB(na, slot + sj, &paddr); 1485758cc3dcSJack F Vogel netmap_load_map(na, rxr->ptag, rxbuf->pmap, addr); 1486758cc3dcSJack F Vogel /* Update descriptor and the cached value */ 1487758cc3dcSJack F Vogel rxr->rx_base[j].read.pkt_addr = htole64(paddr); 1488758cc3dcSJack F Vogel rxbuf->addr = htole64(paddr); 1489758cc3dcSJack F Vogel continue; 1490758cc3dcSJack F Vogel } 1491758cc3dcSJack F Vogel #endif /* DEV_NETMAP */ 1492758cc3dcSJack F Vogel rxbuf->flags = 0; 1493758cc3dcSJack F Vogel rxbuf->buf = m_getjcl(M_NOWAIT, MT_DATA, 1494758cc3dcSJack F Vogel M_PKTHDR, adapter->rx_mbuf_sz); 1495758cc3dcSJack F Vogel if (rxbuf->buf == NULL) { 1496758cc3dcSJack F Vogel error = ENOBUFS; 1497758cc3dcSJack F Vogel goto fail; 1498758cc3dcSJack F Vogel } 1499758cc3dcSJack F Vogel mp = rxbuf->buf; 1500758cc3dcSJack F Vogel mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz; 1501758cc3dcSJack F Vogel /* Get the memory mapping */ 1502758cc3dcSJack F Vogel error = bus_dmamap_load_mbuf_sg(rxr->ptag, 1503758cc3dcSJack F Vogel rxbuf->pmap, mp, seg, 1504758cc3dcSJack F Vogel &nsegs, BUS_DMA_NOWAIT); 1505758cc3dcSJack F Vogel if (error != 0) 1506758cc3dcSJack F Vogel goto fail; 1507758cc3dcSJack F Vogel bus_dmamap_sync(rxr->ptag, 1508758cc3dcSJack F Vogel rxbuf->pmap, BUS_DMASYNC_PREREAD); 1509758cc3dcSJack F Vogel /* Update the descriptor and the cached value */ 1510758cc3dcSJack F Vogel rxr->rx_base[j].read.pkt_addr = htole64(seg[0].ds_addr); 1511758cc3dcSJack F Vogel rxbuf->addr = htole64(seg[0].ds_addr); 1512758cc3dcSJack F Vogel } 1513758cc3dcSJack F Vogel 1514758cc3dcSJack F Vogel 1515758cc3dcSJack F Vogel /* Setup our descriptor indices */ 1516758cc3dcSJack F Vogel rxr->next_to_check = 0; 1517758cc3dcSJack F Vogel rxr->next_to_refresh = 0; 1518758cc3dcSJack F Vogel rxr->lro_enabled = FALSE; 1519758cc3dcSJack F Vogel rxr->rx_copies = 0; 1520758cc3dcSJack F Vogel rxr->rx_bytes = 0; 1521758cc3dcSJack F Vogel rxr->vtag_strip = FALSE; 1522758cc3dcSJack F Vogel 1523758cc3dcSJack F Vogel bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 1524758cc3dcSJack F Vogel BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1525758cc3dcSJack F Vogel 1526758cc3dcSJack F Vogel /* 1527758cc3dcSJack F Vogel ** Now set up the LRO interface: 1528758cc3dcSJack F Vogel */ 1529758cc3dcSJack F Vogel if (ixgbe_rsc_enable) 1530758cc3dcSJack F Vogel ixgbe_setup_hw_rsc(rxr); 1531758cc3dcSJack F Vogel else if (ifp->if_capenable & IFCAP_LRO) { 1532758cc3dcSJack F Vogel int err = tcp_lro_init(lro); 1533758cc3dcSJack F Vogel if (err) { 1534758cc3dcSJack F Vogel device_printf(dev, "LRO Initialization failed!\n"); 1535758cc3dcSJack F Vogel goto fail; 1536758cc3dcSJack F Vogel } 1537758cc3dcSJack F Vogel INIT_DEBUGOUT("RX Soft LRO Initialized\n"); 1538758cc3dcSJack F Vogel rxr->lro_enabled = TRUE; 1539758cc3dcSJack F Vogel lro->ifp = adapter->ifp; 1540758cc3dcSJack F Vogel } 1541758cc3dcSJack F Vogel 1542758cc3dcSJack F Vogel IXGBE_RX_UNLOCK(rxr); 1543758cc3dcSJack F Vogel return (0); 1544758cc3dcSJack F Vogel 1545758cc3dcSJack F Vogel fail: 1546758cc3dcSJack F Vogel ixgbe_free_receive_ring(rxr); 1547758cc3dcSJack F Vogel IXGBE_RX_UNLOCK(rxr); 1548758cc3dcSJack F Vogel return (error); 1549758cc3dcSJack F Vogel } 1550758cc3dcSJack F Vogel 1551758cc3dcSJack F Vogel /********************************************************************* 1552758cc3dcSJack F Vogel * 1553758cc3dcSJack F Vogel * Initialize all receive rings. 1554758cc3dcSJack F Vogel * 1555758cc3dcSJack F Vogel **********************************************************************/ 1556758cc3dcSJack F Vogel int 1557758cc3dcSJack F Vogel ixgbe_setup_receive_structures(struct adapter *adapter) 1558758cc3dcSJack F Vogel { 1559758cc3dcSJack F Vogel struct rx_ring *rxr = adapter->rx_rings; 1560758cc3dcSJack F Vogel int j; 1561758cc3dcSJack F Vogel 1562758cc3dcSJack F Vogel for (j = 0; j < adapter->num_queues; j++, rxr++) 1563758cc3dcSJack F Vogel if (ixgbe_setup_receive_ring(rxr)) 1564758cc3dcSJack F Vogel goto fail; 1565758cc3dcSJack F Vogel 1566758cc3dcSJack F Vogel return (0); 1567758cc3dcSJack F Vogel fail: 1568758cc3dcSJack F Vogel /* 1569758cc3dcSJack F Vogel * Free RX buffers allocated so far, we will only handle 1570758cc3dcSJack F Vogel * the rings that completed, the failing case will have 1571758cc3dcSJack F Vogel * cleaned up for itself. 'j' failed, so its the terminus. 1572758cc3dcSJack F Vogel */ 1573758cc3dcSJack F Vogel for (int i = 0; i < j; ++i) { 1574758cc3dcSJack F Vogel rxr = &adapter->rx_rings[i]; 1575758cc3dcSJack F Vogel ixgbe_free_receive_ring(rxr); 1576758cc3dcSJack F Vogel } 1577758cc3dcSJack F Vogel 1578758cc3dcSJack F Vogel return (ENOBUFS); 1579758cc3dcSJack F Vogel } 1580758cc3dcSJack F Vogel 1581758cc3dcSJack F Vogel 1582758cc3dcSJack F Vogel /********************************************************************* 1583758cc3dcSJack F Vogel * 1584758cc3dcSJack F Vogel * Free all receive rings. 1585758cc3dcSJack F Vogel * 1586758cc3dcSJack F Vogel **********************************************************************/ 1587758cc3dcSJack F Vogel void 1588758cc3dcSJack F Vogel ixgbe_free_receive_structures(struct adapter *adapter) 1589758cc3dcSJack F Vogel { 1590758cc3dcSJack F Vogel struct rx_ring *rxr = adapter->rx_rings; 1591758cc3dcSJack F Vogel 1592758cc3dcSJack F Vogel INIT_DEBUGOUT("ixgbe_free_receive_structures: begin"); 1593758cc3dcSJack F Vogel 1594758cc3dcSJack F Vogel for (int i = 0; i < adapter->num_queues; i++, rxr++) { 1595758cc3dcSJack F Vogel struct lro_ctrl *lro = &rxr->lro; 1596758cc3dcSJack F Vogel ixgbe_free_receive_buffers(rxr); 1597758cc3dcSJack F Vogel /* Free LRO memory */ 1598758cc3dcSJack F Vogel tcp_lro_free(lro); 1599758cc3dcSJack F Vogel /* Free the ring memory as well */ 1600758cc3dcSJack F Vogel ixgbe_dma_free(adapter, &rxr->rxdma); 1601758cc3dcSJack F Vogel } 1602758cc3dcSJack F Vogel 1603758cc3dcSJack F Vogel free(adapter->rx_rings, M_DEVBUF); 1604758cc3dcSJack F Vogel } 1605758cc3dcSJack F Vogel 1606758cc3dcSJack F Vogel 1607758cc3dcSJack F Vogel /********************************************************************* 1608758cc3dcSJack F Vogel * 1609758cc3dcSJack F Vogel * Free receive ring data structures 1610758cc3dcSJack F Vogel * 1611758cc3dcSJack F Vogel **********************************************************************/ 1612758cc3dcSJack F Vogel void 1613758cc3dcSJack F Vogel ixgbe_free_receive_buffers(struct rx_ring *rxr) 1614758cc3dcSJack F Vogel { 1615758cc3dcSJack F Vogel struct adapter *adapter = rxr->adapter; 1616758cc3dcSJack F Vogel struct ixgbe_rx_buf *rxbuf; 1617758cc3dcSJack F Vogel 1618758cc3dcSJack F Vogel INIT_DEBUGOUT("ixgbe_free_receive_buffers: begin"); 1619758cc3dcSJack F Vogel 1620758cc3dcSJack F Vogel /* Cleanup any existing buffers */ 1621758cc3dcSJack F Vogel if (rxr->rx_buffers != NULL) { 1622758cc3dcSJack F Vogel for (int i = 0; i < adapter->num_rx_desc; i++) { 1623758cc3dcSJack F Vogel rxbuf = &rxr->rx_buffers[i]; 1624758cc3dcSJack F Vogel if (rxbuf->buf != NULL) { 1625758cc3dcSJack F Vogel bus_dmamap_sync(rxr->ptag, rxbuf->pmap, 1626758cc3dcSJack F Vogel BUS_DMASYNC_POSTREAD); 1627758cc3dcSJack F Vogel bus_dmamap_unload(rxr->ptag, rxbuf->pmap); 1628758cc3dcSJack F Vogel rxbuf->buf->m_flags |= M_PKTHDR; 1629758cc3dcSJack F Vogel m_freem(rxbuf->buf); 1630758cc3dcSJack F Vogel } 1631758cc3dcSJack F Vogel rxbuf->buf = NULL; 1632758cc3dcSJack F Vogel if (rxbuf->pmap != NULL) { 1633758cc3dcSJack F Vogel bus_dmamap_destroy(rxr->ptag, rxbuf->pmap); 1634758cc3dcSJack F Vogel rxbuf->pmap = NULL; 1635758cc3dcSJack F Vogel } 1636758cc3dcSJack F Vogel } 1637758cc3dcSJack F Vogel if (rxr->rx_buffers != NULL) { 1638758cc3dcSJack F Vogel free(rxr->rx_buffers, M_DEVBUF); 1639758cc3dcSJack F Vogel rxr->rx_buffers = NULL; 1640758cc3dcSJack F Vogel } 1641758cc3dcSJack F Vogel } 1642758cc3dcSJack F Vogel 1643758cc3dcSJack F Vogel if (rxr->ptag != NULL) { 1644758cc3dcSJack F Vogel bus_dma_tag_destroy(rxr->ptag); 1645758cc3dcSJack F Vogel rxr->ptag = NULL; 1646758cc3dcSJack F Vogel } 1647758cc3dcSJack F Vogel 1648758cc3dcSJack F Vogel return; 1649758cc3dcSJack F Vogel } 1650758cc3dcSJack F Vogel 1651758cc3dcSJack F Vogel static __inline void 1652758cc3dcSJack F Vogel ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype) 1653758cc3dcSJack F Vogel { 1654758cc3dcSJack F Vogel 1655758cc3dcSJack F Vogel /* 1656758cc3dcSJack F Vogel * ATM LRO is only for IP/TCP packets and TCP checksum of the packet 1657758cc3dcSJack F Vogel * should be computed by hardware. Also it should not have VLAN tag in 1658758cc3dcSJack F Vogel * ethernet header. In case of IPv6 we do not yet support ext. hdrs. 1659758cc3dcSJack F Vogel */ 1660758cc3dcSJack F Vogel if (rxr->lro_enabled && 1661758cc3dcSJack F Vogel (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 && 1662758cc3dcSJack F Vogel (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && 1663758cc3dcSJack F Vogel ((ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) == 1664758cc3dcSJack F Vogel (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) || 1665758cc3dcSJack F Vogel (ptype & (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) == 1666758cc3dcSJack F Vogel (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) && 1667758cc3dcSJack F Vogel (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == 1668758cc3dcSJack F Vogel (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) { 1669758cc3dcSJack F Vogel /* 1670758cc3dcSJack F Vogel * Send to the stack if: 1671758cc3dcSJack F Vogel ** - LRO not enabled, or 1672758cc3dcSJack F Vogel ** - no LRO resources, or 1673758cc3dcSJack F Vogel ** - lro enqueue fails 1674758cc3dcSJack F Vogel */ 1675758cc3dcSJack F Vogel if (rxr->lro.lro_cnt != 0) 1676758cc3dcSJack F Vogel if (tcp_lro_rx(&rxr->lro, m, 0) == 0) 1677758cc3dcSJack F Vogel return; 1678758cc3dcSJack F Vogel } 1679758cc3dcSJack F Vogel IXGBE_RX_UNLOCK(rxr); 1680758cc3dcSJack F Vogel (*ifp->if_input)(ifp, m); 1681758cc3dcSJack F Vogel IXGBE_RX_LOCK(rxr); 1682758cc3dcSJack F Vogel } 1683758cc3dcSJack F Vogel 1684758cc3dcSJack F Vogel static __inline void 1685758cc3dcSJack F Vogel ixgbe_rx_discard(struct rx_ring *rxr, int i) 1686758cc3dcSJack F Vogel { 1687758cc3dcSJack F Vogel struct ixgbe_rx_buf *rbuf; 1688758cc3dcSJack F Vogel 1689758cc3dcSJack F Vogel rbuf = &rxr->rx_buffers[i]; 1690758cc3dcSJack F Vogel 1691758cc3dcSJack F Vogel 1692758cc3dcSJack F Vogel /* 1693758cc3dcSJack F Vogel ** With advanced descriptors the writeback 1694758cc3dcSJack F Vogel ** clobbers the buffer addrs, so its easier 1695758cc3dcSJack F Vogel ** to just free the existing mbufs and take 1696758cc3dcSJack F Vogel ** the normal refresh path to get new buffers 1697758cc3dcSJack F Vogel ** and mapping. 1698758cc3dcSJack F Vogel */ 1699758cc3dcSJack F Vogel 1700758cc3dcSJack F Vogel if (rbuf->fmp != NULL) {/* Partial chain ? */ 1701758cc3dcSJack F Vogel rbuf->fmp->m_flags |= M_PKTHDR; 1702758cc3dcSJack F Vogel m_freem(rbuf->fmp); 1703758cc3dcSJack F Vogel rbuf->fmp = NULL; 1704758cc3dcSJack F Vogel rbuf->buf = NULL; /* rbuf->buf is part of fmp's chain */ 1705758cc3dcSJack F Vogel } else if (rbuf->buf) { 1706758cc3dcSJack F Vogel m_free(rbuf->buf); 1707758cc3dcSJack F Vogel rbuf->buf = NULL; 1708758cc3dcSJack F Vogel } 1709758cc3dcSJack F Vogel 1710758cc3dcSJack F Vogel rbuf->flags = 0; 1711758cc3dcSJack F Vogel 1712758cc3dcSJack F Vogel return; 1713758cc3dcSJack F Vogel } 1714758cc3dcSJack F Vogel 1715758cc3dcSJack F Vogel 1716758cc3dcSJack F Vogel /********************************************************************* 1717758cc3dcSJack F Vogel * 1718758cc3dcSJack F Vogel * This routine executes in interrupt context. It replenishes 1719758cc3dcSJack F Vogel * the mbufs in the descriptor and sends data which has been 1720758cc3dcSJack F Vogel * dma'ed into host memory to upper layer. 1721758cc3dcSJack F Vogel * 1722758cc3dcSJack F Vogel * We loop at most count times if count is > 0, or until done if 1723758cc3dcSJack F Vogel * count < 0. 1724758cc3dcSJack F Vogel * 1725758cc3dcSJack F Vogel * Return TRUE for more work, FALSE for all clean. 1726758cc3dcSJack F Vogel *********************************************************************/ 1727758cc3dcSJack F Vogel bool 1728758cc3dcSJack F Vogel ixgbe_rxeof(struct ix_queue *que) 1729758cc3dcSJack F Vogel { 1730758cc3dcSJack F Vogel struct adapter *adapter = que->adapter; 1731758cc3dcSJack F Vogel struct rx_ring *rxr = que->rxr; 1732758cc3dcSJack F Vogel struct ifnet *ifp = adapter->ifp; 1733758cc3dcSJack F Vogel struct lro_ctrl *lro = &rxr->lro; 1734758cc3dcSJack F Vogel struct lro_entry *queued; 1735758cc3dcSJack F Vogel int i, nextp, processed = 0; 1736758cc3dcSJack F Vogel u32 staterr = 0; 1737758cc3dcSJack F Vogel u16 count = rxr->process_limit; 1738758cc3dcSJack F Vogel union ixgbe_adv_rx_desc *cur; 1739758cc3dcSJack F Vogel struct ixgbe_rx_buf *rbuf, *nbuf; 1740758cc3dcSJack F Vogel u16 pkt_info; 1741758cc3dcSJack F Vogel 1742758cc3dcSJack F Vogel IXGBE_RX_LOCK(rxr); 1743758cc3dcSJack F Vogel 1744758cc3dcSJack F Vogel #ifdef DEV_NETMAP 1745758cc3dcSJack F Vogel /* Same as the txeof routine: wakeup clients on intr. */ 1746758cc3dcSJack F Vogel if (netmap_rx_irq(ifp, rxr->me, &processed)) { 1747758cc3dcSJack F Vogel IXGBE_RX_UNLOCK(rxr); 1748758cc3dcSJack F Vogel return (FALSE); 1749758cc3dcSJack F Vogel } 1750758cc3dcSJack F Vogel #endif /* DEV_NETMAP */ 1751758cc3dcSJack F Vogel 1752758cc3dcSJack F Vogel for (i = rxr->next_to_check; count != 0;) { 1753758cc3dcSJack F Vogel struct mbuf *sendmp, *mp; 1754758cc3dcSJack F Vogel u32 rsc, ptype; 1755758cc3dcSJack F Vogel u16 len; 1756758cc3dcSJack F Vogel u16 vtag = 0; 1757758cc3dcSJack F Vogel bool eop; 1758758cc3dcSJack F Vogel 1759758cc3dcSJack F Vogel /* Sync the ring. */ 1760758cc3dcSJack F Vogel bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 1761758cc3dcSJack F Vogel BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1762758cc3dcSJack F Vogel 1763758cc3dcSJack F Vogel cur = &rxr->rx_base[i]; 1764758cc3dcSJack F Vogel staterr = le32toh(cur->wb.upper.status_error); 1765758cc3dcSJack F Vogel pkt_info = le16toh(cur->wb.lower.lo_dword.hs_rss.pkt_info); 1766758cc3dcSJack F Vogel 1767758cc3dcSJack F Vogel if ((staterr & IXGBE_RXD_STAT_DD) == 0) 1768758cc3dcSJack F Vogel break; 1769758cc3dcSJack F Vogel if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 1770758cc3dcSJack F Vogel break; 1771758cc3dcSJack F Vogel 1772758cc3dcSJack F Vogel count--; 1773758cc3dcSJack F Vogel sendmp = NULL; 1774758cc3dcSJack F Vogel nbuf = NULL; 1775758cc3dcSJack F Vogel rsc = 0; 1776758cc3dcSJack F Vogel cur->wb.upper.status_error = 0; 1777758cc3dcSJack F Vogel rbuf = &rxr->rx_buffers[i]; 1778758cc3dcSJack F Vogel mp = rbuf->buf; 1779758cc3dcSJack F Vogel 1780758cc3dcSJack F Vogel len = le16toh(cur->wb.upper.length); 1781758cc3dcSJack F Vogel ptype = le32toh(cur->wb.lower.lo_dword.data) & 1782758cc3dcSJack F Vogel IXGBE_RXDADV_PKTTYPE_MASK; 1783758cc3dcSJack F Vogel eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0); 1784758cc3dcSJack F Vogel 1785758cc3dcSJack F Vogel /* Make sure bad packets are discarded */ 1786758cc3dcSJack F Vogel if (eop && (staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) { 1787758cc3dcSJack F Vogel #if 0 // VF-only 1788758cc3dcSJack F Vogel #if __FreeBSD_version >= 1100036 1789758cc3dcSJack F Vogel if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); 1790758cc3dcSJack F Vogel #endif 1791758cc3dcSJack F Vogel #endif 1792758cc3dcSJack F Vogel rxr->rx_discarded++; 1793758cc3dcSJack F Vogel ixgbe_rx_discard(rxr, i); 1794758cc3dcSJack F Vogel goto next_desc; 1795758cc3dcSJack F Vogel } 1796758cc3dcSJack F Vogel 1797758cc3dcSJack F Vogel /* 1798758cc3dcSJack F Vogel ** On 82599 which supports a hardware 1799758cc3dcSJack F Vogel ** LRO (called HW RSC), packets need 1800758cc3dcSJack F Vogel ** not be fragmented across sequential 1801758cc3dcSJack F Vogel ** descriptors, rather the next descriptor 1802758cc3dcSJack F Vogel ** is indicated in bits of the descriptor. 1803758cc3dcSJack F Vogel ** This also means that we might proceses 1804758cc3dcSJack F Vogel ** more than one packet at a time, something 1805758cc3dcSJack F Vogel ** that has never been true before, it 1806758cc3dcSJack F Vogel ** required eliminating global chain pointers 1807758cc3dcSJack F Vogel ** in favor of what we are doing here. -jfv 1808758cc3dcSJack F Vogel */ 1809758cc3dcSJack F Vogel if (!eop) { 1810758cc3dcSJack F Vogel /* 1811758cc3dcSJack F Vogel ** Figure out the next descriptor 1812758cc3dcSJack F Vogel ** of this frame. 1813758cc3dcSJack F Vogel */ 1814758cc3dcSJack F Vogel if (rxr->hw_rsc == TRUE) { 1815758cc3dcSJack F Vogel rsc = ixgbe_rsc_count(cur); 1816758cc3dcSJack F Vogel rxr->rsc_num += (rsc - 1); 1817758cc3dcSJack F Vogel } 1818758cc3dcSJack F Vogel if (rsc) { /* Get hardware index */ 1819758cc3dcSJack F Vogel nextp = ((staterr & 1820758cc3dcSJack F Vogel IXGBE_RXDADV_NEXTP_MASK) >> 1821758cc3dcSJack F Vogel IXGBE_RXDADV_NEXTP_SHIFT); 1822758cc3dcSJack F Vogel } else { /* Just sequential */ 1823758cc3dcSJack F Vogel nextp = i + 1; 1824758cc3dcSJack F Vogel if (nextp == adapter->num_rx_desc) 1825758cc3dcSJack F Vogel nextp = 0; 1826758cc3dcSJack F Vogel } 1827758cc3dcSJack F Vogel nbuf = &rxr->rx_buffers[nextp]; 1828758cc3dcSJack F Vogel prefetch(nbuf); 1829758cc3dcSJack F Vogel } 1830758cc3dcSJack F Vogel /* 1831758cc3dcSJack F Vogel ** Rather than using the fmp/lmp global pointers 1832758cc3dcSJack F Vogel ** we now keep the head of a packet chain in the 1833758cc3dcSJack F Vogel ** buffer struct and pass this along from one 1834758cc3dcSJack F Vogel ** descriptor to the next, until we get EOP. 1835758cc3dcSJack F Vogel */ 1836758cc3dcSJack F Vogel mp->m_len = len; 1837758cc3dcSJack F Vogel /* 1838758cc3dcSJack F Vogel ** See if there is a stored head 1839758cc3dcSJack F Vogel ** that determines what we are 1840758cc3dcSJack F Vogel */ 1841758cc3dcSJack F Vogel sendmp = rbuf->fmp; 1842758cc3dcSJack F Vogel if (sendmp != NULL) { /* secondary frag */ 1843758cc3dcSJack F Vogel rbuf->buf = rbuf->fmp = NULL; 1844758cc3dcSJack F Vogel mp->m_flags &= ~M_PKTHDR; 1845758cc3dcSJack F Vogel sendmp->m_pkthdr.len += mp->m_len; 1846758cc3dcSJack F Vogel } else { 1847758cc3dcSJack F Vogel /* 1848758cc3dcSJack F Vogel * Optimize. This might be a small packet, 1849758cc3dcSJack F Vogel * maybe just a TCP ACK. Do a fast copy that 1850758cc3dcSJack F Vogel * is cache aligned into a new mbuf, and 1851758cc3dcSJack F Vogel * leave the old mbuf+cluster for re-use. 1852758cc3dcSJack F Vogel */ 1853758cc3dcSJack F Vogel if (eop && len <= IXGBE_RX_COPY_LEN) { 1854758cc3dcSJack F Vogel sendmp = m_gethdr(M_NOWAIT, MT_DATA); 1855758cc3dcSJack F Vogel if (sendmp != NULL) { 1856758cc3dcSJack F Vogel sendmp->m_data += 1857758cc3dcSJack F Vogel IXGBE_RX_COPY_ALIGN; 1858758cc3dcSJack F Vogel ixgbe_bcopy(mp->m_data, 1859758cc3dcSJack F Vogel sendmp->m_data, len); 1860758cc3dcSJack F Vogel sendmp->m_len = len; 1861758cc3dcSJack F Vogel rxr->rx_copies++; 1862758cc3dcSJack F Vogel rbuf->flags |= IXGBE_RX_COPY; 1863758cc3dcSJack F Vogel } 1864758cc3dcSJack F Vogel } 1865758cc3dcSJack F Vogel if (sendmp == NULL) { 1866758cc3dcSJack F Vogel rbuf->buf = rbuf->fmp = NULL; 1867758cc3dcSJack F Vogel sendmp = mp; 1868758cc3dcSJack F Vogel } 1869758cc3dcSJack F Vogel 1870758cc3dcSJack F Vogel /* first desc of a non-ps chain */ 1871758cc3dcSJack F Vogel sendmp->m_flags |= M_PKTHDR; 1872758cc3dcSJack F Vogel sendmp->m_pkthdr.len = mp->m_len; 1873758cc3dcSJack F Vogel } 1874758cc3dcSJack F Vogel ++processed; 1875758cc3dcSJack F Vogel 1876758cc3dcSJack F Vogel /* Pass the head pointer on */ 1877758cc3dcSJack F Vogel if (eop == 0) { 1878758cc3dcSJack F Vogel nbuf->fmp = sendmp; 1879758cc3dcSJack F Vogel sendmp = NULL; 1880758cc3dcSJack F Vogel mp->m_next = nbuf->buf; 1881758cc3dcSJack F Vogel } else { /* Sending this frame */ 1882758cc3dcSJack F Vogel sendmp->m_pkthdr.rcvif = ifp; 1883758cc3dcSJack F Vogel rxr->rx_packets++; 1884758cc3dcSJack F Vogel /* capture data for AIM */ 1885758cc3dcSJack F Vogel rxr->bytes += sendmp->m_pkthdr.len; 1886758cc3dcSJack F Vogel rxr->rx_bytes += sendmp->m_pkthdr.len; 1887758cc3dcSJack F Vogel /* Process vlan info */ 1888758cc3dcSJack F Vogel if ((rxr->vtag_strip) && 1889758cc3dcSJack F Vogel (staterr & IXGBE_RXD_STAT_VP)) 1890758cc3dcSJack F Vogel vtag = le16toh(cur->wb.upper.vlan); 1891758cc3dcSJack F Vogel if (vtag) { 1892758cc3dcSJack F Vogel sendmp->m_pkthdr.ether_vtag = vtag; 1893758cc3dcSJack F Vogel sendmp->m_flags |= M_VLANTAG; 1894758cc3dcSJack F Vogel } 1895758cc3dcSJack F Vogel if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) 1896758cc3dcSJack F Vogel ixgbe_rx_checksum(staterr, sendmp, ptype); 1897758cc3dcSJack F Vogel #if __FreeBSD_version >= 800000 1898758cc3dcSJack F Vogel #ifdef RSS 1899758cc3dcSJack F Vogel sendmp->m_pkthdr.flowid = 1900758cc3dcSJack F Vogel le32toh(cur->wb.lower.hi_dword.rss); 1901758cc3dcSJack F Vogel switch (pkt_info & IXGBE_RXDADV_RSSTYPE_MASK) { 1902758cc3dcSJack F Vogel case IXGBE_RXDADV_RSSTYPE_IPV4_TCP: 1903758cc3dcSJack F Vogel M_HASHTYPE_SET(sendmp, M_HASHTYPE_RSS_TCP_IPV4); 1904758cc3dcSJack F Vogel break; 1905758cc3dcSJack F Vogel case IXGBE_RXDADV_RSSTYPE_IPV4: 1906758cc3dcSJack F Vogel M_HASHTYPE_SET(sendmp, M_HASHTYPE_RSS_IPV4); 1907758cc3dcSJack F Vogel break; 1908758cc3dcSJack F Vogel case IXGBE_RXDADV_RSSTYPE_IPV6_TCP: 1909758cc3dcSJack F Vogel M_HASHTYPE_SET(sendmp, M_HASHTYPE_RSS_TCP_IPV6); 1910758cc3dcSJack F Vogel break; 1911758cc3dcSJack F Vogel case IXGBE_RXDADV_RSSTYPE_IPV6_EX: 1912758cc3dcSJack F Vogel M_HASHTYPE_SET(sendmp, M_HASHTYPE_RSS_IPV6_EX); 1913758cc3dcSJack F Vogel break; 1914758cc3dcSJack F Vogel case IXGBE_RXDADV_RSSTYPE_IPV6: 1915758cc3dcSJack F Vogel M_HASHTYPE_SET(sendmp, M_HASHTYPE_RSS_IPV6); 1916758cc3dcSJack F Vogel break; 1917758cc3dcSJack F Vogel case IXGBE_RXDADV_RSSTYPE_IPV6_TCP_EX: 1918758cc3dcSJack F Vogel M_HASHTYPE_SET(sendmp, M_HASHTYPE_RSS_TCP_IPV6_EX); 1919758cc3dcSJack F Vogel break; 1920758cc3dcSJack F Vogel case IXGBE_RXDADV_RSSTYPE_IPV4_UDP: 1921758cc3dcSJack F Vogel M_HASHTYPE_SET(sendmp, M_HASHTYPE_RSS_UDP_IPV4); 1922758cc3dcSJack F Vogel break; 1923758cc3dcSJack F Vogel case IXGBE_RXDADV_RSSTYPE_IPV6_UDP: 1924758cc3dcSJack F Vogel M_HASHTYPE_SET(sendmp, M_HASHTYPE_RSS_UDP_IPV6); 1925758cc3dcSJack F Vogel break; 1926758cc3dcSJack F Vogel case IXGBE_RXDADV_RSSTYPE_IPV6_UDP_EX: 1927758cc3dcSJack F Vogel M_HASHTYPE_SET(sendmp, M_HASHTYPE_RSS_UDP_IPV6_EX); 1928758cc3dcSJack F Vogel break; 1929758cc3dcSJack F Vogel default: 1930758cc3dcSJack F Vogel M_HASHTYPE_SET(sendmp, M_HASHTYPE_OPAQUE); 1931758cc3dcSJack F Vogel } 1932758cc3dcSJack F Vogel #else /* RSS */ 1933758cc3dcSJack F Vogel sendmp->m_pkthdr.flowid = que->msix; 1934758cc3dcSJack F Vogel M_HASHTYPE_SET(sendmp, M_HASHTYPE_OPAQUE); 1935758cc3dcSJack F Vogel #endif /* RSS */ 1936758cc3dcSJack F Vogel #endif /* FreeBSD_version */ 1937758cc3dcSJack F Vogel } 1938758cc3dcSJack F Vogel next_desc: 1939758cc3dcSJack F Vogel bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 1940758cc3dcSJack F Vogel BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1941758cc3dcSJack F Vogel 1942758cc3dcSJack F Vogel /* Advance our pointers to the next descriptor. */ 1943758cc3dcSJack F Vogel if (++i == rxr->num_desc) 1944758cc3dcSJack F Vogel i = 0; 1945758cc3dcSJack F Vogel 1946758cc3dcSJack F Vogel /* Now send to the stack or do LRO */ 1947758cc3dcSJack F Vogel if (sendmp != NULL) { 1948758cc3dcSJack F Vogel rxr->next_to_check = i; 1949758cc3dcSJack F Vogel ixgbe_rx_input(rxr, ifp, sendmp, ptype); 1950758cc3dcSJack F Vogel i = rxr->next_to_check; 1951758cc3dcSJack F Vogel } 1952758cc3dcSJack F Vogel 1953758cc3dcSJack F Vogel /* Every 8 descriptors we go to refresh mbufs */ 1954758cc3dcSJack F Vogel if (processed == 8) { 1955758cc3dcSJack F Vogel ixgbe_refresh_mbufs(rxr, i); 1956758cc3dcSJack F Vogel processed = 0; 1957758cc3dcSJack F Vogel } 1958758cc3dcSJack F Vogel } 1959758cc3dcSJack F Vogel 1960758cc3dcSJack F Vogel /* Refresh any remaining buf structs */ 1961758cc3dcSJack F Vogel if (ixgbe_rx_unrefreshed(rxr)) 1962758cc3dcSJack F Vogel ixgbe_refresh_mbufs(rxr, i); 1963758cc3dcSJack F Vogel 1964758cc3dcSJack F Vogel rxr->next_to_check = i; 1965758cc3dcSJack F Vogel 1966758cc3dcSJack F Vogel /* 1967758cc3dcSJack F Vogel * Flush any outstanding LRO work 1968758cc3dcSJack F Vogel */ 1969758cc3dcSJack F Vogel while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) { 1970758cc3dcSJack F Vogel SLIST_REMOVE_HEAD(&lro->lro_active, next); 1971758cc3dcSJack F Vogel tcp_lro_flush(lro, queued); 1972758cc3dcSJack F Vogel } 1973758cc3dcSJack F Vogel 1974758cc3dcSJack F Vogel IXGBE_RX_UNLOCK(rxr); 1975758cc3dcSJack F Vogel 1976758cc3dcSJack F Vogel /* 1977758cc3dcSJack F Vogel ** Still have cleaning to do? 1978758cc3dcSJack F Vogel */ 1979758cc3dcSJack F Vogel if ((staterr & IXGBE_RXD_STAT_DD) != 0) 1980758cc3dcSJack F Vogel return (TRUE); 1981758cc3dcSJack F Vogel else 1982758cc3dcSJack F Vogel return (FALSE); 1983758cc3dcSJack F Vogel } 1984758cc3dcSJack F Vogel 1985758cc3dcSJack F Vogel 1986758cc3dcSJack F Vogel /********************************************************************* 1987758cc3dcSJack F Vogel * 1988758cc3dcSJack F Vogel * Verify that the hardware indicated that the checksum is valid. 1989758cc3dcSJack F Vogel * Inform the stack about the status of checksum so that stack 1990758cc3dcSJack F Vogel * doesn't spend time verifying the checksum. 1991758cc3dcSJack F Vogel * 1992758cc3dcSJack F Vogel *********************************************************************/ 1993758cc3dcSJack F Vogel static void 1994758cc3dcSJack F Vogel ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype) 1995758cc3dcSJack F Vogel { 1996758cc3dcSJack F Vogel u16 status = (u16) staterr; 1997758cc3dcSJack F Vogel u8 errors = (u8) (staterr >> 24); 1998758cc3dcSJack F Vogel bool sctp = FALSE; 1999758cc3dcSJack F Vogel 2000758cc3dcSJack F Vogel if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && 2001758cc3dcSJack F Vogel (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0) 2002758cc3dcSJack F Vogel sctp = TRUE; 2003758cc3dcSJack F Vogel 2004758cc3dcSJack F Vogel if (status & IXGBE_RXD_STAT_IPCS) { 2005758cc3dcSJack F Vogel if (!(errors & IXGBE_RXD_ERR_IPE)) { 2006758cc3dcSJack F Vogel /* IP Checksum Good */ 2007758cc3dcSJack F Vogel mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED; 2008758cc3dcSJack F Vogel mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; 2009758cc3dcSJack F Vogel 2010758cc3dcSJack F Vogel } else 2011758cc3dcSJack F Vogel mp->m_pkthdr.csum_flags = 0; 2012758cc3dcSJack F Vogel } 2013758cc3dcSJack F Vogel if (status & IXGBE_RXD_STAT_L4CS) { 2014758cc3dcSJack F Vogel u64 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 2015758cc3dcSJack F Vogel #if __FreeBSD_version >= 800000 2016758cc3dcSJack F Vogel if (sctp) 2017758cc3dcSJack F Vogel type = CSUM_SCTP_VALID; 2018758cc3dcSJack F Vogel #endif 2019758cc3dcSJack F Vogel if (!(errors & IXGBE_RXD_ERR_TCPE)) { 2020758cc3dcSJack F Vogel mp->m_pkthdr.csum_flags |= type; 2021758cc3dcSJack F Vogel if (!sctp) 2022758cc3dcSJack F Vogel mp->m_pkthdr.csum_data = htons(0xffff); 2023758cc3dcSJack F Vogel } 2024758cc3dcSJack F Vogel } 2025758cc3dcSJack F Vogel return; 2026758cc3dcSJack F Vogel } 2027758cc3dcSJack F Vogel 2028758cc3dcSJack F Vogel /******************************************************************** 2029758cc3dcSJack F Vogel * Manage DMA'able memory. 2030758cc3dcSJack F Vogel *******************************************************************/ 2031758cc3dcSJack F Vogel static void 2032758cc3dcSJack F Vogel ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error) 2033758cc3dcSJack F Vogel { 2034758cc3dcSJack F Vogel if (error) 2035758cc3dcSJack F Vogel return; 2036758cc3dcSJack F Vogel *(bus_addr_t *) arg = segs->ds_addr; 2037758cc3dcSJack F Vogel return; 2038758cc3dcSJack F Vogel } 2039758cc3dcSJack F Vogel 2040758cc3dcSJack F Vogel int 2041758cc3dcSJack F Vogel ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size, 2042758cc3dcSJack F Vogel struct ixgbe_dma_alloc *dma, int mapflags) 2043758cc3dcSJack F Vogel { 2044758cc3dcSJack F Vogel device_t dev = adapter->dev; 2045758cc3dcSJack F Vogel int r; 2046758cc3dcSJack F Vogel 2047758cc3dcSJack F Vogel r = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */ 2048758cc3dcSJack F Vogel DBA_ALIGN, 0, /* alignment, bounds */ 2049758cc3dcSJack F Vogel BUS_SPACE_MAXADDR, /* lowaddr */ 2050758cc3dcSJack F Vogel BUS_SPACE_MAXADDR, /* highaddr */ 2051758cc3dcSJack F Vogel NULL, NULL, /* filter, filterarg */ 2052758cc3dcSJack F Vogel size, /* maxsize */ 2053758cc3dcSJack F Vogel 1, /* nsegments */ 2054758cc3dcSJack F Vogel size, /* maxsegsize */ 2055758cc3dcSJack F Vogel BUS_DMA_ALLOCNOW, /* flags */ 2056758cc3dcSJack F Vogel NULL, /* lockfunc */ 2057758cc3dcSJack F Vogel NULL, /* lockfuncarg */ 2058758cc3dcSJack F Vogel &dma->dma_tag); 2059758cc3dcSJack F Vogel if (r != 0) { 2060758cc3dcSJack F Vogel device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; " 2061758cc3dcSJack F Vogel "error %u\n", r); 2062758cc3dcSJack F Vogel goto fail_0; 2063758cc3dcSJack F Vogel } 2064758cc3dcSJack F Vogel r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr, 2065758cc3dcSJack F Vogel BUS_DMA_NOWAIT, &dma->dma_map); 2066758cc3dcSJack F Vogel if (r != 0) { 2067758cc3dcSJack F Vogel device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; " 2068758cc3dcSJack F Vogel "error %u\n", r); 2069758cc3dcSJack F Vogel goto fail_1; 2070758cc3dcSJack F Vogel } 2071758cc3dcSJack F Vogel r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 2072758cc3dcSJack F Vogel size, 2073758cc3dcSJack F Vogel ixgbe_dmamap_cb, 2074758cc3dcSJack F Vogel &dma->dma_paddr, 2075758cc3dcSJack F Vogel mapflags | BUS_DMA_NOWAIT); 2076758cc3dcSJack F Vogel if (r != 0) { 2077758cc3dcSJack F Vogel device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; " 2078758cc3dcSJack F Vogel "error %u\n", r); 2079758cc3dcSJack F Vogel goto fail_2; 2080758cc3dcSJack F Vogel } 2081758cc3dcSJack F Vogel dma->dma_size = size; 2082758cc3dcSJack F Vogel return (0); 2083758cc3dcSJack F Vogel fail_2: 2084758cc3dcSJack F Vogel bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 2085758cc3dcSJack F Vogel fail_1: 2086758cc3dcSJack F Vogel bus_dma_tag_destroy(dma->dma_tag); 2087758cc3dcSJack F Vogel fail_0: 2088758cc3dcSJack F Vogel dma->dma_tag = NULL; 2089758cc3dcSJack F Vogel return (r); 2090758cc3dcSJack F Vogel } 2091758cc3dcSJack F Vogel 2092758cc3dcSJack F Vogel void 2093758cc3dcSJack F Vogel ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma) 2094758cc3dcSJack F Vogel { 2095758cc3dcSJack F Vogel bus_dmamap_sync(dma->dma_tag, dma->dma_map, 2096758cc3dcSJack F Vogel BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2097758cc3dcSJack F Vogel bus_dmamap_unload(dma->dma_tag, dma->dma_map); 2098758cc3dcSJack F Vogel bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 2099758cc3dcSJack F Vogel bus_dma_tag_destroy(dma->dma_tag); 2100758cc3dcSJack F Vogel } 2101758cc3dcSJack F Vogel 2102758cc3dcSJack F Vogel 2103758cc3dcSJack F Vogel /********************************************************************* 2104758cc3dcSJack F Vogel * 2105758cc3dcSJack F Vogel * Allocate memory for the transmit and receive rings, and then 2106758cc3dcSJack F Vogel * the descriptors associated with each, called only once at attach. 2107758cc3dcSJack F Vogel * 2108758cc3dcSJack F Vogel **********************************************************************/ 2109758cc3dcSJack F Vogel int 2110758cc3dcSJack F Vogel ixgbe_allocate_queues(struct adapter *adapter) 2111758cc3dcSJack F Vogel { 2112758cc3dcSJack F Vogel device_t dev = adapter->dev; 2113758cc3dcSJack F Vogel struct ix_queue *que; 2114758cc3dcSJack F Vogel struct tx_ring *txr; 2115758cc3dcSJack F Vogel struct rx_ring *rxr; 2116758cc3dcSJack F Vogel int rsize, tsize, error = IXGBE_SUCCESS; 2117758cc3dcSJack F Vogel int txconf = 0, rxconf = 0; 2118758cc3dcSJack F Vogel 2119758cc3dcSJack F Vogel /* First allocate the top level queue structs */ 2120758cc3dcSJack F Vogel if (!(adapter->queues = 2121758cc3dcSJack F Vogel (struct ix_queue *) malloc(sizeof(struct ix_queue) * 2122758cc3dcSJack F Vogel adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2123758cc3dcSJack F Vogel device_printf(dev, "Unable to allocate queue memory\n"); 2124758cc3dcSJack F Vogel error = ENOMEM; 2125758cc3dcSJack F Vogel goto fail; 2126758cc3dcSJack F Vogel } 2127758cc3dcSJack F Vogel 2128758cc3dcSJack F Vogel /* First allocate the TX ring struct memory */ 2129758cc3dcSJack F Vogel if (!(adapter->tx_rings = 2130758cc3dcSJack F Vogel (struct tx_ring *) malloc(sizeof(struct tx_ring) * 2131758cc3dcSJack F Vogel adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2132758cc3dcSJack F Vogel device_printf(dev, "Unable to allocate TX ring memory\n"); 2133758cc3dcSJack F Vogel error = ENOMEM; 2134758cc3dcSJack F Vogel goto tx_fail; 2135758cc3dcSJack F Vogel } 2136758cc3dcSJack F Vogel 2137758cc3dcSJack F Vogel /* Next allocate the RX */ 2138758cc3dcSJack F Vogel if (!(adapter->rx_rings = 2139758cc3dcSJack F Vogel (struct rx_ring *) malloc(sizeof(struct rx_ring) * 2140758cc3dcSJack F Vogel adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2141758cc3dcSJack F Vogel device_printf(dev, "Unable to allocate RX ring memory\n"); 2142758cc3dcSJack F Vogel error = ENOMEM; 2143758cc3dcSJack F Vogel goto rx_fail; 2144758cc3dcSJack F Vogel } 2145758cc3dcSJack F Vogel 2146758cc3dcSJack F Vogel /* For the ring itself */ 2147758cc3dcSJack F Vogel tsize = roundup2(adapter->num_tx_desc * 2148758cc3dcSJack F Vogel sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN); 2149758cc3dcSJack F Vogel 2150758cc3dcSJack F Vogel /* 2151758cc3dcSJack F Vogel * Now set up the TX queues, txconf is needed to handle the 2152758cc3dcSJack F Vogel * possibility that things fail midcourse and we need to 2153758cc3dcSJack F Vogel * undo memory gracefully 2154758cc3dcSJack F Vogel */ 2155758cc3dcSJack F Vogel for (int i = 0; i < adapter->num_queues; i++, txconf++) { 2156758cc3dcSJack F Vogel /* Set up some basics */ 2157758cc3dcSJack F Vogel txr = &adapter->tx_rings[i]; 2158758cc3dcSJack F Vogel txr->adapter = adapter; 2159758cc3dcSJack F Vogel txr->me = i; 2160758cc3dcSJack F Vogel txr->num_desc = adapter->num_tx_desc; 2161758cc3dcSJack F Vogel 2162758cc3dcSJack F Vogel /* Initialize the TX side lock */ 2163758cc3dcSJack F Vogel snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", 2164758cc3dcSJack F Vogel device_get_nameunit(dev), txr->me); 2165758cc3dcSJack F Vogel mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF); 2166758cc3dcSJack F Vogel 2167758cc3dcSJack F Vogel if (ixgbe_dma_malloc(adapter, tsize, 2168758cc3dcSJack F Vogel &txr->txdma, BUS_DMA_NOWAIT)) { 2169758cc3dcSJack F Vogel device_printf(dev, 2170758cc3dcSJack F Vogel "Unable to allocate TX Descriptor memory\n"); 2171758cc3dcSJack F Vogel error = ENOMEM; 2172758cc3dcSJack F Vogel goto err_tx_desc; 2173758cc3dcSJack F Vogel } 2174758cc3dcSJack F Vogel txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr; 2175758cc3dcSJack F Vogel bzero((void *)txr->tx_base, tsize); 2176758cc3dcSJack F Vogel 2177758cc3dcSJack F Vogel /* Now allocate transmit buffers for the ring */ 2178758cc3dcSJack F Vogel if (ixgbe_allocate_transmit_buffers(txr)) { 2179758cc3dcSJack F Vogel device_printf(dev, 2180758cc3dcSJack F Vogel "Critical Failure setting up transmit buffers\n"); 2181758cc3dcSJack F Vogel error = ENOMEM; 2182758cc3dcSJack F Vogel goto err_tx_desc; 2183758cc3dcSJack F Vogel } 2184758cc3dcSJack F Vogel #ifndef IXGBE_LEGACY_TX 2185758cc3dcSJack F Vogel /* Allocate a buf ring */ 2186758cc3dcSJack F Vogel txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF, 2187758cc3dcSJack F Vogel M_WAITOK, &txr->tx_mtx); 2188758cc3dcSJack F Vogel if (txr->br == NULL) { 2189758cc3dcSJack F Vogel device_printf(dev, 2190758cc3dcSJack F Vogel "Critical Failure setting up buf ring\n"); 2191758cc3dcSJack F Vogel error = ENOMEM; 2192758cc3dcSJack F Vogel goto err_tx_desc; 2193758cc3dcSJack F Vogel } 2194758cc3dcSJack F Vogel #endif 2195758cc3dcSJack F Vogel } 2196758cc3dcSJack F Vogel 2197758cc3dcSJack F Vogel /* 2198758cc3dcSJack F Vogel * Next the RX queues... 2199758cc3dcSJack F Vogel */ 2200758cc3dcSJack F Vogel rsize = roundup2(adapter->num_rx_desc * 2201758cc3dcSJack F Vogel sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); 2202758cc3dcSJack F Vogel for (int i = 0; i < adapter->num_queues; i++, rxconf++) { 2203758cc3dcSJack F Vogel rxr = &adapter->rx_rings[i]; 2204758cc3dcSJack F Vogel /* Set up some basics */ 2205758cc3dcSJack F Vogel rxr->adapter = adapter; 2206758cc3dcSJack F Vogel rxr->me = i; 2207758cc3dcSJack F Vogel rxr->num_desc = adapter->num_rx_desc; 2208758cc3dcSJack F Vogel 2209758cc3dcSJack F Vogel /* Initialize the RX side lock */ 2210758cc3dcSJack F Vogel snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", 2211758cc3dcSJack F Vogel device_get_nameunit(dev), rxr->me); 2212758cc3dcSJack F Vogel mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF); 2213758cc3dcSJack F Vogel 2214758cc3dcSJack F Vogel if (ixgbe_dma_malloc(adapter, rsize, 2215758cc3dcSJack F Vogel &rxr->rxdma, BUS_DMA_NOWAIT)) { 2216758cc3dcSJack F Vogel device_printf(dev, 2217758cc3dcSJack F Vogel "Unable to allocate RxDescriptor memory\n"); 2218758cc3dcSJack F Vogel error = ENOMEM; 2219758cc3dcSJack F Vogel goto err_rx_desc; 2220758cc3dcSJack F Vogel } 2221758cc3dcSJack F Vogel rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr; 2222758cc3dcSJack F Vogel bzero((void *)rxr->rx_base, rsize); 2223758cc3dcSJack F Vogel 2224758cc3dcSJack F Vogel /* Allocate receive buffers for the ring*/ 2225758cc3dcSJack F Vogel if (ixgbe_allocate_receive_buffers(rxr)) { 2226758cc3dcSJack F Vogel device_printf(dev, 2227758cc3dcSJack F Vogel "Critical Failure setting up receive buffers\n"); 2228758cc3dcSJack F Vogel error = ENOMEM; 2229758cc3dcSJack F Vogel goto err_rx_desc; 2230758cc3dcSJack F Vogel } 2231758cc3dcSJack F Vogel } 2232758cc3dcSJack F Vogel 2233758cc3dcSJack F Vogel /* 2234758cc3dcSJack F Vogel ** Finally set up the queue holding structs 2235758cc3dcSJack F Vogel */ 2236758cc3dcSJack F Vogel for (int i = 0; i < adapter->num_queues; i++) { 2237758cc3dcSJack F Vogel que = &adapter->queues[i]; 2238758cc3dcSJack F Vogel que->adapter = adapter; 2239758cc3dcSJack F Vogel que->me = i; 2240758cc3dcSJack F Vogel que->txr = &adapter->tx_rings[i]; 2241758cc3dcSJack F Vogel que->rxr = &adapter->rx_rings[i]; 2242758cc3dcSJack F Vogel } 2243758cc3dcSJack F Vogel 2244758cc3dcSJack F Vogel return (0); 2245758cc3dcSJack F Vogel 2246758cc3dcSJack F Vogel err_rx_desc: 2247758cc3dcSJack F Vogel for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) 2248758cc3dcSJack F Vogel ixgbe_dma_free(adapter, &rxr->rxdma); 2249758cc3dcSJack F Vogel err_tx_desc: 2250758cc3dcSJack F Vogel for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) 2251758cc3dcSJack F Vogel ixgbe_dma_free(adapter, &txr->txdma); 2252758cc3dcSJack F Vogel free(adapter->rx_rings, M_DEVBUF); 2253758cc3dcSJack F Vogel rx_fail: 2254758cc3dcSJack F Vogel free(adapter->tx_rings, M_DEVBUF); 2255758cc3dcSJack F Vogel tx_fail: 2256758cc3dcSJack F Vogel free(adapter->queues, M_DEVBUF); 2257758cc3dcSJack F Vogel fail: 2258758cc3dcSJack F Vogel return (error); 2259758cc3dcSJack F Vogel } 2260