1 /*
2 * BSD LICENSE
3 *
4 * Copyright(c) 2017 Cavium, Inc.. All rights reserved.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 * * Neither the name of Cavium, Inc. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34 #include "lio_bsd.h"
35 #include "lio_common.h"
36 #include "lio_droq.h"
37 #include "lio_iq.h"
38 #include "lio_response_manager.h"
39 #include "lio_device.h"
40 #include "lio_ctrl.h"
41 #include "lio_main.h"
42 #include "lio_network.h"
43 #include "lio_rxtx.h"
44
45 int
lio_xmit(struct lio * lio,struct lio_instr_queue * iq,struct mbuf ** m_headp)46 lio_xmit(struct lio *lio, struct lio_instr_queue *iq,
47 struct mbuf **m_headp)
48 {
49 struct lio_data_pkt ndata;
50 union lio_cmd_setup cmdsetup;
51 struct lio_mbuf_free_info *finfo = NULL;
52 struct octeon_device *oct = iq->oct_dev;
53 struct lio_iq_stats *stats;
54 struct octeon_instr_irh *irh;
55 struct lio_request_list *tx_buf;
56 union lio_tx_info *tx_info;
57 struct mbuf *m_head;
58 bus_dma_segment_t segs[LIO_MAX_SG];
59 bus_dmamap_t map;
60 uint64_t dptr = 0;
61 uint32_t tag = 0;
62 int iq_no = 0;
63 int nsegs;
64 int status = 0;
65
66 iq_no = iq->txpciq.s.q_no;
67 tag = iq_no;
68 stats = &oct->instr_queue[iq_no]->stats;
69 tx_buf = iq->request_list + iq->host_write_index;
70
71 /*
72 * Check for all conditions in which the current packet cannot be
73 * transmitted.
74 */
75 if (!(atomic_load_acq_int(&lio->ifstate) & LIO_IFSTATE_RUNNING) ||
76 (!lio->linfo.link.s.link_up)) {
77 lio_dev_info(oct, "Transmit failed link_status : %d\n",
78 lio->linfo.link.s.link_up);
79 status = ENETDOWN;
80 goto drop_packet;
81 }
82
83 if (lio_iq_is_full(oct, iq_no)) {
84 /* Defer sending if queue is full */
85 lio_dev_dbg(oct, "Transmit failed iq:%d full\n", iq_no);
86 stats->tx_iq_busy++;
87 return (ENOBUFS);
88 }
89
90 map = tx_buf->map;
91 status = bus_dmamap_load_mbuf_sg(iq->txtag, map, *m_headp, segs, &nsegs,
92 BUS_DMA_NOWAIT);
93 if (status == EFBIG) {
94 struct mbuf *m;
95
96 m = m_defrag(*m_headp, M_NOWAIT);
97 if (m == NULL) {
98 stats->mbuf_defrag_failed++;
99 goto drop_packet;
100 }
101
102 *m_headp = m;
103 status = bus_dmamap_load_mbuf_sg(iq->txtag, map,
104 *m_headp, segs, &nsegs,
105 BUS_DMA_NOWAIT);
106 }
107
108 if (status == ENOMEM) {
109 goto retry;
110 } else if (status) {
111 stats->tx_dmamap_fail++;
112 lio_dev_dbg(oct, "bus_dmamap_load_mbuf_sg failed with error %d. iq:%d",
113 status, iq_no);
114 goto drop_packet;
115 }
116
117 m_head = *m_headp;
118
119 /* Info used to unmap and free the buffers. */
120 finfo = &tx_buf->finfo;
121 finfo->map = map;
122 finfo->mb = m_head;
123
124 /* Prepare the attributes for the data to be passed to OSI. */
125 bzero(&ndata, sizeof(struct lio_data_pkt));
126
127 ndata.buf = (void *)finfo;
128 ndata.q_no = iq_no;
129 ndata.datasize = m_head->m_pkthdr.len;
130
131 cmdsetup.cmd_setup64 = 0;
132 cmdsetup.s.iq_no = iq_no;
133
134 if (m_head->m_pkthdr.csum_flags & CSUM_IP)
135 cmdsetup.s.ip_csum = 1;
136
137 if ((m_head->m_pkthdr.csum_flags & (CSUM_IP_TCP | CSUM_IP6_TCP)) ||
138 (m_head->m_pkthdr.csum_flags & (CSUM_IP_UDP | CSUM_IP6_UDP)))
139 cmdsetup.s.transport_csum = 1;
140
141 if (nsegs == 1) {
142 cmdsetup.s.u.datasize = segs[0].ds_len;
143 lio_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag);
144
145 dptr = segs[0].ds_addr;
146 ndata.cmd.cmd3.dptr = dptr;
147 ndata.reqtype = LIO_REQTYPE_NORESP_NET;
148
149 } else {
150 struct lio_gather *g;
151 int i;
152
153 mtx_lock(&lio->glist_lock[iq_no]);
154 g = (struct lio_gather *)
155 lio_delete_first_node(&lio->ghead[iq_no]);
156 mtx_unlock(&lio->glist_lock[iq_no]);
157
158 if (g == NULL) {
159 lio_dev_err(oct,
160 "Transmit scatter gather: glist null!\n");
161 goto retry;
162 }
163
164 cmdsetup.s.gather = 1;
165 cmdsetup.s.u.gatherptrs = nsegs;
166 lio_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag);
167
168 bzero(g->sg, g->sg_size);
169
170 i = 0;
171 while (nsegs--) {
172 g->sg[(i >> 2)].ptr[(i & 3)] = segs[i].ds_addr;
173 lio_add_sg_size(&g->sg[(i >> 2)], segs[i].ds_len,
174 (i & 3));
175 i++;
176 }
177
178 dptr = g->sg_dma_ptr;
179
180 ndata.cmd.cmd3.dptr = dptr;
181 finfo->g = g;
182
183 ndata.reqtype = LIO_REQTYPE_NORESP_NET_SG;
184 }
185
186 irh = (struct octeon_instr_irh *)&ndata.cmd.cmd3.irh;
187 tx_info = (union lio_tx_info *)&ndata.cmd.cmd3.ossp[0];
188
189 if (m_head->m_pkthdr.csum_flags & (CSUM_IP_TSO | CSUM_IP6_TSO)) {
190 tx_info->s.gso_size = m_head->m_pkthdr.tso_segsz;
191 tx_info->s.gso_segs = howmany(m_head->m_pkthdr.len,
192 m_head->m_pkthdr.tso_segsz);
193 stats->tx_gso++;
194 }
195
196 /* HW insert VLAN tag */
197 if (m_head->m_flags & M_VLANTAG) {
198 irh->priority = m_head->m_pkthdr.ether_vtag >> 13;
199 irh->vlan = m_head->m_pkthdr.ether_vtag & 0xfff;
200 }
201
202 status = lio_send_data_pkt(oct, &ndata);
203 if (status == LIO_IQ_SEND_FAILED)
204 goto retry;
205
206 if (tx_info->s.gso_segs)
207 stats->tx_done += tx_info->s.gso_segs;
208 else
209 stats->tx_done++;
210
211 stats->tx_tot_bytes += ndata.datasize;
212
213 return (0);
214
215 retry:
216 return (ENOBUFS);
217
218 drop_packet:
219 stats->tx_dropped++;
220 lio_dev_err(oct, "IQ%d Transmit dropped: %llu\n", iq_no,
221 LIO_CAST64(stats->tx_dropped));
222
223 m_freem(*m_headp);
224 *m_headp = NULL;
225
226 return (status);
227 }
228
229 int
lio_mq_start_locked(if_t ifp,struct lio_instr_queue * iq)230 lio_mq_start_locked(if_t ifp, struct lio_instr_queue *iq)
231 {
232 struct lio *lio = if_getsoftc(ifp);
233 struct mbuf *next;
234 int err = 0;
235
236 if (((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) ||
237 (!lio->linfo.link.s.link_up))
238 return (-ENETDOWN);
239
240 /* Process the queue */
241 while ((next = drbr_peek(ifp, iq->br)) != NULL) {
242 err = lio_xmit(lio, iq, &next);
243 if (err) {
244 if (next == NULL)
245 drbr_advance(ifp, iq->br);
246 else
247 drbr_putback(ifp, iq->br, next);
248 break;
249 }
250 drbr_advance(ifp, iq->br);
251 /* Send a copy of the frame to the BPF listener */
252 ETHER_BPF_MTAP(ifp, next);
253 if (((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) ||
254 (!lio->linfo.link.s.link_up))
255 break;
256 }
257
258 return (err);
259 }
260
261 int
lio_mq_start(if_t ifp,struct mbuf * m)262 lio_mq_start(if_t ifp, struct mbuf *m)
263 {
264 struct lio *lio = if_getsoftc(ifp);
265 struct octeon_device *oct = lio->oct_dev;
266 struct lio_instr_queue *iq;
267 int err = 0, i;
268 #ifdef RSS
269 uint32_t bucket_id;
270 #endif
271
272 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) {
273 #ifdef RSS
274 if (rss_hash2bucket(m->m_pkthdr.flowid, M_HASHTYPE_GET(m),
275 &bucket_id) == 0) {
276 i = bucket_id % oct->num_iqs;
277 if (bucket_id > oct->num_iqs)
278 lio_dev_dbg(oct,
279 "bucket_id (%d) > num_iqs (%d)\n",
280 bucket_id, oct->num_iqs);
281 } else
282 #endif
283 i = m->m_pkthdr.flowid % oct->num_iqs;
284 } else
285 i = curcpu % oct->num_iqs;
286
287 iq = oct->instr_queue[i];
288
289 err = drbr_enqueue(ifp, iq->br, m);
290 if (err)
291 return (err);
292
293 if (mtx_trylock(&iq->enq_lock)) {
294 lio_mq_start_locked(ifp, iq);
295 mtx_unlock(&iq->enq_lock);
296 }
297
298 return (err);
299 }
300
301 void
lio_qflush(if_t ifp)302 lio_qflush(if_t ifp)
303 {
304 struct lio *lio = if_getsoftc(ifp);
305 struct octeon_device *oct = lio->oct_dev;
306 struct lio_instr_queue *iq;
307 struct mbuf *m;
308 int i;
309
310 for (i = 0; i < LIO_MAX_INSTR_QUEUES(oct); i++) {
311 if (!(oct->io_qmask.iq & BIT_ULL(i)))
312 continue;
313
314 iq = oct->instr_queue[i];
315
316 mtx_lock(&iq->enq_lock);
317 while ((m = buf_ring_dequeue_sc(iq->br)) != NULL)
318 m_freem(m);
319
320 mtx_unlock(&iq->enq_lock);
321 }
322
323 if_qflush(ifp);
324 }
325