1 /*
2 * BSD LICENSE
3 *
4 * Copyright(c) 2017 Cavium, Inc.. All rights reserved.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 * * Neither the name of Cavium, Inc. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34 /* \file lio_droq.h
35 * \brief Implementation of Octeon Output queues. "Output" is with
36 * respect to the Octeon device on the NIC. From this driver's point of
37 * view they are ingress queues.
38 */
39
40 #ifndef __LIO_DROQ_H__
41 #define __LIO_DROQ_H__
42
43 /*
44 * Octeon descriptor format.
45 * The descriptor ring is made of descriptors which have 2 64-bit values:
46 * -# Physical (bus) address of the data buffer.
47 * -# Physical (bus) address of a lio_droq_info structure.
48 * The Octeon device DMA's incoming packets and its information at the address
49 * given by these descriptor fields.
50 */
51 struct lio_droq_desc {
52 /* The buffer pointer */
53 uint64_t buffer_ptr;
54
55 /* The Info pointer */
56 uint64_t info_ptr;
57 };
58
59 #define LIO_DROQ_DESC_SIZE (sizeof(struct lio_droq_desc))
60
61 /*
62 * Information about packet DMA'ed by Octeon.
63 * The format of the information available at Info Pointer after Octeon
64 * has posted a packet. Not all descriptors have valid information. Only
65 * the Info field of the first descriptor for a packet has information
66 * about the packet.
67 */
68 struct lio_droq_info {
69 /* The Length of the packet. */
70 uint64_t length;
71
72 /* The Output Receive Header. */
73 union octeon_rh rh;
74
75 };
76
77 #define LIO_DROQ_INFO_SIZE (sizeof(struct lio_droq_info))
78
79 /*
80 * Pointer to data buffer.
81 * Driver keeps a pointer to the data buffer that it made available to
82 * the Octeon device. Since the descriptor ring keeps physical (bus)
83 * addresses, this field is required for the driver to keep track of
84 * the virtual address pointers.
85 */
86 struct lio_recv_buffer {
87 /* Packet buffer, including metadata. */
88 void *buffer;
89
90 /* Data in the packet buffer. */
91 uint8_t *data;
92 };
93
94 #define LIO_DROQ_RECVBUF_SIZE (sizeof(struct lio_recv_buffer))
95
96 /* Output Queue statistics. Each output queue has four stats fields. */
97 struct lio_droq_stats {
98 /* Number of packets received in this queue. */
99 uint64_t pkts_received;
100
101 /* Bytes received by this queue. */
102 uint64_t bytes_received;
103
104 /* Packets dropped due to no dispatch function. */
105 uint64_t dropped_nodispatch;
106
107 /* Packets dropped due to no memory available. */
108 uint64_t dropped_nomem;
109
110 /* Packets dropped due to large number of pkts to process. */
111 uint64_t dropped_toomany;
112
113 /* Number of packets sent to stack from this queue. */
114 uint64_t rx_pkts_received;
115
116 /* Number of Bytes sent to stack from this queue. */
117 uint64_t rx_bytes_received;
118
119 /* Num of Packets dropped due to receive path failures. */
120 uint64_t rx_dropped;
121
122 uint64_t rx_vxlan;
123
124 /* Num of failures of lio_recv_buffer_alloc() */
125 uint64_t rx_alloc_failure;
126
127 };
128
129 /*
130 * The maximum number of buffers that can be dispatched from the
131 * output/dma queue. Set to 64 assuming 1K buffers in DROQ and the fact that
132 * max packet size from DROQ is 64K.
133 */
134 #define LIO_MAX_RECV_BUFS 64
135
136 /*
137 * Receive Packet format used when dispatching output queue packets
138 * with non-raw opcodes.
139 * The received packet will be sent to the upper layers using this
140 * structure which is passed as a parameter to the dispatch function
141 */
142 struct lio_recv_pkt {
143 /* Number of buffers in this received packet */
144 uint16_t buffer_count;
145
146 /* Id of the device that is sending the packet up */
147 uint16_t octeon_id;
148
149 /* Length of data in the packet buffer */
150 uint32_t length;
151
152 /* The receive header */
153 union octeon_rh rh;
154
155 /* Pointer to the OS-specific packet buffer */
156 struct mbuf *buffer_ptr[LIO_MAX_RECV_BUFS];
157
158 /* Size of the buffers pointed to by ptr's in buffer_ptr */
159 uint32_t buffer_size[LIO_MAX_RECV_BUFS];
160 };
161
162 #define LIO_RECV_PKT_SIZE (sizeof(struct lio_recv_pkt))
163
164 /*
165 * The first parameter of a dispatch function.
166 * For a raw mode opcode, the driver dispatches with the device
167 * pointer in this structure.
168 * For non-raw mode opcode, the driver dispatches the recv_pkt
169 * created to contain the buffers with data received from Octeon.
170 * ---------------------
171 * | *recv_pkt ----|---
172 * |-------------------| |
173 * | 0 or more bytes | |
174 * | reserved by driver| |
175 * |-------------------|<-/
176 * | lio_recv_pkt |
177 * | |
178 * |___________________|
179 */
180 struct lio_recv_info {
181 void *rsvd;
182 struct lio_recv_pkt *recv_pkt;
183 };
184
185 #define LIO_RECV_INFO_SIZE (sizeof(struct lio_recv_info))
186
187 /*
188 * Allocate a recv_info structure. The recv_pkt pointer in the recv_info
189 * structure is filled in before this call returns.
190 * @param extra_bytes - extra bytes to be allocated at the end of the recv info
191 * structure.
192 * @return - pointer to a newly allocated recv_info structure.
193 */
194 static inline struct lio_recv_info *
lio_alloc_recv_info(int extra_bytes)195 lio_alloc_recv_info(int extra_bytes)
196 {
197 struct lio_recv_info *recv_info;
198 uint8_t *buf;
199
200 buf = malloc(LIO_RECV_PKT_SIZE + LIO_RECV_INFO_SIZE +
201 extra_bytes, M_DEVBUF, M_NOWAIT | M_ZERO);
202 if (buf == NULL)
203 return (NULL);
204
205 recv_info = (struct lio_recv_info *)buf;
206 recv_info->recv_pkt = (struct lio_recv_pkt *)(buf + LIO_RECV_INFO_SIZE);
207 recv_info->rsvd = NULL;
208 if (extra_bytes)
209 recv_info->rsvd = buf + LIO_RECV_INFO_SIZE + LIO_RECV_PKT_SIZE;
210
211 return (recv_info);
212 }
213
214 /*
215 * Free a recv_info structure.
216 * @param recv_info - Pointer to receive_info to be freed
217 */
218 static inline void
lio_free_recv_info(struct lio_recv_info * recv_info)219 lio_free_recv_info(struct lio_recv_info *recv_info)
220 {
221
222 free(recv_info, M_DEVBUF);
223 }
224
225 typedef int (*lio_dispatch_fn_t)(struct lio_recv_info *, void *);
226
227 /*
228 * Used by NIC module to register packet handler and to get device
229 * information for each octeon device.
230 */
231 struct lio_droq_ops {
232 /*
233 * This registered function will be called by the driver with
234 * the pointer to buffer from droq and length of
235 * data in the buffer. The receive header gives the port
236 * number to the caller. Function pointer is set by caller.
237 */
238 void (*fptr) (void *, uint32_t, union octeon_rh *, void *,
239 void *);
240 void *farg;
241
242 /*
243 * Flag indicating if the DROQ handler should drop packets that
244 * it cannot handle in one iteration. Set by caller.
245 */
246 uint32_t drop_on_max;
247 };
248
249 /*
250 * The Descriptor Ring Output Queue structure.
251 * This structure has all the information required to implement a
252 * Octeon DROQ.
253 */
254 struct lio_droq {
255 /* A lock to protect access to this ring. */
256 struct mtx lock;
257
258 uint32_t q_no;
259
260 uint32_t pkt_count;
261
262 struct lio_droq_ops ops;
263
264 struct octeon_device *oct_dev;
265
266 /* The 8B aligned descriptor ring starts at this address. */
267 struct lio_droq_desc *desc_ring;
268
269 /* Index in the ring where the driver should read the next packet */
270 uint32_t read_idx;
271
272 /*
273 * Index in the ring where the driver will refill the descriptor's
274 * buffer
275 */
276 uint32_t refill_idx;
277
278 /* Packets pending to be processed */
279 volatile int pkts_pending;
280
281 /* Number of descriptors in this ring. */
282 uint32_t max_count;
283
284 /* The number of descriptors pending refill. */
285 uint32_t refill_count;
286
287 uint32_t pkts_per_intr;
288 uint32_t refill_threshold;
289
290 /*
291 * The max number of descriptors in DROQ without a buffer.
292 * This field is used to keep track of empty space threshold. If the
293 * refill_count reaches this value, the DROQ cannot accept a max-sized
294 * (64K) packet.
295 */
296 uint32_t max_empty_descs;
297
298 /*
299 * The receive buffer list. This list has the virtual addresses of
300 * the buffers.
301 */
302 struct lio_recv_buffer *recv_buf_list;
303
304 /* The size of each buffer pointed by the buffer pointer. */
305 uint32_t buffer_size;
306
307 /*
308 * Offset to packet credit register.
309 * Host writes number of info/buffer ptrs available to this register
310 */
311 uint32_t pkts_credit_reg;
312
313 /*
314 * Offset packet sent register.
315 * Octeon writes the number of packets DMA'ed to host memory
316 * in this register.
317 */
318 uint32_t pkts_sent_reg;
319
320 struct lio_stailq_head dispatch_stq_head;
321
322 /* Statistics for this DROQ. */
323 struct lio_droq_stats stats;
324
325 /* DMA mapped address of the DROQ descriptor ring. */
326 vm_paddr_t desc_ring_dma;
327
328 /* application context */
329 void *app_ctx;
330
331 uint32_t cpu_id;
332
333 struct task droq_task;
334 struct taskqueue *droq_taskqueue;
335
336 struct lro_ctrl lro;
337 };
338
339 #define LIO_DROQ_SIZE (sizeof(struct lio_droq))
340
341 /*
342 * Allocates space for the descriptor ring for the droq and sets the
343 * base addr, num desc etc in Octeon registers.
344 *
345 * @param oct_dev - pointer to the octeon device structure
346 * @param q_no - droq no.
347 * @param app_ctx - pointer to application context
348 * @return Success: 0 Failure: 1
349 */
350 int lio_init_droq(struct octeon_device *oct_dev,
351 uint32_t q_no, uint32_t num_descs, uint32_t desc_size,
352 void *app_ctx);
353
354 /*
355 * Frees the space for descriptor ring for the droq.
356 *
357 * @param oct_dev - pointer to the octeon device structure
358 * @param q_no - droq no.
359 * @return: Success: 0 Failure: 1
360 */
361 int lio_delete_droq(struct octeon_device *oct_dev, uint32_t q_no);
362
363 /*
364 * Register a change in droq operations. The ops field has a pointer to a
365 * function which will called by the DROQ handler for all packets arriving
366 * on output queues given by q_no irrespective of the type of packet.
367 * The ops field also has a flag which if set tells the DROQ handler to
368 * drop packets if it receives more than what it can process in one
369 * invocation of the handler.
370 * @param oct - octeon device
371 * @param q_no - octeon output queue number (0 <= q_no <= MAX_OCTEON_DROQ-1
372 * @param ops - the droq_ops settings for this queue
373 * @return - 0 on success, -ENODEV or -EINVAL on error.
374 */
375 int lio_register_droq_ops(struct octeon_device *oct, uint32_t q_no,
376 struct lio_droq_ops *ops);
377
378 /*
379 * Resets the function pointer and flag settings made by
380 * lio_register_droq_ops(). After this routine is called, the DROQ handler
381 * will lookup dispatch function for each arriving packet on the output queue
382 * given by q_no.
383 * @param oct - octeon device
384 * @param q_no - octeon output queue number (0 <= q_no <= MAX_OCTEON_DROQ-1
385 * @return - 0 on success, -ENODEV or -EINVAL on error.
386 */
387 int lio_unregister_droq_ops(struct octeon_device *oct, uint32_t q_no);
388
389 /*
390 * Register a dispatch function for a opcode/subcode. The driver will call
391 * this dispatch function when it receives a packet with the given
392 * opcode/subcode in its output queues along with the user specified
393 * argument.
394 * @param oct - the octeon device to register with.
395 * @param opcode - the opcode for which the dispatch will be registered.
396 * @param subcode - the subcode for which the dispatch will be registered
397 * @param fn - the dispatch function.
398 * @param fn_arg - user specified that will be passed along with the
399 * dispatch function by the driver.
400 * @return Success: 0; Failure: 1
401 */
402 int lio_register_dispatch_fn(struct octeon_device *oct, uint16_t opcode,
403 uint16_t subcode, lio_dispatch_fn_t fn,
404 void *fn_arg);
405
406 /*
407 * Remove registration for an opcode/subcode. This will delete the mapping for
408 * an opcode/subcode. The dispatch function will be unregistered and will no
409 * longer be called if a packet with the opcode/subcode arrives in the driver
410 * output queues.
411 * @param oct - the octeon device to unregister from.
412 * @param opcode - the opcode to be unregistered.
413 * @param subcode - the subcode to be unregistered.
414 *
415 * @return Success: 0; Failure: 1
416 */
417 int lio_unregister_dispatch_fn(struct octeon_device *oct, uint16_t opcode,
418 uint16_t subcode);
419
420 uint32_t lio_droq_check_hw_for_pkts(struct lio_droq *droq);
421
422 int lio_create_droq(struct octeon_device *oct, uint32_t q_no,
423 uint32_t num_descs, uint32_t desc_size, void *app_ctx);
424
425 int lio_droq_process_packets(struct octeon_device *oct,
426 struct lio_droq *droq, uint32_t budget);
427
428 uint32_t lio_droq_refill(struct octeon_device *octeon_dev,
429 struct lio_droq *droq);
430 void lio_droq_bh(void *ptr, int pending __unused);
431 #endif /* __LIO_DROQ_H__ */
432