1 // SPDX-License-Identifier: GPL-2.0
2 /* Marvell Octeon EP (EndPoint) Ethernet Driver
3  *
4  * Copyright (C) 2020 Marvell.
5  *
6  */
7 
8 #include <linux/pci.h>
9 #include <linux/etherdevice.h>
10 #include <linux/vmalloc.h>
11 
12 #include "octep_config.h"
13 #include "octep_main.h"
14 
15 static void octep_oq_reset_indices(struct octep_oq *oq)
16 {
17 	oq->host_read_idx = 0;
18 	oq->host_refill_idx = 0;
19 	oq->refill_count = 0;
20 	oq->last_pkt_count = 0;
21 	oq->pkts_pending = 0;
22 }
23 
24 /**
25  * octep_oq_fill_ring_buffers() - fill initial receive buffers for Rx ring.
26  *
27  * @oq: Octeon Rx queue data structure.
28  *
29  * Return: 0, if successfully filled receive buffers for all descriptors.
30  *         -1, if failed to allocate a buffer or failed to map for DMA.
31  */
32 static int octep_oq_fill_ring_buffers(struct octep_oq *oq)
33 {
34 	struct octep_oq_desc_hw *desc_ring = oq->desc_ring;
35 	struct page *page;
36 	u32 i;
37 
38 	for (i = 0; i < oq->max_count; i++) {
39 		page = dev_alloc_page();
40 		if (unlikely(!page)) {
41 			dev_err(oq->dev, "Rx buffer alloc failed\n");
42 			goto rx_buf_alloc_err;
43 		}
44 		desc_ring[i].buffer_ptr = dma_map_page(oq->dev, page, 0,
45 						       PAGE_SIZE,
46 						       DMA_FROM_DEVICE);
47 		if (dma_mapping_error(oq->dev, desc_ring[i].buffer_ptr)) {
48 			dev_err(oq->dev,
49 				"OQ-%d buffer alloc: DMA mapping error!\n",
50 				oq->q_no);
51 			put_page(page);
52 			goto dma_map_err;
53 		}
54 		oq->buff_info[i].page = page;
55 	}
56 
57 	return 0;
58 
59 dma_map_err:
60 rx_buf_alloc_err:
61 	while (i) {
62 		i--;
63 		dma_unmap_page(oq->dev, desc_ring[i].buffer_ptr, PAGE_SIZE, DMA_FROM_DEVICE);
64 		put_page(oq->buff_info[i].page);
65 		oq->buff_info[i].page = NULL;
66 	}
67 
68 	return -1;
69 }
70 
71 /**
72  * octep_oq_refill() - refill buffers for used Rx ring descriptors.
73  *
74  * @oct: Octeon device private data structure.
75  * @oq: Octeon Rx queue data structure.
76  *
77  * Return: number of descriptors successfully refilled with receive buffers.
78  */
79 static int octep_oq_refill(struct octep_device *oct, struct octep_oq *oq)
80 {
81 	struct octep_oq_desc_hw *desc_ring = oq->desc_ring;
82 	struct page *page;
83 	u32 refill_idx, i;
84 
85 	refill_idx = oq->host_refill_idx;
86 	for (i = 0; i < oq->refill_count; i++) {
87 		page = dev_alloc_page();
88 		if (unlikely(!page)) {
89 			dev_err(oq->dev, "refill: rx buffer alloc failed\n");
90 			oq->stats.alloc_failures++;
91 			break;
92 		}
93 
94 		desc_ring[refill_idx].buffer_ptr = dma_map_page(oq->dev, page, 0,
95 								PAGE_SIZE, DMA_FROM_DEVICE);
96 		if (dma_mapping_error(oq->dev, desc_ring[refill_idx].buffer_ptr)) {
97 			dev_err(oq->dev,
98 				"OQ-%d buffer refill: DMA mapping error!\n",
99 				oq->q_no);
100 			put_page(page);
101 			oq->stats.alloc_failures++;
102 			break;
103 		}
104 		oq->buff_info[refill_idx].page = page;
105 		refill_idx++;
106 		if (refill_idx == oq->max_count)
107 			refill_idx = 0;
108 	}
109 	oq->host_refill_idx = refill_idx;
110 	oq->refill_count -= i;
111 
112 	return i;
113 }
114 
115 /**
116  * octep_setup_oq() - Setup a Rx queue.
117  *
118  * @oct: Octeon device private data structure.
119  * @q_no: Rx queue number to be setup.
120  *
121  * Allocate resources for a Rx queue.
122  */
123 static int octep_setup_oq(struct octep_device *oct, int q_no)
124 {
125 	struct octep_oq *oq;
126 	u32 desc_ring_size;
127 
128 	oq = vzalloc(sizeof(*oq));
129 	if (!oq)
130 		goto create_oq_fail;
131 	oct->oq[q_no] = oq;
132 
133 	oq->octep_dev = oct;
134 	oq->netdev = oct->netdev;
135 	oq->dev = &oct->pdev->dev;
136 	oq->q_no = q_no;
137 	oq->max_count = CFG_GET_OQ_NUM_DESC(oct->conf);
138 	oq->ring_size_mask = oq->max_count - 1;
139 	oq->buffer_size = CFG_GET_OQ_BUF_SIZE(oct->conf);
140 	oq->max_single_buffer_size = oq->buffer_size - OCTEP_OQ_RESP_HW_SIZE;
141 
142 	/* When the hardware/firmware supports additional capabilities,
143 	 * additional header is filled-in by Octeon after length field in
144 	 * Rx packets. this header contains additional packet information.
145 	 */
146 	if (oct->caps_enabled)
147 		oq->max_single_buffer_size -= OCTEP_OQ_RESP_HW_EXT_SIZE;
148 
149 	oq->refill_threshold = CFG_GET_OQ_REFILL_THRESHOLD(oct->conf);
150 
151 	desc_ring_size = oq->max_count * OCTEP_OQ_DESC_SIZE;
152 	oq->desc_ring = dma_alloc_coherent(oq->dev, desc_ring_size,
153 					   &oq->desc_ring_dma, GFP_KERNEL);
154 
155 	if (unlikely(!oq->desc_ring)) {
156 		dev_err(oq->dev,
157 			"Failed to allocate DMA memory for OQ-%d !!\n", q_no);
158 		goto desc_dma_alloc_err;
159 	}
160 
161 	oq->buff_info = vzalloc(oq->max_count * OCTEP_OQ_RECVBUF_SIZE);
162 	if (unlikely(!oq->buff_info)) {
163 		dev_err(&oct->pdev->dev,
164 			"Failed to allocate buffer info for OQ-%d\n", q_no);
165 		goto buf_list_err;
166 	}
167 
168 	if (octep_oq_fill_ring_buffers(oq))
169 		goto oq_fill_buff_err;
170 
171 	octep_oq_reset_indices(oq);
172 	oct->hw_ops.setup_oq_regs(oct, q_no);
173 	oct->num_oqs++;
174 
175 	return 0;
176 
177 oq_fill_buff_err:
178 	vfree(oq->buff_info);
179 	oq->buff_info = NULL;
180 buf_list_err:
181 	dma_free_coherent(oq->dev, desc_ring_size,
182 			  oq->desc_ring, oq->desc_ring_dma);
183 	oq->desc_ring = NULL;
184 desc_dma_alloc_err:
185 	vfree(oq);
186 	oct->oq[q_no] = NULL;
187 create_oq_fail:
188 	return -1;
189 }
190 
191 /**
192  * octep_oq_free_ring_buffers() - Free ring buffers.
193  *
194  * @oq: Octeon Rx queue data structure.
195  *
196  * Free receive buffers in unused Rx queue descriptors.
197  */
198 static void octep_oq_free_ring_buffers(struct octep_oq *oq)
199 {
200 	struct octep_oq_desc_hw *desc_ring = oq->desc_ring;
201 	int  i;
202 
203 	if (!oq->desc_ring || !oq->buff_info)
204 		return;
205 
206 	for (i = 0; i < oq->max_count; i++)  {
207 		if (oq->buff_info[i].page) {
208 			dma_unmap_page(oq->dev, desc_ring[i].buffer_ptr,
209 				       PAGE_SIZE, DMA_FROM_DEVICE);
210 			put_page(oq->buff_info[i].page);
211 			oq->buff_info[i].page = NULL;
212 			desc_ring[i].buffer_ptr = 0;
213 		}
214 	}
215 	octep_oq_reset_indices(oq);
216 }
217 
218 /**
219  * octep_free_oq() - Free Rx queue resources.
220  *
221  * @oq: Octeon Rx queue data structure.
222  *
223  * Free all resources of a Rx queue.
224  */
225 static int octep_free_oq(struct octep_oq *oq)
226 {
227 	struct octep_device *oct = oq->octep_dev;
228 	int q_no = oq->q_no;
229 
230 	octep_oq_free_ring_buffers(oq);
231 
232 	vfree(oq->buff_info);
233 
234 	if (oq->desc_ring)
235 		dma_free_coherent(oq->dev,
236 				  oq->max_count * OCTEP_OQ_DESC_SIZE,
237 				  oq->desc_ring, oq->desc_ring_dma);
238 
239 	vfree(oq);
240 	oct->oq[q_no] = NULL;
241 	oct->num_oqs--;
242 	return 0;
243 }
244 
245 /**
246  * octep_setup_oqs() - setup resources for all Rx queues.
247  *
248  * @oct: Octeon device private data structure.
249  */
250 int octep_setup_oqs(struct octep_device *oct)
251 {
252 	int i, retval = 0;
253 
254 	oct->num_oqs = 0;
255 	for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) {
256 		retval = octep_setup_oq(oct, i);
257 		if (retval) {
258 			dev_err(&oct->pdev->dev,
259 				"Failed to setup OQ(RxQ)-%d.\n", i);
260 			goto oq_setup_err;
261 		}
262 		dev_dbg(&oct->pdev->dev, "Successfully setup OQ(RxQ)-%d.\n", i);
263 	}
264 
265 	return 0;
266 
267 oq_setup_err:
268 	while (i) {
269 		i--;
270 		octep_free_oq(oct->oq[i]);
271 	}
272 	return -1;
273 }
274 
275 /**
276  * octep_oq_dbell_init() - Initialize Rx queue doorbell.
277  *
278  * @oct: Octeon device private data structure.
279  *
280  * Write number of descriptors to Rx queue doorbell register.
281  */
282 void octep_oq_dbell_init(struct octep_device *oct)
283 {
284 	int i;
285 
286 	for (i = 0; i < oct->num_oqs; i++)
287 		writel(oct->oq[i]->max_count, oct->oq[i]->pkts_credit_reg);
288 }
289 
290 /**
291  * octep_free_oqs() - Free resources of all Rx queues.
292  *
293  * @oct: Octeon device private data structure.
294  */
295 void octep_free_oqs(struct octep_device *oct)
296 {
297 	int i;
298 
299 	for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) {
300 		if (!oct->oq[i])
301 			continue;
302 		octep_free_oq(oct->oq[i]);
303 		dev_dbg(&oct->pdev->dev,
304 			"Successfully freed OQ(RxQ)-%d.\n", i);
305 	}
306 }
307 
308 /**
309  * octep_oq_check_hw_for_pkts() - Check for new Rx packets.
310  *
311  * @oct: Octeon device private data structure.
312  * @oq: Octeon Rx queue data structure.
313  *
314  * Return: packets received after previous check.
315  */
316 static int octep_oq_check_hw_for_pkts(struct octep_device *oct,
317 				      struct octep_oq *oq)
318 {
319 	u32 pkt_count, new_pkts;
320 
321 	pkt_count = readl(oq->pkts_sent_reg);
322 	new_pkts = pkt_count - oq->last_pkt_count;
323 
324 	/* Clear the hardware packets counter register if the rx queue is
325 	 * being processed continuously with-in a single interrupt and
326 	 * reached half its max value.
327 	 * this counter is not cleared every time read, to save write cycles.
328 	 */
329 	if (unlikely(pkt_count > 0xF0000000U)) {
330 		writel(pkt_count, oq->pkts_sent_reg);
331 		pkt_count = readl(oq->pkts_sent_reg);
332 		new_pkts += pkt_count;
333 	}
334 	oq->last_pkt_count = pkt_count;
335 	oq->pkts_pending += new_pkts;
336 	return new_pkts;
337 }
338 
339 /**
340  * __octep_oq_process_rx() - Process hardware Rx queue and push to stack.
341  *
342  * @oct: Octeon device private data structure.
343  * @oq: Octeon Rx queue data structure.
344  * @pkts_to_process: number of packets to be processed.
345  *
346  * Process the new packets in Rx queue.
347  * Packets larger than single Rx buffer arrive in consecutive descriptors.
348  * But, count returned by the API only accounts full packets, not fragments.
349  *
350  * Return: number of packets processed and pushed to stack.
351  */
352 static int __octep_oq_process_rx(struct octep_device *oct,
353 				 struct octep_oq *oq, u16 pkts_to_process)
354 {
355 	struct octep_oq_resp_hw_ext *resp_hw_ext = NULL;
356 	struct octep_rx_buffer *buff_info;
357 	struct octep_oq_resp_hw *resp_hw;
358 	u32 pkt, rx_bytes, desc_used;
359 	struct sk_buff *skb;
360 	u16 data_offset;
361 	u32 read_idx;
362 
363 	read_idx = oq->host_read_idx;
364 	rx_bytes = 0;
365 	desc_used = 0;
366 	for (pkt = 0; pkt < pkts_to_process; pkt++) {
367 		buff_info = (struct octep_rx_buffer *)&oq->buff_info[read_idx];
368 		dma_unmap_page(oq->dev, oq->desc_ring[read_idx].buffer_ptr,
369 			       PAGE_SIZE, DMA_FROM_DEVICE);
370 		resp_hw = page_address(buff_info->page);
371 		buff_info->page = NULL;
372 
373 		/* Swap the length field that is in Big-Endian to CPU */
374 		buff_info->len = be64_to_cpu(resp_hw->length);
375 		if (oct->caps_enabled & OCTEP_CAP_RX_CHECKSUM) {
376 			/* Extended response header is immediately after
377 			 * response header (resp_hw)
378 			 */
379 			resp_hw_ext = (struct octep_oq_resp_hw_ext *)
380 				      (resp_hw + 1);
381 			buff_info->len -= OCTEP_OQ_RESP_HW_EXT_SIZE;
382 			/* Packet Data is immediately after
383 			 * extended response header.
384 			 */
385 			data_offset = OCTEP_OQ_RESP_HW_SIZE +
386 				      OCTEP_OQ_RESP_HW_EXT_SIZE;
387 		} else {
388 			/* Data is immediately after
389 			 * Hardware Rx response header.
390 			 */
391 			data_offset = OCTEP_OQ_RESP_HW_SIZE;
392 		}
393 		rx_bytes += buff_info->len;
394 
395 		if (buff_info->len <= oq->max_single_buffer_size) {
396 			skb = build_skb((void *)resp_hw, PAGE_SIZE);
397 			skb_reserve(skb, data_offset);
398 			skb_put(skb, buff_info->len);
399 			read_idx++;
400 			desc_used++;
401 			if (read_idx == oq->max_count)
402 				read_idx = 0;
403 		} else {
404 			struct skb_shared_info *shinfo;
405 			u16 data_len;
406 
407 			skb = build_skb((void *)resp_hw, PAGE_SIZE);
408 			skb_reserve(skb, data_offset);
409 			/* Head fragment includes response header(s);
410 			 * subsequent fragments contains only data.
411 			 */
412 			skb_put(skb, oq->max_single_buffer_size);
413 			read_idx++;
414 			desc_used++;
415 			if (read_idx == oq->max_count)
416 				read_idx = 0;
417 
418 			shinfo = skb_shinfo(skb);
419 			data_len = buff_info->len - oq->max_single_buffer_size;
420 			while (data_len) {
421 				dma_unmap_page(oq->dev, oq->desc_ring[read_idx].buffer_ptr,
422 					       PAGE_SIZE, DMA_FROM_DEVICE);
423 				buff_info = (struct octep_rx_buffer *)
424 					    &oq->buff_info[read_idx];
425 				if (data_len < oq->buffer_size) {
426 					buff_info->len = data_len;
427 					data_len = 0;
428 				} else {
429 					buff_info->len = oq->buffer_size;
430 					data_len -= oq->buffer_size;
431 				}
432 
433 				skb_add_rx_frag(skb, shinfo->nr_frags,
434 						buff_info->page, 0,
435 						buff_info->len,
436 						buff_info->len);
437 				buff_info->page = NULL;
438 				read_idx++;
439 				desc_used++;
440 				if (read_idx == oq->max_count)
441 					read_idx = 0;
442 			}
443 		}
444 
445 		skb->dev = oq->netdev;
446 		skb->protocol =  eth_type_trans(skb, skb->dev);
447 		if (resp_hw_ext &&
448 		    resp_hw_ext->csum_verified == OCTEP_CSUM_VERIFIED)
449 			skb->ip_summed = CHECKSUM_UNNECESSARY;
450 		else
451 			skb->ip_summed = CHECKSUM_NONE;
452 		napi_gro_receive(oq->napi, skb);
453 	}
454 
455 	oq->host_read_idx = read_idx;
456 	oq->refill_count += desc_used;
457 	oq->stats.packets += pkt;
458 	oq->stats.bytes += rx_bytes;
459 
460 	return pkt;
461 }
462 
463 /**
464  * octep_oq_process_rx() - Process Rx queue.
465  *
466  * @oq: Octeon Rx queue data structure.
467  * @budget: max number of packets can be processed in one invocation.
468  *
469  * Check for newly received packets and process them.
470  * Keeps checking for new packets until budget is used or no new packets seen.
471  *
472  * Return: number of packets processed.
473  */
474 int octep_oq_process_rx(struct octep_oq *oq, int budget)
475 {
476 	u32 pkts_available, pkts_processed, total_pkts_processed;
477 	struct octep_device *oct = oq->octep_dev;
478 
479 	pkts_available = 0;
480 	pkts_processed = 0;
481 	total_pkts_processed = 0;
482 	while (total_pkts_processed < budget) {
483 		 /* update pending count only when current one exhausted */
484 		if (oq->pkts_pending == 0)
485 			octep_oq_check_hw_for_pkts(oct, oq);
486 		pkts_available = min(budget - total_pkts_processed,
487 				     oq->pkts_pending);
488 		if (!pkts_available)
489 			break;
490 
491 		pkts_processed = __octep_oq_process_rx(oct, oq,
492 						       pkts_available);
493 		oq->pkts_pending -= pkts_processed;
494 		total_pkts_processed += pkts_processed;
495 	}
496 
497 	if (oq->refill_count >= oq->refill_threshold) {
498 		u32 desc_refilled = octep_oq_refill(oct, oq);
499 
500 		/* flush pending writes before updating credits */
501 		wmb();
502 		writel(desc_refilled, oq->pkts_credit_reg);
503 	}
504 
505 	return total_pkts_processed;
506 }
507