1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * mtu3_qmu.c - Queue Management Unit driver for device controller
4  *
5  * Copyright (C) 2016 MediaTek Inc.
6  *
7  * Author: Chunfeng Yun <chunfeng.yun@mediatek.com>
8  */
9 
10 /*
11  * Queue Management Unit (QMU) is designed to unload SW effort
12  * to serve DMA interrupts.
13  * By preparing General Purpose Descriptor (GPD) and Buffer Descriptor (BD),
14  * SW links data buffers and triggers QMU to send / receive data to
15  * host / from device at a time.
16  * And now only GPD is supported.
17  *
18  * For more detailed information, please refer to QMU Programming Guide
19  */
20 
21 #include <asm/cache.h>
22 #include <cpu_func.h>
23 #include <linux/iopoll.h>
24 #include <linux/types.h>
25 
26 #include "mtu3.h"
27 
28 #define QMU_CHECKSUM_LEN	16
29 
30 #define GPD_FLAGS_HWO	BIT(0)
31 #define GPD_FLAGS_BDP	BIT(1)
32 #define GPD_FLAGS_BPS	BIT(2)
33 #define GPD_FLAGS_IOC	BIT(7)
34 
35 #define GPD_EXT_FLAG_ZLP	BIT(5)
36 
37 #define DCACHELINE_SIZE		CONFIG_SYS_CACHELINE_SIZE
38 
mtu3_flush_cache(uintptr_t addr,u32 len)39 void mtu3_flush_cache(uintptr_t addr, u32 len)
40 {
41 	WARN_ON(!(void *)addr || len == 0);
42 
43 	flush_dcache_range(addr & ~(DCACHELINE_SIZE - 1),
44 			   ALIGN(addr + len, DCACHELINE_SIZE));
45 }
46 
mtu3_inval_cache(uintptr_t addr,u32 len)47 void mtu3_inval_cache(uintptr_t addr, u32 len)
48 {
49 	WARN_ON(!(void *)addr || len == 0);
50 
51 	invalidate_dcache_range(addr & ~(DCACHELINE_SIZE - 1),
52 				ALIGN(addr + len, DCACHELINE_SIZE));
53 }
54 
gpd_dma_to_virt(struct mtu3_gpd_ring * ring,dma_addr_t dma_addr)55 static struct qmu_gpd *gpd_dma_to_virt(struct mtu3_gpd_ring *ring,
56 				       dma_addr_t dma_addr)
57 {
58 	dma_addr_t dma_base = ring->dma;
59 	struct qmu_gpd *gpd_head = ring->start;
60 	u32 offset = (dma_addr - dma_base) / sizeof(*gpd_head);
61 
62 	if (offset >= MAX_GPD_NUM)
63 		return NULL;
64 
65 	return gpd_head + offset;
66 }
67 
gpd_virt_to_dma(struct mtu3_gpd_ring * ring,struct qmu_gpd * gpd)68 static dma_addr_t gpd_virt_to_dma(struct mtu3_gpd_ring *ring,
69 				  struct qmu_gpd *gpd)
70 {
71 	dma_addr_t dma_base = ring->dma;
72 	struct qmu_gpd *gpd_head = ring->start;
73 	u32 offset;
74 
75 	offset = gpd - gpd_head;
76 	if (offset >= MAX_GPD_NUM)
77 		return 0;
78 
79 	return dma_base + (offset * sizeof(*gpd));
80 }
81 
gpd_ring_init(struct mtu3_gpd_ring * ring,struct qmu_gpd * gpd)82 static void gpd_ring_init(struct mtu3_gpd_ring *ring, struct qmu_gpd *gpd)
83 {
84 	ring->start = gpd;
85 	ring->enqueue = gpd;
86 	ring->dequeue = gpd;
87 	ring->end = gpd + MAX_GPD_NUM - 1;
88 }
89 
reset_gpd_list(struct mtu3_ep * mep)90 static void reset_gpd_list(struct mtu3_ep *mep)
91 {
92 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;
93 	struct qmu_gpd *gpd = ring->start;
94 
95 	if (gpd) {
96 		gpd->flag &= ~GPD_FLAGS_HWO;
97 		gpd_ring_init(ring, gpd);
98 		mtu3_flush_cache((uintptr_t)gpd, sizeof(*gpd));
99 	}
100 }
101 
mtu3_gpd_ring_alloc(struct mtu3_ep * mep)102 int mtu3_gpd_ring_alloc(struct mtu3_ep *mep)
103 {
104 	struct qmu_gpd *gpd;
105 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;
106 
107 	/* software own all gpds as default */
108 	gpd = memalign(DCACHELINE_SIZE, QMU_GPD_RING_SIZE);
109 	if (!gpd)
110 		return -ENOMEM;
111 
112 	memset(gpd, 0, QMU_GPD_RING_SIZE);
113 	ring->dma = (dma_addr_t)gpd;
114 	gpd_ring_init(ring, gpd);
115 
116 	return 0;
117 }
118 
mtu3_gpd_ring_free(struct mtu3_ep * mep)119 void mtu3_gpd_ring_free(struct mtu3_ep *mep)
120 {
121 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;
122 
123 	kfree(ring->start);
124 	memset(ring, 0, sizeof(*ring));
125 }
126 
mtu3_qmu_resume(struct mtu3_ep * mep)127 void mtu3_qmu_resume(struct mtu3_ep *mep)
128 {
129 	struct mtu3 *mtu = mep->mtu;
130 	void __iomem *mbase = mtu->mac_base;
131 	int epnum = mep->epnum;
132 	u32 offset;
133 
134 	offset = mep->is_in ? USB_QMU_TQCSR(epnum) : USB_QMU_RQCSR(epnum);
135 
136 	mtu3_writel(mbase, offset, QMU_Q_RESUME);
137 	if (!(mtu3_readl(mbase, offset) & QMU_Q_ACTIVE))
138 		mtu3_writel(mbase, offset, QMU_Q_RESUME);
139 }
140 
advance_enq_gpd(struct mtu3_gpd_ring * ring)141 static struct qmu_gpd *advance_enq_gpd(struct mtu3_gpd_ring *ring)
142 {
143 	if (ring->enqueue < ring->end)
144 		ring->enqueue++;
145 	else
146 		ring->enqueue = ring->start;
147 
148 	return ring->enqueue;
149 }
150 
advance_deq_gpd(struct mtu3_gpd_ring * ring)151 static struct qmu_gpd *advance_deq_gpd(struct mtu3_gpd_ring *ring)
152 {
153 	if (ring->dequeue < ring->end)
154 		ring->dequeue++;
155 	else
156 		ring->dequeue = ring->start;
157 
158 	return ring->dequeue;
159 }
160 
161 /* check if a ring is emtpy */
gpd_ring_empty(struct mtu3_gpd_ring * ring)162 static int gpd_ring_empty(struct mtu3_gpd_ring *ring)
163 {
164 	struct qmu_gpd *enq = ring->enqueue;
165 	struct qmu_gpd *next;
166 
167 	if (ring->enqueue < ring->end)
168 		next = enq + 1;
169 	else
170 		next = ring->start;
171 
172 	/* one gpd is reserved to simplify gpd preparation */
173 	return next == ring->dequeue;
174 }
175 
mtu3_prepare_transfer(struct mtu3_ep * mep)176 int mtu3_prepare_transfer(struct mtu3_ep *mep)
177 {
178 	return gpd_ring_empty(&mep->gpd_ring);
179 }
180 
mtu3_prepare_tx_gpd(struct mtu3_ep * mep,struct mtu3_request * mreq)181 static int mtu3_prepare_tx_gpd(struct mtu3_ep *mep, struct mtu3_request *mreq)
182 {
183 	struct qmu_gpd *enq;
184 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;
185 	struct qmu_gpd *gpd = ring->enqueue;
186 	struct usb_request *req = &mreq->request;
187 
188 	/* set all fields to zero as default value */
189 	memset(gpd, 0, sizeof(*gpd));
190 
191 	gpd->buffer = cpu_to_le32((u32)req->dma);
192 	gpd->buf_len = cpu_to_le16(req->length);
193 
194 	/* get the next GPD */
195 	enq = advance_enq_gpd(ring);
196 	dev_dbg(mep->mtu->dev, "TX-EP%d queue gpd=%p, enq=%p\n",
197 		mep->epnum, gpd, enq);
198 
199 	enq->flag &= ~GPD_FLAGS_HWO;
200 	gpd->next_gpd = cpu_to_le32((u32)gpd_virt_to_dma(ring, enq));
201 	mtu3_flush_cache((uintptr_t)enq, sizeof(*gpd));
202 
203 	if (req->zero)
204 		gpd->ext_flag |= GPD_EXT_FLAG_ZLP;
205 
206 	gpd->flag |= GPD_FLAGS_IOC | GPD_FLAGS_HWO;
207 
208 	mreq->gpd = gpd;
209 
210 	if (req->length)
211 		mtu3_flush_cache((uintptr_t)req->buf, req->length);
212 
213 	mtu3_flush_cache((uintptr_t)gpd, sizeof(*gpd));
214 
215 	return 0;
216 }
217 
mtu3_prepare_rx_gpd(struct mtu3_ep * mep,struct mtu3_request * mreq)218 static int mtu3_prepare_rx_gpd(struct mtu3_ep *mep, struct mtu3_request *mreq)
219 {
220 	struct qmu_gpd *enq;
221 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;
222 	struct qmu_gpd *gpd = ring->enqueue;
223 	struct usb_request *req = &mreq->request;
224 
225 	/* set all fields to zero as default value */
226 	memset(gpd, 0, sizeof(*gpd));
227 
228 	gpd->buffer = cpu_to_le32((u32)req->dma);
229 	gpd->data_buf_len = cpu_to_le16(req->length);
230 
231 	/* get the next GPD */
232 	enq = advance_enq_gpd(ring);
233 	dev_dbg(mep->mtu->dev, "RX-EP%d queue gpd=%p, enq=%p\n",
234 		mep->epnum, gpd, enq);
235 
236 	enq->flag &= ~GPD_FLAGS_HWO;
237 	gpd->next_gpd = cpu_to_le32((u32)gpd_virt_to_dma(ring, enq));
238 	mtu3_flush_cache((uintptr_t)enq, sizeof(*gpd));
239 
240 	gpd->flag |= GPD_FLAGS_IOC | GPD_FLAGS_HWO;
241 
242 	mreq->gpd = gpd;
243 
244 	mtu3_inval_cache((uintptr_t)req->buf, req->length);
245 	mtu3_flush_cache((uintptr_t)gpd, sizeof(*gpd));
246 
247 	return 0;
248 }
249 
mtu3_insert_gpd(struct mtu3_ep * mep,struct mtu3_request * mreq)250 void mtu3_insert_gpd(struct mtu3_ep *mep, struct mtu3_request *mreq)
251 {
252 	if (mep->is_in)
253 		mtu3_prepare_tx_gpd(mep, mreq);
254 	else
255 		mtu3_prepare_rx_gpd(mep, mreq);
256 }
257 
mtu3_qmu_start(struct mtu3_ep * mep)258 int mtu3_qmu_start(struct mtu3_ep *mep)
259 {
260 	struct mtu3 *mtu = mep->mtu;
261 	void __iomem *mbase = mtu->mac_base;
262 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;
263 	u8 epnum = mep->epnum;
264 
265 	if (mep->is_in) {
266 		/* set QMU start address */
267 		mtu3_writel(mbase, USB_QMU_TQSAR(epnum), ring->dma);
268 		mtu3_setbits(mbase, MU3D_EP_TXCR0(epnum), TX_DMAREQEN);
269 		/* send zero length packet according to ZLP flag in GPD */
270 		mtu3_setbits(mbase, U3D_QCR1, QMU_TX_ZLP(epnum));
271 		mtu3_writel(mbase, U3D_TQERRIESR0,
272 			    QMU_TX_LEN_ERR(epnum) | QMU_TX_CS_ERR(epnum));
273 
274 		if (mtu3_readl(mbase, USB_QMU_TQCSR(epnum)) & QMU_Q_ACTIVE) {
275 			dev_warn(mtu->dev, "Tx %d Active Now!\n", epnum);
276 			return 0;
277 		}
278 		mtu3_writel(mbase, USB_QMU_TQCSR(epnum), QMU_Q_START);
279 
280 	} else {
281 		mtu3_writel(mbase, USB_QMU_RQSAR(epnum), ring->dma);
282 		mtu3_setbits(mbase, MU3D_EP_RXCR0(epnum), RX_DMAREQEN);
283 		/* don't expect ZLP */
284 		mtu3_clrbits(mbase, U3D_QCR3, QMU_RX_ZLP(epnum));
285 		/* move to next GPD when receive ZLP */
286 		mtu3_setbits(mbase, U3D_QCR3, QMU_RX_COZ(epnum));
287 		mtu3_writel(mbase, U3D_RQERRIESR0,
288 			    QMU_RX_LEN_ERR(epnum) | QMU_RX_CS_ERR(epnum));
289 		mtu3_writel(mbase, U3D_RQERRIESR1, QMU_RX_ZLP_ERR(epnum));
290 
291 		if (mtu3_readl(mbase, USB_QMU_RQCSR(epnum)) & QMU_Q_ACTIVE) {
292 			dev_warn(mtu->dev, "Rx %d Active Now!\n", epnum);
293 			return 0;
294 		}
295 		mtu3_writel(mbase, USB_QMU_RQCSR(epnum), QMU_Q_START);
296 	}
297 
298 	return 0;
299 }
300 
301 /* may called in atomic context */
mtu3_qmu_stop(struct mtu3_ep * mep)302 void mtu3_qmu_stop(struct mtu3_ep *mep)
303 {
304 	struct mtu3 *mtu = mep->mtu;
305 	void __iomem *mbase = mtu->mac_base;
306 	int epnum = mep->epnum;
307 	u32 value = 0;
308 	u32 qcsr;
309 	int ret;
310 
311 	qcsr = mep->is_in ? USB_QMU_TQCSR(epnum) : USB_QMU_RQCSR(epnum);
312 
313 	if (!(mtu3_readl(mbase, qcsr) & QMU_Q_ACTIVE)) {
314 		dev_dbg(mtu->dev, "%s's qmu is inactive now!\n", mep->name);
315 		return;
316 	}
317 	mtu3_writel(mbase, qcsr, QMU_Q_STOP);
318 
319 	ret = readl_poll_timeout(mbase + qcsr, value,
320 				 !(value & QMU_Q_ACTIVE), 1000);
321 	if (ret) {
322 		dev_err(mtu->dev, "stop %s's qmu failed\n", mep->name);
323 		return;
324 	}
325 
326 	dev_dbg(mtu->dev, "%s's qmu stop now!\n", mep->name);
327 }
328 
mtu3_qmu_flush(struct mtu3_ep * mep)329 void mtu3_qmu_flush(struct mtu3_ep *mep)
330 {
331 	dev_dbg(mep->mtu->dev, "%s flush QMU %s\n", __func__,
332 		((mep->is_in) ? "TX" : "RX"));
333 
334 	/*Stop QMU */
335 	mtu3_qmu_stop(mep);
336 	reset_gpd_list(mep);
337 }
338 
339 /*
340  * NOTE: request list maybe is already empty as following case:
341  * queue_tx --> qmu_interrupt(clear interrupt pending, schedule tasklet)-->
342  * queue_tx --> process_tasklet(meanwhile, the second one is transferred,
343  * tasklet process both of them)-->qmu_interrupt for second one.
344  * To avoid upper case, put qmu_done_tx in ISR directly to process it.
345  */
qmu_done_tx(struct mtu3 * mtu,u8 epnum)346 static void qmu_done_tx(struct mtu3 *mtu, u8 epnum)
347 {
348 	struct mtu3_ep *mep = mtu->in_eps + epnum;
349 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;
350 	void __iomem *mbase = mtu->mac_base;
351 	struct qmu_gpd *gpd = ring->dequeue;
352 	struct qmu_gpd *gpd_current = NULL;
353 	struct usb_request *req = NULL;
354 	struct mtu3_request *mreq;
355 	dma_addr_t cur_gpd_dma;
356 
357 	/*transfer phy address got from QMU register to virtual address */
358 	cur_gpd_dma = mtu3_readl(mbase, USB_QMU_TQCPR(epnum));
359 	gpd_current = gpd_dma_to_virt(ring, cur_gpd_dma);
360 	mtu3_inval_cache((uintptr_t)gpd, sizeof(*gpd));
361 
362 	dev_dbg(mtu->dev, "%s EP%d, last=%p, current=%p, enq=%p\n",
363 		__func__, epnum, gpd, gpd_current, ring->enqueue);
364 
365 	while (gpd != gpd_current && !(gpd->flag & GPD_FLAGS_HWO)) {
366 		mreq = next_request(mep);
367 
368 		if (!mreq || mreq->gpd != gpd) {
369 			dev_err(mtu->dev, "no correct TX req is found\n");
370 			break;
371 		}
372 
373 		req = &mreq->request;
374 		req->actual = le16_to_cpu(gpd->buf_len);
375 		mtu3_req_complete(mep, req, 0);
376 
377 		gpd = advance_deq_gpd(ring);
378 		mtu3_inval_cache((uintptr_t)gpd, sizeof(*gpd));
379 	}
380 
381 	dev_dbg(mtu->dev, "%s EP%d, deq=%p, enq=%p, complete\n",
382 		__func__, epnum, ring->dequeue, ring->enqueue);
383 }
384 
qmu_done_rx(struct mtu3 * mtu,u8 epnum)385 static void qmu_done_rx(struct mtu3 *mtu, u8 epnum)
386 {
387 	struct mtu3_ep *mep = mtu->out_eps + epnum;
388 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;
389 	void __iomem *mbase = mtu->mac_base;
390 	struct qmu_gpd *gpd = ring->dequeue;
391 	struct qmu_gpd *gpd_current = NULL;
392 	struct usb_request *req = NULL;
393 	struct mtu3_request *mreq;
394 	dma_addr_t cur_gpd_dma;
395 
396 	cur_gpd_dma = mtu3_readl(mbase, USB_QMU_RQCPR(epnum));
397 	gpd_current = gpd_dma_to_virt(ring, cur_gpd_dma);
398 	mtu3_inval_cache((uintptr_t)gpd, sizeof(*gpd));
399 
400 	dev_dbg(mtu->dev, "%s EP%d, last=%p, current=%p, enq=%p\n",
401 		__func__, epnum, gpd, gpd_current, ring->enqueue);
402 
403 	while (gpd != gpd_current && !(gpd->flag & GPD_FLAGS_HWO)) {
404 		mreq = next_request(mep);
405 
406 		if (!mreq || mreq->gpd != gpd) {
407 			dev_err(mtu->dev, "no correct RX req is found\n");
408 			break;
409 		}
410 		req = &mreq->request;
411 
412 		req->actual = le16_to_cpu(gpd->buf_len);
413 		mtu3_req_complete(mep, req, 0);
414 
415 		gpd = advance_deq_gpd(ring);
416 		mtu3_inval_cache((uintptr_t)gpd, sizeof(*gpd));
417 	}
418 
419 	dev_dbg(mtu->dev, "%s EP%d, deq=%p, enq=%p, complete\n",
420 		__func__, epnum, ring->dequeue, ring->enqueue);
421 }
422 
qmu_done_isr(struct mtu3 * mtu,u32 done_status)423 static void qmu_done_isr(struct mtu3 *mtu, u32 done_status)
424 {
425 	int i;
426 
427 	for (i = 1; i < mtu->num_eps; i++) {
428 		if (done_status & QMU_RX_DONE_INT(i))
429 			qmu_done_rx(mtu, i);
430 		if (done_status & QMU_TX_DONE_INT(i))
431 			qmu_done_tx(mtu, i);
432 	}
433 }
434 
qmu_exception_isr(struct mtu3 * mtu,u32 qmu_status)435 static void qmu_exception_isr(struct mtu3 *mtu, u32 qmu_status)
436 {
437 	void __iomem *mbase = mtu->mac_base;
438 	u32 errval;
439 	int i;
440 
441 	if ((qmu_status & RXQ_CSERR_INT) || (qmu_status & RXQ_LENERR_INT)) {
442 		errval = mtu3_readl(mbase, U3D_RQERRIR0);
443 		for (i = 1; i < mtu->num_eps; i++) {
444 			if (errval & QMU_RX_CS_ERR(i))
445 				dev_err(mtu->dev, "Rx %d CS error!\n", i);
446 
447 			if (errval & QMU_RX_LEN_ERR(i))
448 				dev_err(mtu->dev, "RX %d Length error\n", i);
449 		}
450 		mtu3_writel(mbase, U3D_RQERRIR0, errval);
451 	}
452 
453 	if (qmu_status & RXQ_ZLPERR_INT) {
454 		errval = mtu3_readl(mbase, U3D_RQERRIR1);
455 		for (i = 1; i < mtu->num_eps; i++) {
456 			if (errval & QMU_RX_ZLP_ERR(i))
457 				dev_dbg(mtu->dev, "RX EP%d Recv ZLP\n", i);
458 		}
459 		mtu3_writel(mbase, U3D_RQERRIR1, errval);
460 	}
461 
462 	if ((qmu_status & TXQ_CSERR_INT) || (qmu_status & TXQ_LENERR_INT)) {
463 		errval = mtu3_readl(mbase, U3D_TQERRIR0);
464 		for (i = 1; i < mtu->num_eps; i++) {
465 			if (errval & QMU_TX_CS_ERR(i))
466 				dev_err(mtu->dev, "Tx %d checksum error!\n", i);
467 
468 			if (errval & QMU_TX_LEN_ERR(i))
469 				dev_err(mtu->dev, "Tx %d zlp error!\n", i);
470 		}
471 		mtu3_writel(mbase, U3D_TQERRIR0, errval);
472 	}
473 }
474 
mtu3_qmu_isr(struct mtu3 * mtu)475 irqreturn_t mtu3_qmu_isr(struct mtu3 *mtu)
476 {
477 	void __iomem *mbase = mtu->mac_base;
478 	u32 qmu_status;
479 	u32 qmu_done_status;
480 
481 	/* U3D_QISAR1 is read update */
482 	qmu_status = mtu3_readl(mbase, U3D_QISAR1);
483 	qmu_status &= mtu3_readl(mbase, U3D_QIER1);
484 
485 	qmu_done_status = mtu3_readl(mbase, U3D_QISAR0);
486 	qmu_done_status &= mtu3_readl(mbase, U3D_QIER0);
487 	mtu3_writel(mbase, U3D_QISAR0, qmu_done_status); /* W1C */
488 	dev_dbg(mtu->dev, "=== QMUdone[tx=%x, rx=%x] QMUexp[%x] ===\n",
489 		(qmu_done_status & 0xFFFF), qmu_done_status >> 16,
490 		qmu_status);
491 
492 	if (qmu_done_status)
493 		qmu_done_isr(mtu, qmu_done_status);
494 
495 	if (qmu_status)
496 		qmu_exception_isr(mtu, qmu_status);
497 
498 	return IRQ_HANDLED;
499 }
500 
mtu3_qmu_init(struct mtu3 * mtu)501 void mtu3_qmu_init(struct mtu3 *mtu)
502 {
503 	compiletime_assert(QMU_GPD_SIZE == 16, "QMU_GPD size SHOULD be 16B");
504 }
505 
mtu3_qmu_exit(struct mtu3 * mtu)506 void mtu3_qmu_exit(struct mtu3 *mtu)
507 {
508 }
509