1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
4 * Synopsys DesignWare eDMA core driver
5 *
6 * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com>
7 */
8
9 #include <linux/module.h>
10 #include <linux/device.h>
11 #include <linux/kernel.h>
12 #include <linux/pm_runtime.h>
13 #include <linux/dmaengine.h>
14 #include <linux/err.h>
15 #include <linux/interrupt.h>
16 #include <linux/irq.h>
17 #include <linux/dma/edma.h>
18 #include <linux/dma-mapping.h>
19
20 #include "dw-edma-core.h"
21 #include "dw-edma-v0-core.h"
22 #include "../dmaengine.h"
23 #include "../virt-dma.h"
24
25 static inline
dchan2dev(struct dma_chan * dchan)26 struct device *dchan2dev(struct dma_chan *dchan)
27 {
28 return &dchan->dev->device;
29 }
30
31 static inline
chan2dev(struct dw_edma_chan * chan)32 struct device *chan2dev(struct dw_edma_chan *chan)
33 {
34 return &chan->vc.chan.dev->device;
35 }
36
37 static inline
vd2dw_edma_desc(struct virt_dma_desc * vd)38 struct dw_edma_desc *vd2dw_edma_desc(struct virt_dma_desc *vd)
39 {
40 return container_of(vd, struct dw_edma_desc, vd);
41 }
42
dw_edma_alloc_burst(struct dw_edma_chunk * chunk)43 static struct dw_edma_burst *dw_edma_alloc_burst(struct dw_edma_chunk *chunk)
44 {
45 struct dw_edma_burst *burst;
46
47 burst = kzalloc(sizeof(*burst), GFP_NOWAIT);
48 if (unlikely(!burst))
49 return NULL;
50
51 INIT_LIST_HEAD(&burst->list);
52 if (chunk->burst) {
53 /* Create and add new element into the linked list */
54 chunk->bursts_alloc++;
55 list_add_tail(&burst->list, &chunk->burst->list);
56 } else {
57 /* List head */
58 chunk->bursts_alloc = 0;
59 chunk->burst = burst;
60 }
61
62 return burst;
63 }
64
dw_edma_alloc_chunk(struct dw_edma_desc * desc)65 static struct dw_edma_chunk *dw_edma_alloc_chunk(struct dw_edma_desc *desc)
66 {
67 struct dw_edma_chan *chan = desc->chan;
68 struct dw_edma *dw = chan->chip->dw;
69 struct dw_edma_chunk *chunk;
70
71 chunk = kzalloc(sizeof(*chunk), GFP_NOWAIT);
72 if (unlikely(!chunk))
73 return NULL;
74
75 INIT_LIST_HEAD(&chunk->list);
76 chunk->chan = chan;
77 /* Toggling change bit (CB) in each chunk, this is a mechanism to
78 * inform the eDMA HW block that this is a new linked list ready
79 * to be consumed.
80 * - Odd chunks originate CB equal to 0
81 * - Even chunks originate CB equal to 1
82 */
83 chunk->cb = !(desc->chunks_alloc % 2);
84 if (chan->dir == EDMA_DIR_WRITE) {
85 chunk->ll_region.paddr = dw->ll_region_wr[chan->id].paddr;
86 chunk->ll_region.vaddr = dw->ll_region_wr[chan->id].vaddr;
87 } else {
88 chunk->ll_region.paddr = dw->ll_region_rd[chan->id].paddr;
89 chunk->ll_region.vaddr = dw->ll_region_rd[chan->id].vaddr;
90 }
91
92 if (desc->chunk) {
93 /* Create and add new element into the linked list */
94 if (!dw_edma_alloc_burst(chunk)) {
95 kfree(chunk);
96 return NULL;
97 }
98 desc->chunks_alloc++;
99 list_add_tail(&chunk->list, &desc->chunk->list);
100 } else {
101 /* List head */
102 chunk->burst = NULL;
103 desc->chunks_alloc = 0;
104 desc->chunk = chunk;
105 }
106
107 return chunk;
108 }
109
dw_edma_alloc_desc(struct dw_edma_chan * chan)110 static struct dw_edma_desc *dw_edma_alloc_desc(struct dw_edma_chan *chan)
111 {
112 struct dw_edma_desc *desc;
113
114 desc = kzalloc(sizeof(*desc), GFP_NOWAIT);
115 if (unlikely(!desc))
116 return NULL;
117
118 desc->chan = chan;
119 if (!dw_edma_alloc_chunk(desc)) {
120 kfree(desc);
121 return NULL;
122 }
123
124 return desc;
125 }
126
dw_edma_free_burst(struct dw_edma_chunk * chunk)127 static void dw_edma_free_burst(struct dw_edma_chunk *chunk)
128 {
129 struct dw_edma_burst *child, *_next;
130
131 /* Remove all the list elements */
132 list_for_each_entry_safe(child, _next, &chunk->burst->list, list) {
133 list_del(&child->list);
134 kfree(child);
135 chunk->bursts_alloc--;
136 }
137
138 /* Remove the list head */
139 kfree(child);
140 chunk->burst = NULL;
141 }
142
dw_edma_free_chunk(struct dw_edma_desc * desc)143 static void dw_edma_free_chunk(struct dw_edma_desc *desc)
144 {
145 struct dw_edma_chunk *child, *_next;
146
147 if (!desc->chunk)
148 return;
149
150 /* Remove all the list elements */
151 list_for_each_entry_safe(child, _next, &desc->chunk->list, list) {
152 dw_edma_free_burst(child);
153 list_del(&child->list);
154 kfree(child);
155 desc->chunks_alloc--;
156 }
157
158 /* Remove the list head */
159 kfree(child);
160 desc->chunk = NULL;
161 }
162
dw_edma_free_desc(struct dw_edma_desc * desc)163 static void dw_edma_free_desc(struct dw_edma_desc *desc)
164 {
165 dw_edma_free_chunk(desc);
166 kfree(desc);
167 }
168
vchan_free_desc(struct virt_dma_desc * vdesc)169 static void vchan_free_desc(struct virt_dma_desc *vdesc)
170 {
171 dw_edma_free_desc(vd2dw_edma_desc(vdesc));
172 }
173
dw_edma_start_transfer(struct dw_edma_chan * chan)174 static void dw_edma_start_transfer(struct dw_edma_chan *chan)
175 {
176 struct dw_edma_chunk *child;
177 struct dw_edma_desc *desc;
178 struct virt_dma_desc *vd;
179
180 vd = vchan_next_desc(&chan->vc);
181 if (!vd)
182 return;
183
184 desc = vd2dw_edma_desc(vd);
185 if (!desc)
186 return;
187
188 child = list_first_entry_or_null(&desc->chunk->list,
189 struct dw_edma_chunk, list);
190 if (!child)
191 return;
192
193 dw_edma_v0_core_start(child, !desc->xfer_sz);
194 desc->xfer_sz += child->ll_region.sz;
195 dw_edma_free_burst(child);
196 list_del(&child->list);
197 kfree(child);
198 desc->chunks_alloc--;
199 }
200
dw_edma_device_config(struct dma_chan * dchan,struct dma_slave_config * config)201 static int dw_edma_device_config(struct dma_chan *dchan,
202 struct dma_slave_config *config)
203 {
204 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
205
206 memcpy(&chan->config, config, sizeof(*config));
207 chan->configured = true;
208
209 return 0;
210 }
211
dw_edma_device_pause(struct dma_chan * dchan)212 static int dw_edma_device_pause(struct dma_chan *dchan)
213 {
214 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
215 int err = 0;
216
217 if (!chan->configured)
218 err = -EPERM;
219 else if (chan->status != EDMA_ST_BUSY)
220 err = -EPERM;
221 else if (chan->request != EDMA_REQ_NONE)
222 err = -EPERM;
223 else
224 chan->request = EDMA_REQ_PAUSE;
225
226 return err;
227 }
228
dw_edma_device_resume(struct dma_chan * dchan)229 static int dw_edma_device_resume(struct dma_chan *dchan)
230 {
231 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
232 int err = 0;
233
234 if (!chan->configured) {
235 err = -EPERM;
236 } else if (chan->status != EDMA_ST_PAUSE) {
237 err = -EPERM;
238 } else if (chan->request != EDMA_REQ_NONE) {
239 err = -EPERM;
240 } else {
241 chan->status = EDMA_ST_BUSY;
242 dw_edma_start_transfer(chan);
243 }
244
245 return err;
246 }
247
dw_edma_device_terminate_all(struct dma_chan * dchan)248 static int dw_edma_device_terminate_all(struct dma_chan *dchan)
249 {
250 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
251 int err = 0;
252 LIST_HEAD(head);
253
254 if (!chan->configured) {
255 /* Do nothing */
256 } else if (chan->status == EDMA_ST_PAUSE) {
257 chan->status = EDMA_ST_IDLE;
258 chan->configured = false;
259 } else if (chan->status == EDMA_ST_IDLE) {
260 chan->configured = false;
261 } else if (dw_edma_v0_core_ch_status(chan) == DMA_COMPLETE) {
262 /*
263 * The channel is in a false BUSY state, probably didn't
264 * receive or lost an interrupt
265 */
266 chan->status = EDMA_ST_IDLE;
267 chan->configured = false;
268 } else if (chan->request > EDMA_REQ_PAUSE) {
269 err = -EPERM;
270 } else {
271 chan->request = EDMA_REQ_STOP;
272 }
273
274 return err;
275 }
276
dw_edma_device_issue_pending(struct dma_chan * dchan)277 static void dw_edma_device_issue_pending(struct dma_chan *dchan)
278 {
279 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
280 unsigned long flags;
281
282 spin_lock_irqsave(&chan->vc.lock, flags);
283 if (chan->configured && chan->request == EDMA_REQ_NONE &&
284 chan->status == EDMA_ST_IDLE && vchan_issue_pending(&chan->vc)) {
285 chan->status = EDMA_ST_BUSY;
286 dw_edma_start_transfer(chan);
287 }
288 spin_unlock_irqrestore(&chan->vc.lock, flags);
289 }
290
291 static enum dma_status
dw_edma_device_tx_status(struct dma_chan * dchan,dma_cookie_t cookie,struct dma_tx_state * txstate)292 dw_edma_device_tx_status(struct dma_chan *dchan, dma_cookie_t cookie,
293 struct dma_tx_state *txstate)
294 {
295 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
296 struct dw_edma_desc *desc;
297 struct virt_dma_desc *vd;
298 unsigned long flags;
299 enum dma_status ret;
300 u32 residue = 0;
301
302 ret = dma_cookie_status(dchan, cookie, txstate);
303 if (ret == DMA_COMPLETE)
304 return ret;
305
306 if (ret == DMA_IN_PROGRESS && chan->status == EDMA_ST_PAUSE)
307 ret = DMA_PAUSED;
308
309 if (!txstate)
310 goto ret_residue;
311
312 spin_lock_irqsave(&chan->vc.lock, flags);
313 vd = vchan_find_desc(&chan->vc, cookie);
314 if (vd) {
315 desc = vd2dw_edma_desc(vd);
316 if (desc)
317 residue = desc->alloc_sz - desc->xfer_sz;
318 }
319 spin_unlock_irqrestore(&chan->vc.lock, flags);
320
321 ret_residue:
322 dma_set_residue(txstate, residue);
323
324 return ret;
325 }
326
327 static struct dma_async_tx_descriptor *
dw_edma_device_transfer(struct dw_edma_transfer * xfer)328 dw_edma_device_transfer(struct dw_edma_transfer *xfer)
329 {
330 struct dw_edma_chan *chan = dchan2dw_edma_chan(xfer->dchan);
331 enum dma_transfer_direction dir = xfer->direction;
332 phys_addr_t src_addr, dst_addr;
333 struct scatterlist *sg = NULL;
334 struct dw_edma_chunk *chunk;
335 struct dw_edma_burst *burst;
336 struct dw_edma_desc *desc;
337 u32 cnt = 0;
338 int i;
339
340 if (!chan->configured)
341 return NULL;
342
343 switch (chan->config.direction) {
344 case DMA_DEV_TO_MEM: /* local DMA */
345 if (dir == DMA_DEV_TO_MEM && chan->dir == EDMA_DIR_READ)
346 break;
347 return NULL;
348 case DMA_MEM_TO_DEV: /* local DMA */
349 if (dir == DMA_MEM_TO_DEV && chan->dir == EDMA_DIR_WRITE)
350 break;
351 return NULL;
352 default: /* remote DMA */
353 if (dir == DMA_MEM_TO_DEV && chan->dir == EDMA_DIR_READ)
354 break;
355 if (dir == DMA_DEV_TO_MEM && chan->dir == EDMA_DIR_WRITE)
356 break;
357 return NULL;
358 }
359
360 if (xfer->type == EDMA_XFER_CYCLIC) {
361 if (!xfer->xfer.cyclic.len || !xfer->xfer.cyclic.cnt)
362 return NULL;
363 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
364 if (xfer->xfer.sg.len < 1)
365 return NULL;
366 } else if (xfer->type == EDMA_XFER_INTERLEAVED) {
367 if (!xfer->xfer.il->numf)
368 return NULL;
369 if (xfer->xfer.il->numf > 0 && xfer->xfer.il->frame_size > 0)
370 return NULL;
371 } else {
372 return NULL;
373 }
374
375 desc = dw_edma_alloc_desc(chan);
376 if (unlikely(!desc))
377 goto err_alloc;
378
379 chunk = dw_edma_alloc_chunk(desc);
380 if (unlikely(!chunk))
381 goto err_alloc;
382
383 if (xfer->type == EDMA_XFER_INTERLEAVED) {
384 src_addr = xfer->xfer.il->src_start;
385 dst_addr = xfer->xfer.il->dst_start;
386 } else {
387 src_addr = chan->config.src_addr;
388 dst_addr = chan->config.dst_addr;
389 }
390
391 if (xfer->type == EDMA_XFER_CYCLIC) {
392 cnt = xfer->xfer.cyclic.cnt;
393 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
394 cnt = xfer->xfer.sg.len;
395 sg = xfer->xfer.sg.sgl;
396 } else if (xfer->type == EDMA_XFER_INTERLEAVED) {
397 if (xfer->xfer.il->numf > 0)
398 cnt = xfer->xfer.il->numf;
399 else
400 cnt = xfer->xfer.il->frame_size;
401 }
402
403 for (i = 0; i < cnt; i++) {
404 if (xfer->type == EDMA_XFER_SCATTER_GATHER && !sg)
405 break;
406
407 if (chunk->bursts_alloc == chan->ll_max) {
408 chunk = dw_edma_alloc_chunk(desc);
409 if (unlikely(!chunk))
410 goto err_alloc;
411 }
412
413 burst = dw_edma_alloc_burst(chunk);
414 if (unlikely(!burst))
415 goto err_alloc;
416
417 if (xfer->type == EDMA_XFER_CYCLIC)
418 burst->sz = xfer->xfer.cyclic.len;
419 else if (xfer->type == EDMA_XFER_SCATTER_GATHER)
420 burst->sz = sg_dma_len(sg);
421 else if (xfer->type == EDMA_XFER_INTERLEAVED)
422 burst->sz = xfer->xfer.il->sgl[i].size;
423
424 chunk->ll_region.sz += burst->sz;
425 desc->alloc_sz += burst->sz;
426
427 if (chan->dir == EDMA_DIR_WRITE) {
428 burst->sar = src_addr;
429 if (xfer->type == EDMA_XFER_CYCLIC) {
430 burst->dar = xfer->xfer.cyclic.paddr;
431 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
432 src_addr += sg_dma_len(sg);
433 burst->dar = sg_dma_address(sg);
434 /* Unlike the typical assumption by other
435 * drivers/IPs the peripheral memory isn't
436 * a FIFO memory, in this case, it's a
437 * linear memory and that why the source
438 * and destination addresses are increased
439 * by the same portion (data length)
440 */
441 }
442 } else {
443 burst->dar = dst_addr;
444 if (xfer->type == EDMA_XFER_CYCLIC) {
445 burst->sar = xfer->xfer.cyclic.paddr;
446 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
447 dst_addr += sg_dma_len(sg);
448 burst->sar = sg_dma_address(sg);
449 /* Unlike the typical assumption by other
450 * drivers/IPs the peripheral memory isn't
451 * a FIFO memory, in this case, it's a
452 * linear memory and that why the source
453 * and destination addresses are increased
454 * by the same portion (data length)
455 */
456 }
457 }
458
459 if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
460 sg = sg_next(sg);
461 } else if (xfer->type == EDMA_XFER_INTERLEAVED &&
462 xfer->xfer.il->frame_size > 0) {
463 struct dma_interleaved_template *il = xfer->xfer.il;
464 struct data_chunk *dc = &il->sgl[i];
465
466 if (il->src_sgl) {
467 src_addr += burst->sz;
468 src_addr += dmaengine_get_src_icg(il, dc);
469 }
470
471 if (il->dst_sgl) {
472 dst_addr += burst->sz;
473 dst_addr += dmaengine_get_dst_icg(il, dc);
474 }
475 }
476 }
477
478 return vchan_tx_prep(&chan->vc, &desc->vd, xfer->flags);
479
480 err_alloc:
481 if (desc)
482 dw_edma_free_desc(desc);
483
484 return NULL;
485 }
486
487 static struct dma_async_tx_descriptor *
dw_edma_device_prep_slave_sg(struct dma_chan * dchan,struct scatterlist * sgl,unsigned int len,enum dma_transfer_direction direction,unsigned long flags,void * context)488 dw_edma_device_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,
489 unsigned int len,
490 enum dma_transfer_direction direction,
491 unsigned long flags, void *context)
492 {
493 struct dw_edma_transfer xfer;
494
495 xfer.dchan = dchan;
496 xfer.direction = direction;
497 xfer.xfer.sg.sgl = sgl;
498 xfer.xfer.sg.len = len;
499 xfer.flags = flags;
500 xfer.type = EDMA_XFER_SCATTER_GATHER;
501
502 return dw_edma_device_transfer(&xfer);
503 }
504
505 static struct dma_async_tx_descriptor *
dw_edma_device_prep_dma_cyclic(struct dma_chan * dchan,dma_addr_t paddr,size_t len,size_t count,enum dma_transfer_direction direction,unsigned long flags)506 dw_edma_device_prep_dma_cyclic(struct dma_chan *dchan, dma_addr_t paddr,
507 size_t len, size_t count,
508 enum dma_transfer_direction direction,
509 unsigned long flags)
510 {
511 struct dw_edma_transfer xfer;
512
513 xfer.dchan = dchan;
514 xfer.direction = direction;
515 xfer.xfer.cyclic.paddr = paddr;
516 xfer.xfer.cyclic.len = len;
517 xfer.xfer.cyclic.cnt = count;
518 xfer.flags = flags;
519 xfer.type = EDMA_XFER_CYCLIC;
520
521 return dw_edma_device_transfer(&xfer);
522 }
523
524 static struct dma_async_tx_descriptor *
dw_edma_device_prep_interleaved_dma(struct dma_chan * dchan,struct dma_interleaved_template * ilt,unsigned long flags)525 dw_edma_device_prep_interleaved_dma(struct dma_chan *dchan,
526 struct dma_interleaved_template *ilt,
527 unsigned long flags)
528 {
529 struct dw_edma_transfer xfer;
530
531 xfer.dchan = dchan;
532 xfer.direction = ilt->dir;
533 xfer.xfer.il = ilt;
534 xfer.flags = flags;
535 xfer.type = EDMA_XFER_INTERLEAVED;
536
537 return dw_edma_device_transfer(&xfer);
538 }
539
dw_edma_done_interrupt(struct dw_edma_chan * chan)540 static void dw_edma_done_interrupt(struct dw_edma_chan *chan)
541 {
542 struct dw_edma_desc *desc;
543 struct virt_dma_desc *vd;
544 unsigned long flags;
545
546 dw_edma_v0_core_clear_done_int(chan);
547
548 spin_lock_irqsave(&chan->vc.lock, flags);
549 vd = vchan_next_desc(&chan->vc);
550 if (vd) {
551 switch (chan->request) {
552 case EDMA_REQ_NONE:
553 desc = vd2dw_edma_desc(vd);
554 if (desc->chunks_alloc) {
555 chan->status = EDMA_ST_BUSY;
556 dw_edma_start_transfer(chan);
557 } else {
558 list_del(&vd->node);
559 vchan_cookie_complete(vd);
560 chan->status = EDMA_ST_IDLE;
561 }
562 break;
563
564 case EDMA_REQ_STOP:
565 list_del(&vd->node);
566 vchan_cookie_complete(vd);
567 chan->request = EDMA_REQ_NONE;
568 chan->status = EDMA_ST_IDLE;
569 break;
570
571 case EDMA_REQ_PAUSE:
572 chan->request = EDMA_REQ_NONE;
573 chan->status = EDMA_ST_PAUSE;
574 break;
575
576 default:
577 break;
578 }
579 }
580 spin_unlock_irqrestore(&chan->vc.lock, flags);
581 }
582
dw_edma_abort_interrupt(struct dw_edma_chan * chan)583 static void dw_edma_abort_interrupt(struct dw_edma_chan *chan)
584 {
585 struct virt_dma_desc *vd;
586 unsigned long flags;
587
588 dw_edma_v0_core_clear_abort_int(chan);
589
590 spin_lock_irqsave(&chan->vc.lock, flags);
591 vd = vchan_next_desc(&chan->vc);
592 if (vd) {
593 list_del(&vd->node);
594 vchan_cookie_complete(vd);
595 }
596 spin_unlock_irqrestore(&chan->vc.lock, flags);
597 chan->request = EDMA_REQ_NONE;
598 chan->status = EDMA_ST_IDLE;
599 }
600
dw_edma_interrupt(int irq,void * data,bool write)601 static irqreturn_t dw_edma_interrupt(int irq, void *data, bool write)
602 {
603 struct dw_edma_irq *dw_irq = data;
604 struct dw_edma *dw = dw_irq->dw;
605 unsigned long total, pos, val;
606 unsigned long off;
607 u32 mask;
608
609 if (write) {
610 total = dw->wr_ch_cnt;
611 off = 0;
612 mask = dw_irq->wr_mask;
613 } else {
614 total = dw->rd_ch_cnt;
615 off = dw->wr_ch_cnt;
616 mask = dw_irq->rd_mask;
617 }
618
619 val = dw_edma_v0_core_status_done_int(dw, write ?
620 EDMA_DIR_WRITE :
621 EDMA_DIR_READ);
622 val &= mask;
623 for_each_set_bit(pos, &val, total) {
624 struct dw_edma_chan *chan = &dw->chan[pos + off];
625
626 dw_edma_done_interrupt(chan);
627 }
628
629 val = dw_edma_v0_core_status_abort_int(dw, write ?
630 EDMA_DIR_WRITE :
631 EDMA_DIR_READ);
632 val &= mask;
633 for_each_set_bit(pos, &val, total) {
634 struct dw_edma_chan *chan = &dw->chan[pos + off];
635
636 dw_edma_abort_interrupt(chan);
637 }
638
639 return IRQ_HANDLED;
640 }
641
dw_edma_interrupt_write(int irq,void * data)642 static inline irqreturn_t dw_edma_interrupt_write(int irq, void *data)
643 {
644 return dw_edma_interrupt(irq, data, true);
645 }
646
dw_edma_interrupt_read(int irq,void * data)647 static inline irqreturn_t dw_edma_interrupt_read(int irq, void *data)
648 {
649 return dw_edma_interrupt(irq, data, false);
650 }
651
dw_edma_interrupt_common(int irq,void * data)652 static irqreturn_t dw_edma_interrupt_common(int irq, void *data)
653 {
654 dw_edma_interrupt(irq, data, true);
655 dw_edma_interrupt(irq, data, false);
656
657 return IRQ_HANDLED;
658 }
659
dw_edma_alloc_chan_resources(struct dma_chan * dchan)660 static int dw_edma_alloc_chan_resources(struct dma_chan *dchan)
661 {
662 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
663
664 if (chan->status != EDMA_ST_IDLE)
665 return -EBUSY;
666
667 pm_runtime_get(chan->chip->dev);
668
669 return 0;
670 }
671
dw_edma_free_chan_resources(struct dma_chan * dchan)672 static void dw_edma_free_chan_resources(struct dma_chan *dchan)
673 {
674 unsigned long timeout = jiffies + msecs_to_jiffies(5000);
675 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
676 int ret;
677
678 while (time_before(jiffies, timeout)) {
679 ret = dw_edma_device_terminate_all(dchan);
680 if (!ret)
681 break;
682
683 if (time_after_eq(jiffies, timeout))
684 return;
685
686 cpu_relax();
687 }
688
689 pm_runtime_put(chan->chip->dev);
690 }
691
dw_edma_channel_setup(struct dw_edma_chip * chip,bool write,u32 wr_alloc,u32 rd_alloc)692 static int dw_edma_channel_setup(struct dw_edma_chip *chip, bool write,
693 u32 wr_alloc, u32 rd_alloc)
694 {
695 struct dw_edma_region *dt_region;
696 struct device *dev = chip->dev;
697 struct dw_edma *dw = chip->dw;
698 struct dw_edma_chan *chan;
699 struct dw_edma_irq *irq;
700 struct dma_device *dma;
701 u32 alloc, off_alloc;
702 u32 i, j, cnt;
703 int err = 0;
704 u32 pos;
705
706 if (write) {
707 i = 0;
708 cnt = dw->wr_ch_cnt;
709 dma = &dw->wr_edma;
710 alloc = wr_alloc;
711 off_alloc = 0;
712 } else {
713 i = dw->wr_ch_cnt;
714 cnt = dw->rd_ch_cnt;
715 dma = &dw->rd_edma;
716 alloc = rd_alloc;
717 off_alloc = wr_alloc;
718 }
719
720 INIT_LIST_HEAD(&dma->channels);
721 for (j = 0; (alloc || dw->nr_irqs == 1) && j < cnt; j++, i++) {
722 chan = &dw->chan[i];
723
724 dt_region = devm_kzalloc(dev, sizeof(*dt_region), GFP_KERNEL);
725 if (!dt_region)
726 return -ENOMEM;
727
728 chan->vc.chan.private = dt_region;
729
730 chan->chip = chip;
731 chan->id = j;
732 chan->dir = write ? EDMA_DIR_WRITE : EDMA_DIR_READ;
733 chan->configured = false;
734 chan->request = EDMA_REQ_NONE;
735 chan->status = EDMA_ST_IDLE;
736
737 if (write)
738 chan->ll_max = (dw->ll_region_wr[j].sz / EDMA_LL_SZ);
739 else
740 chan->ll_max = (dw->ll_region_rd[j].sz / EDMA_LL_SZ);
741 chan->ll_max -= 1;
742
743 dev_vdbg(dev, "L. List:\tChannel %s[%u] max_cnt=%u\n",
744 write ? "write" : "read", j, chan->ll_max);
745
746 if (dw->nr_irqs == 1)
747 pos = 0;
748 else
749 pos = off_alloc + (j % alloc);
750
751 irq = &dw->irq[pos];
752
753 if (write)
754 irq->wr_mask |= BIT(j);
755 else
756 irq->rd_mask |= BIT(j);
757
758 irq->dw = dw;
759 memcpy(&chan->msi, &irq->msi, sizeof(chan->msi));
760
761 dev_vdbg(dev, "MSI:\t\tChannel %s[%u] addr=0x%.8x%.8x, data=0x%.8x\n",
762 write ? "write" : "read", j,
763 chan->msi.address_hi, chan->msi.address_lo,
764 chan->msi.data);
765
766 chan->vc.desc_free = vchan_free_desc;
767 vchan_init(&chan->vc, dma);
768
769 if (write) {
770 dt_region->paddr = dw->dt_region_wr[j].paddr;
771 dt_region->vaddr = dw->dt_region_wr[j].vaddr;
772 dt_region->sz = dw->dt_region_wr[j].sz;
773 } else {
774 dt_region->paddr = dw->dt_region_rd[j].paddr;
775 dt_region->vaddr = dw->dt_region_rd[j].vaddr;
776 dt_region->sz = dw->dt_region_rd[j].sz;
777 }
778
779 dw_edma_v0_core_device_config(chan);
780 }
781
782 /* Set DMA channel capabilities */
783 dma_cap_zero(dma->cap_mask);
784 dma_cap_set(DMA_SLAVE, dma->cap_mask);
785 dma_cap_set(DMA_CYCLIC, dma->cap_mask);
786 dma_cap_set(DMA_PRIVATE, dma->cap_mask);
787 dma_cap_set(DMA_INTERLEAVE, dma->cap_mask);
788 dma->directions = BIT(write ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV);
789 dma->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
790 dma->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
791 dma->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
792 dma->chancnt = cnt;
793
794 /* Set DMA channel callbacks */
795 dma->dev = chip->dev;
796 dma->device_alloc_chan_resources = dw_edma_alloc_chan_resources;
797 dma->device_free_chan_resources = dw_edma_free_chan_resources;
798 dma->device_config = dw_edma_device_config;
799 dma->device_pause = dw_edma_device_pause;
800 dma->device_resume = dw_edma_device_resume;
801 dma->device_terminate_all = dw_edma_device_terminate_all;
802 dma->device_issue_pending = dw_edma_device_issue_pending;
803 dma->device_tx_status = dw_edma_device_tx_status;
804 dma->device_prep_slave_sg = dw_edma_device_prep_slave_sg;
805 dma->device_prep_dma_cyclic = dw_edma_device_prep_dma_cyclic;
806 dma->device_prep_interleaved_dma = dw_edma_device_prep_interleaved_dma;
807
808 dma_set_max_seg_size(dma->dev, U32_MAX);
809
810 /* Register DMA device */
811 err = dma_async_device_register(dma);
812
813 return err;
814 }
815
dw_edma_dec_irq_alloc(int * nr_irqs,u32 * alloc,u16 cnt)816 static inline void dw_edma_dec_irq_alloc(int *nr_irqs, u32 *alloc, u16 cnt)
817 {
818 if (*nr_irqs && *alloc < cnt) {
819 (*alloc)++;
820 (*nr_irqs)--;
821 }
822 }
823
dw_edma_add_irq_mask(u32 * mask,u32 alloc,u16 cnt)824 static inline void dw_edma_add_irq_mask(u32 *mask, u32 alloc, u16 cnt)
825 {
826 while (*mask * alloc < cnt)
827 (*mask)++;
828 }
829
dw_edma_irq_request(struct dw_edma_chip * chip,u32 * wr_alloc,u32 * rd_alloc)830 static int dw_edma_irq_request(struct dw_edma_chip *chip,
831 u32 *wr_alloc, u32 *rd_alloc)
832 {
833 struct device *dev = chip->dev;
834 struct dw_edma *dw = chip->dw;
835 u32 wr_mask = 1;
836 u32 rd_mask = 1;
837 int i, err = 0;
838 u32 ch_cnt;
839 int irq;
840
841 ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt;
842
843 if (dw->nr_irqs < 1)
844 return -EINVAL;
845
846 if (dw->nr_irqs == 1) {
847 /* Common IRQ shared among all channels */
848 irq = dw->ops->irq_vector(dev, 0);
849 err = request_irq(irq, dw_edma_interrupt_common,
850 IRQF_SHARED, dw->name, &dw->irq[0]);
851 if (err) {
852 dw->nr_irqs = 0;
853 return err;
854 }
855
856 if (irq_get_msi_desc(irq))
857 get_cached_msi_msg(irq, &dw->irq[0].msi);
858 } else {
859 /* Distribute IRQs equally among all channels */
860 int tmp = dw->nr_irqs;
861
862 while (tmp && (*wr_alloc + *rd_alloc) < ch_cnt) {
863 dw_edma_dec_irq_alloc(&tmp, wr_alloc, dw->wr_ch_cnt);
864 dw_edma_dec_irq_alloc(&tmp, rd_alloc, dw->rd_ch_cnt);
865 }
866
867 dw_edma_add_irq_mask(&wr_mask, *wr_alloc, dw->wr_ch_cnt);
868 dw_edma_add_irq_mask(&rd_mask, *rd_alloc, dw->rd_ch_cnt);
869
870 for (i = 0; i < (*wr_alloc + *rd_alloc); i++) {
871 irq = dw->ops->irq_vector(dev, i);
872 err = request_irq(irq,
873 i < *wr_alloc ?
874 dw_edma_interrupt_write :
875 dw_edma_interrupt_read,
876 IRQF_SHARED, dw->name,
877 &dw->irq[i]);
878 if (err) {
879 dw->nr_irqs = i;
880 return err;
881 }
882
883 if (irq_get_msi_desc(irq))
884 get_cached_msi_msg(irq, &dw->irq[i].msi);
885 }
886
887 dw->nr_irqs = i;
888 }
889
890 return err;
891 }
892
dw_edma_probe(struct dw_edma_chip * chip)893 int dw_edma_probe(struct dw_edma_chip *chip)
894 {
895 struct device *dev;
896 struct dw_edma *dw;
897 u32 wr_alloc = 0;
898 u32 rd_alloc = 0;
899 int i, err;
900
901 if (!chip)
902 return -EINVAL;
903
904 dev = chip->dev;
905 if (!dev)
906 return -EINVAL;
907
908 dw = chip->dw;
909 if (!dw || !dw->irq || !dw->ops || !dw->ops->irq_vector)
910 return -EINVAL;
911
912 raw_spin_lock_init(&dw->lock);
913
914 dw->wr_ch_cnt = min_t(u16, dw->wr_ch_cnt,
915 dw_edma_v0_core_ch_count(dw, EDMA_DIR_WRITE));
916 dw->wr_ch_cnt = min_t(u16, dw->wr_ch_cnt, EDMA_MAX_WR_CH);
917
918 dw->rd_ch_cnt = min_t(u16, dw->rd_ch_cnt,
919 dw_edma_v0_core_ch_count(dw, EDMA_DIR_READ));
920 dw->rd_ch_cnt = min_t(u16, dw->rd_ch_cnt, EDMA_MAX_RD_CH);
921
922 if (!dw->wr_ch_cnt && !dw->rd_ch_cnt)
923 return -EINVAL;
924
925 dev_vdbg(dev, "Channels:\twrite=%d, read=%d\n",
926 dw->wr_ch_cnt, dw->rd_ch_cnt);
927
928 /* Allocate channels */
929 dw->chan = devm_kcalloc(dev, dw->wr_ch_cnt + dw->rd_ch_cnt,
930 sizeof(*dw->chan), GFP_KERNEL);
931 if (!dw->chan)
932 return -ENOMEM;
933
934 snprintf(dw->name, sizeof(dw->name), "dw-edma-core:%d", chip->id);
935
936 /* Disable eDMA, only to establish the ideal initial conditions */
937 dw_edma_v0_core_off(dw);
938
939 /* Request IRQs */
940 err = dw_edma_irq_request(chip, &wr_alloc, &rd_alloc);
941 if (err)
942 return err;
943
944 /* Setup write channels */
945 err = dw_edma_channel_setup(chip, true, wr_alloc, rd_alloc);
946 if (err)
947 goto err_irq_free;
948
949 /* Setup read channels */
950 err = dw_edma_channel_setup(chip, false, wr_alloc, rd_alloc);
951 if (err)
952 goto err_irq_free;
953
954 /* Power management */
955 pm_runtime_enable(dev);
956
957 /* Turn debugfs on */
958 dw_edma_v0_core_debugfs_on(chip);
959
960 return 0;
961
962 err_irq_free:
963 for (i = (dw->nr_irqs - 1); i >= 0; i--)
964 free_irq(dw->ops->irq_vector(dev, i), &dw->irq[i]);
965
966 dw->nr_irqs = 0;
967
968 return err;
969 }
970 EXPORT_SYMBOL_GPL(dw_edma_probe);
971
dw_edma_remove(struct dw_edma_chip * chip)972 int dw_edma_remove(struct dw_edma_chip *chip)
973 {
974 struct dw_edma_chan *chan, *_chan;
975 struct device *dev = chip->dev;
976 struct dw_edma *dw = chip->dw;
977 int i;
978
979 /* Disable eDMA */
980 dw_edma_v0_core_off(dw);
981
982 /* Free irqs */
983 for (i = (dw->nr_irqs - 1); i >= 0; i--)
984 free_irq(dw->ops->irq_vector(dev, i), &dw->irq[i]);
985
986 /* Power management */
987 pm_runtime_disable(dev);
988
989 /* Deregister eDMA device */
990 dma_async_device_unregister(&dw->wr_edma);
991 list_for_each_entry_safe(chan, _chan, &dw->wr_edma.channels,
992 vc.chan.device_node) {
993 tasklet_kill(&chan->vc.task);
994 list_del(&chan->vc.chan.device_node);
995 }
996
997 dma_async_device_unregister(&dw->rd_edma);
998 list_for_each_entry_safe(chan, _chan, &dw->rd_edma.channels,
999 vc.chan.device_node) {
1000 tasklet_kill(&chan->vc.task);
1001 list_del(&chan->vc.chan.device_node);
1002 }
1003
1004 /* Turn debugfs off */
1005 dw_edma_v0_core_debugfs_off(chip);
1006
1007 return 0;
1008 }
1009 EXPORT_SYMBOL_GPL(dw_edma_remove);
1010
1011 MODULE_LICENSE("GPL v2");
1012 MODULE_DESCRIPTION("Synopsys DesignWare eDMA controller core driver");
1013 MODULE_AUTHOR("Gustavo Pimentel <gustavo.pimentel@synopsys.com>");
1014