1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright 2012 Marvell International Ltd. 4 */ 5 6 #include <linux/err.h> 7 #include <linux/module.h> 8 #include <linux/init.h> 9 #include <linux/types.h> 10 #include <linux/interrupt.h> 11 #include <linux/dma-mapping.h> 12 #include <linux/slab.h> 13 #include <linux/dmaengine.h> 14 #include <linux/platform_device.h> 15 #include <linux/device.h> 16 #include <linux/platform_data/mmp_dma.h> 17 #include <linux/dmapool.h> 18 #include <linux/of_device.h> 19 #include <linux/of_dma.h> 20 #include <linux/of.h> 21 #include <linux/dma/mmp-pdma.h> 22 23 #include "dmaengine.h" 24 25 #define DCSR 0x0000 26 #define DALGN 0x00a0 27 #define DINT 0x00f0 28 #define DDADR 0x0200 29 #define DSADR(n) (0x0204 + ((n) << 4)) 30 #define DTADR(n) (0x0208 + ((n) << 4)) 31 #define DCMD 0x020c 32 33 #define DCSR_RUN BIT(31) /* Run Bit (read / write) */ 34 #define DCSR_NODESC BIT(30) /* No-Descriptor Fetch (read / write) */ 35 #define DCSR_STOPIRQEN BIT(29) /* Stop Interrupt Enable (read / write) */ 36 #define DCSR_REQPEND BIT(8) /* Request Pending (read-only) */ 37 #define DCSR_STOPSTATE BIT(3) /* Stop State (read-only) */ 38 #define DCSR_ENDINTR BIT(2) /* End Interrupt (read / write) */ 39 #define DCSR_STARTINTR BIT(1) /* Start Interrupt (read / write) */ 40 #define DCSR_BUSERR BIT(0) /* Bus Error Interrupt (read / write) */ 41 42 #define DCSR_EORIRQEN BIT(28) /* End of Receive Interrupt Enable (R/W) */ 43 #define DCSR_EORJMPEN BIT(27) /* Jump to next descriptor on EOR */ 44 #define DCSR_EORSTOPEN BIT(26) /* STOP on an EOR */ 45 #define DCSR_SETCMPST BIT(25) /* Set Descriptor Compare Status */ 46 #define DCSR_CLRCMPST BIT(24) /* Clear Descriptor Compare Status */ 47 #define DCSR_CMPST BIT(10) /* The Descriptor Compare Status */ 48 #define DCSR_EORINTR BIT(9) /* The end of Receive */ 49 50 #define DRCMR(n) ((((n) < 64) ? 0x0100 : 0x1100) + (((n) & 0x3f) << 2)) 51 #define DRCMR_MAPVLD BIT(7) /* Map Valid (read / write) */ 52 #define DRCMR_CHLNUM 0x1f /* mask for Channel Number (read / write) */ 53 54 #define DDADR_DESCADDR 0xfffffff0 /* Address of next descriptor (mask) */ 55 #define DDADR_STOP BIT(0) /* Stop (read / write) */ 56 57 #define DCMD_INCSRCADDR BIT(31) /* Source Address Increment Setting. */ 58 #define DCMD_INCTRGADDR BIT(30) /* Target Address Increment Setting. */ 59 #define DCMD_FLOWSRC BIT(29) /* Flow Control by the source. */ 60 #define DCMD_FLOWTRG BIT(28) /* Flow Control by the target. */ 61 #define DCMD_STARTIRQEN BIT(22) /* Start Interrupt Enable */ 62 #define DCMD_ENDIRQEN BIT(21) /* End Interrupt Enable */ 63 #define DCMD_ENDIAN BIT(18) /* Device Endian-ness. */ 64 #define DCMD_BURST8 (1 << 16) /* 8 byte burst */ 65 #define DCMD_BURST16 (2 << 16) /* 16 byte burst */ 66 #define DCMD_BURST32 (3 << 16) /* 32 byte burst */ 67 #define DCMD_WIDTH1 (1 << 14) /* 1 byte width */ 68 #define DCMD_WIDTH2 (2 << 14) /* 2 byte width (HalfWord) */ 69 #define DCMD_WIDTH4 (3 << 14) /* 4 byte width (Word) */ 70 #define DCMD_LENGTH 0x01fff /* length mask (max = 8K - 1) */ 71 72 #define PDMA_MAX_DESC_BYTES DCMD_LENGTH 73 74 struct mmp_pdma_desc_hw { 75 u32 ddadr; /* Points to the next descriptor + flags */ 76 u32 dsadr; /* DSADR value for the current transfer */ 77 u32 dtadr; /* DTADR value for the current transfer */ 78 u32 dcmd; /* DCMD value for the current transfer */ 79 } __aligned(32); 80 81 struct mmp_pdma_desc_sw { 82 struct mmp_pdma_desc_hw desc; 83 struct list_head node; 84 struct list_head tx_list; 85 struct dma_async_tx_descriptor async_tx; 86 }; 87 88 struct mmp_pdma_phy; 89 90 struct mmp_pdma_chan { 91 struct device *dev; 92 struct dma_chan chan; 93 struct dma_async_tx_descriptor desc; 94 struct mmp_pdma_phy *phy; 95 enum dma_transfer_direction dir; 96 struct dma_slave_config slave_config; 97 98 struct mmp_pdma_desc_sw *cyclic_first; /* first desc_sw if channel 99 * is in cyclic mode */ 100 101 /* channel's basic info */ 102 struct tasklet_struct tasklet; 103 u32 dcmd; 104 u32 drcmr; 105 u32 dev_addr; 106 107 /* list for desc */ 108 spinlock_t desc_lock; /* Descriptor list lock */ 109 struct list_head chain_pending; /* Link descriptors queue for pending */ 110 struct list_head chain_running; /* Link descriptors queue for running */ 111 bool idle; /* channel statue machine */ 112 bool byte_align; 113 114 struct dma_pool *desc_pool; /* Descriptors pool */ 115 }; 116 117 struct mmp_pdma_phy { 118 int idx; 119 void __iomem *base; 120 struct mmp_pdma_chan *vchan; 121 }; 122 123 struct mmp_pdma_device { 124 int dma_channels; 125 void __iomem *base; 126 struct device *dev; 127 struct dma_device device; 128 struct mmp_pdma_phy *phy; 129 spinlock_t phy_lock; /* protect alloc/free phy channels */ 130 }; 131 132 #define tx_to_mmp_pdma_desc(tx) \ 133 container_of(tx, struct mmp_pdma_desc_sw, async_tx) 134 #define to_mmp_pdma_desc(lh) \ 135 container_of(lh, struct mmp_pdma_desc_sw, node) 136 #define to_mmp_pdma_chan(dchan) \ 137 container_of(dchan, struct mmp_pdma_chan, chan) 138 #define to_mmp_pdma_dev(dmadev) \ 139 container_of(dmadev, struct mmp_pdma_device, device) 140 141 static int mmp_pdma_config_write(struct dma_chan *dchan, 142 struct dma_slave_config *cfg, 143 enum dma_transfer_direction direction); 144 145 static void set_desc(struct mmp_pdma_phy *phy, dma_addr_t addr) 146 { 147 u32 reg = (phy->idx << 4) + DDADR; 148 149 writel(addr, phy->base + reg); 150 } 151 152 static void enable_chan(struct mmp_pdma_phy *phy) 153 { 154 u32 reg, dalgn; 155 156 if (!phy->vchan) 157 return; 158 159 reg = DRCMR(phy->vchan->drcmr); 160 writel(DRCMR_MAPVLD | phy->idx, phy->base + reg); 161 162 dalgn = readl(phy->base + DALGN); 163 if (phy->vchan->byte_align) 164 dalgn |= 1 << phy->idx; 165 else 166 dalgn &= ~(1 << phy->idx); 167 writel(dalgn, phy->base + DALGN); 168 169 reg = (phy->idx << 2) + DCSR; 170 writel(readl(phy->base + reg) | DCSR_RUN, phy->base + reg); 171 } 172 173 static void disable_chan(struct mmp_pdma_phy *phy) 174 { 175 u32 reg; 176 177 if (!phy) 178 return; 179 180 reg = (phy->idx << 2) + DCSR; 181 writel(readl(phy->base + reg) & ~DCSR_RUN, phy->base + reg); 182 } 183 184 static int clear_chan_irq(struct mmp_pdma_phy *phy) 185 { 186 u32 dcsr; 187 u32 dint = readl(phy->base + DINT); 188 u32 reg = (phy->idx << 2) + DCSR; 189 190 if (!(dint & BIT(phy->idx))) 191 return -EAGAIN; 192 193 /* clear irq */ 194 dcsr = readl(phy->base + reg); 195 writel(dcsr, phy->base + reg); 196 if ((dcsr & DCSR_BUSERR) && (phy->vchan)) 197 dev_warn(phy->vchan->dev, "DCSR_BUSERR\n"); 198 199 return 0; 200 } 201 202 static irqreturn_t mmp_pdma_chan_handler(int irq, void *dev_id) 203 { 204 struct mmp_pdma_phy *phy = dev_id; 205 206 if (clear_chan_irq(phy) != 0) 207 return IRQ_NONE; 208 209 tasklet_schedule(&phy->vchan->tasklet); 210 return IRQ_HANDLED; 211 } 212 213 static irqreturn_t mmp_pdma_int_handler(int irq, void *dev_id) 214 { 215 struct mmp_pdma_device *pdev = dev_id; 216 struct mmp_pdma_phy *phy; 217 u32 dint = readl(pdev->base + DINT); 218 int i, ret; 219 int irq_num = 0; 220 221 while (dint) { 222 i = __ffs(dint); 223 /* only handle interrupts belonging to pdma driver*/ 224 if (i >= pdev->dma_channels) 225 break; 226 dint &= (dint - 1); 227 phy = &pdev->phy[i]; 228 ret = mmp_pdma_chan_handler(irq, phy); 229 if (ret == IRQ_HANDLED) 230 irq_num++; 231 } 232 233 if (irq_num) 234 return IRQ_HANDLED; 235 236 return IRQ_NONE; 237 } 238 239 /* lookup free phy channel as descending priority */ 240 static struct mmp_pdma_phy *lookup_phy(struct mmp_pdma_chan *pchan) 241 { 242 int prio, i; 243 struct mmp_pdma_device *pdev = to_mmp_pdma_dev(pchan->chan.device); 244 struct mmp_pdma_phy *phy, *found = NULL; 245 unsigned long flags; 246 247 /* 248 * dma channel priorities 249 * ch 0 - 3, 16 - 19 <--> (0) 250 * ch 4 - 7, 20 - 23 <--> (1) 251 * ch 8 - 11, 24 - 27 <--> (2) 252 * ch 12 - 15, 28 - 31 <--> (3) 253 */ 254 255 spin_lock_irqsave(&pdev->phy_lock, flags); 256 for (prio = 0; prio <= ((pdev->dma_channels - 1) & 0xf) >> 2; prio++) { 257 for (i = 0; i < pdev->dma_channels; i++) { 258 if (prio != (i & 0xf) >> 2) 259 continue; 260 phy = &pdev->phy[i]; 261 if (!phy->vchan) { 262 phy->vchan = pchan; 263 found = phy; 264 goto out_unlock; 265 } 266 } 267 } 268 269 out_unlock: 270 spin_unlock_irqrestore(&pdev->phy_lock, flags); 271 return found; 272 } 273 274 static void mmp_pdma_free_phy(struct mmp_pdma_chan *pchan) 275 { 276 struct mmp_pdma_device *pdev = to_mmp_pdma_dev(pchan->chan.device); 277 unsigned long flags; 278 u32 reg; 279 280 if (!pchan->phy) 281 return; 282 283 /* clear the channel mapping in DRCMR */ 284 reg = DRCMR(pchan->drcmr); 285 writel(0, pchan->phy->base + reg); 286 287 spin_lock_irqsave(&pdev->phy_lock, flags); 288 pchan->phy->vchan = NULL; 289 pchan->phy = NULL; 290 spin_unlock_irqrestore(&pdev->phy_lock, flags); 291 } 292 293 /** 294 * start_pending_queue - transfer any pending transactions 295 * pending list ==> running list 296 */ 297 static void start_pending_queue(struct mmp_pdma_chan *chan) 298 { 299 struct mmp_pdma_desc_sw *desc; 300 301 /* still in running, irq will start the pending list */ 302 if (!chan->idle) { 303 dev_dbg(chan->dev, "DMA controller still busy\n"); 304 return; 305 } 306 307 if (list_empty(&chan->chain_pending)) { 308 /* chance to re-fetch phy channel with higher prio */ 309 mmp_pdma_free_phy(chan); 310 dev_dbg(chan->dev, "no pending list\n"); 311 return; 312 } 313 314 if (!chan->phy) { 315 chan->phy = lookup_phy(chan); 316 if (!chan->phy) { 317 dev_dbg(chan->dev, "no free dma channel\n"); 318 return; 319 } 320 } 321 322 /* 323 * pending -> running 324 * reintilize pending list 325 */ 326 desc = list_first_entry(&chan->chain_pending, 327 struct mmp_pdma_desc_sw, node); 328 list_splice_tail_init(&chan->chain_pending, &chan->chain_running); 329 330 /* 331 * Program the descriptor's address into the DMA controller, 332 * then start the DMA transaction 333 */ 334 set_desc(chan->phy, desc->async_tx.phys); 335 enable_chan(chan->phy); 336 chan->idle = false; 337 } 338 339 340 /* desc->tx_list ==> pending list */ 341 static dma_cookie_t mmp_pdma_tx_submit(struct dma_async_tx_descriptor *tx) 342 { 343 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(tx->chan); 344 struct mmp_pdma_desc_sw *desc = tx_to_mmp_pdma_desc(tx); 345 struct mmp_pdma_desc_sw *child; 346 unsigned long flags; 347 dma_cookie_t cookie = -EBUSY; 348 349 spin_lock_irqsave(&chan->desc_lock, flags); 350 351 list_for_each_entry(child, &desc->tx_list, node) { 352 cookie = dma_cookie_assign(&child->async_tx); 353 } 354 355 /* softly link to pending list - desc->tx_list ==> pending list */ 356 list_splice_tail_init(&desc->tx_list, &chan->chain_pending); 357 358 spin_unlock_irqrestore(&chan->desc_lock, flags); 359 360 return cookie; 361 } 362 363 static struct mmp_pdma_desc_sw * 364 mmp_pdma_alloc_descriptor(struct mmp_pdma_chan *chan) 365 { 366 struct mmp_pdma_desc_sw *desc; 367 dma_addr_t pdesc; 368 369 desc = dma_pool_zalloc(chan->desc_pool, GFP_ATOMIC, &pdesc); 370 if (!desc) { 371 dev_err(chan->dev, "out of memory for link descriptor\n"); 372 return NULL; 373 } 374 375 INIT_LIST_HEAD(&desc->tx_list); 376 dma_async_tx_descriptor_init(&desc->async_tx, &chan->chan); 377 /* each desc has submit */ 378 desc->async_tx.tx_submit = mmp_pdma_tx_submit; 379 desc->async_tx.phys = pdesc; 380 381 return desc; 382 } 383 384 /** 385 * mmp_pdma_alloc_chan_resources - Allocate resources for DMA channel. 386 * 387 * This function will create a dma pool for descriptor allocation. 388 * Request irq only when channel is requested 389 * Return - The number of allocated descriptors. 390 */ 391 392 static int mmp_pdma_alloc_chan_resources(struct dma_chan *dchan) 393 { 394 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 395 396 if (chan->desc_pool) 397 return 1; 398 399 chan->desc_pool = dma_pool_create(dev_name(&dchan->dev->device), 400 chan->dev, 401 sizeof(struct mmp_pdma_desc_sw), 402 __alignof__(struct mmp_pdma_desc_sw), 403 0); 404 if (!chan->desc_pool) { 405 dev_err(chan->dev, "unable to allocate descriptor pool\n"); 406 return -ENOMEM; 407 } 408 409 mmp_pdma_free_phy(chan); 410 chan->idle = true; 411 chan->dev_addr = 0; 412 return 1; 413 } 414 415 static void mmp_pdma_free_desc_list(struct mmp_pdma_chan *chan, 416 struct list_head *list) 417 { 418 struct mmp_pdma_desc_sw *desc, *_desc; 419 420 list_for_each_entry_safe(desc, _desc, list, node) { 421 list_del(&desc->node); 422 dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys); 423 } 424 } 425 426 static void mmp_pdma_free_chan_resources(struct dma_chan *dchan) 427 { 428 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 429 unsigned long flags; 430 431 spin_lock_irqsave(&chan->desc_lock, flags); 432 mmp_pdma_free_desc_list(chan, &chan->chain_pending); 433 mmp_pdma_free_desc_list(chan, &chan->chain_running); 434 spin_unlock_irqrestore(&chan->desc_lock, flags); 435 436 dma_pool_destroy(chan->desc_pool); 437 chan->desc_pool = NULL; 438 chan->idle = true; 439 chan->dev_addr = 0; 440 mmp_pdma_free_phy(chan); 441 return; 442 } 443 444 static struct dma_async_tx_descriptor * 445 mmp_pdma_prep_memcpy(struct dma_chan *dchan, 446 dma_addr_t dma_dst, dma_addr_t dma_src, 447 size_t len, unsigned long flags) 448 { 449 struct mmp_pdma_chan *chan; 450 struct mmp_pdma_desc_sw *first = NULL, *prev = NULL, *new; 451 size_t copy = 0; 452 453 if (!dchan) 454 return NULL; 455 456 if (!len) 457 return NULL; 458 459 chan = to_mmp_pdma_chan(dchan); 460 chan->byte_align = false; 461 462 if (!chan->dir) { 463 chan->dir = DMA_MEM_TO_MEM; 464 chan->dcmd = DCMD_INCTRGADDR | DCMD_INCSRCADDR; 465 chan->dcmd |= DCMD_BURST32; 466 } 467 468 do { 469 /* Allocate the link descriptor from DMA pool */ 470 new = mmp_pdma_alloc_descriptor(chan); 471 if (!new) { 472 dev_err(chan->dev, "no memory for desc\n"); 473 goto fail; 474 } 475 476 copy = min_t(size_t, len, PDMA_MAX_DESC_BYTES); 477 if (dma_src & 0x7 || dma_dst & 0x7) 478 chan->byte_align = true; 479 480 new->desc.dcmd = chan->dcmd | (DCMD_LENGTH & copy); 481 new->desc.dsadr = dma_src; 482 new->desc.dtadr = dma_dst; 483 484 if (!first) 485 first = new; 486 else 487 prev->desc.ddadr = new->async_tx.phys; 488 489 new->async_tx.cookie = 0; 490 async_tx_ack(&new->async_tx); 491 492 prev = new; 493 len -= copy; 494 495 if (chan->dir == DMA_MEM_TO_DEV) { 496 dma_src += copy; 497 } else if (chan->dir == DMA_DEV_TO_MEM) { 498 dma_dst += copy; 499 } else if (chan->dir == DMA_MEM_TO_MEM) { 500 dma_src += copy; 501 dma_dst += copy; 502 } 503 504 /* Insert the link descriptor to the LD ring */ 505 list_add_tail(&new->node, &first->tx_list); 506 } while (len); 507 508 first->async_tx.flags = flags; /* client is in control of this ack */ 509 first->async_tx.cookie = -EBUSY; 510 511 /* last desc and fire IRQ */ 512 new->desc.ddadr = DDADR_STOP; 513 new->desc.dcmd |= DCMD_ENDIRQEN; 514 515 chan->cyclic_first = NULL; 516 517 return &first->async_tx; 518 519 fail: 520 if (first) 521 mmp_pdma_free_desc_list(chan, &first->tx_list); 522 return NULL; 523 } 524 525 static struct dma_async_tx_descriptor * 526 mmp_pdma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, 527 unsigned int sg_len, enum dma_transfer_direction dir, 528 unsigned long flags, void *context) 529 { 530 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 531 struct mmp_pdma_desc_sw *first = NULL, *prev = NULL, *new = NULL; 532 size_t len, avail; 533 struct scatterlist *sg; 534 dma_addr_t addr; 535 int i; 536 537 if ((sgl == NULL) || (sg_len == 0)) 538 return NULL; 539 540 chan->byte_align = false; 541 542 mmp_pdma_config_write(dchan, &chan->slave_config, dir); 543 544 for_each_sg(sgl, sg, sg_len, i) { 545 addr = sg_dma_address(sg); 546 avail = sg_dma_len(sgl); 547 548 do { 549 len = min_t(size_t, avail, PDMA_MAX_DESC_BYTES); 550 if (addr & 0x7) 551 chan->byte_align = true; 552 553 /* allocate and populate the descriptor */ 554 new = mmp_pdma_alloc_descriptor(chan); 555 if (!new) { 556 dev_err(chan->dev, "no memory for desc\n"); 557 goto fail; 558 } 559 560 new->desc.dcmd = chan->dcmd | (DCMD_LENGTH & len); 561 if (dir == DMA_MEM_TO_DEV) { 562 new->desc.dsadr = addr; 563 new->desc.dtadr = chan->dev_addr; 564 } else { 565 new->desc.dsadr = chan->dev_addr; 566 new->desc.dtadr = addr; 567 } 568 569 if (!first) 570 first = new; 571 else 572 prev->desc.ddadr = new->async_tx.phys; 573 574 new->async_tx.cookie = 0; 575 async_tx_ack(&new->async_tx); 576 prev = new; 577 578 /* Insert the link descriptor to the LD ring */ 579 list_add_tail(&new->node, &first->tx_list); 580 581 /* update metadata */ 582 addr += len; 583 avail -= len; 584 } while (avail); 585 } 586 587 first->async_tx.cookie = -EBUSY; 588 first->async_tx.flags = flags; 589 590 /* last desc and fire IRQ */ 591 new->desc.ddadr = DDADR_STOP; 592 new->desc.dcmd |= DCMD_ENDIRQEN; 593 594 chan->dir = dir; 595 chan->cyclic_first = NULL; 596 597 return &first->async_tx; 598 599 fail: 600 if (first) 601 mmp_pdma_free_desc_list(chan, &first->tx_list); 602 return NULL; 603 } 604 605 static struct dma_async_tx_descriptor * 606 mmp_pdma_prep_dma_cyclic(struct dma_chan *dchan, 607 dma_addr_t buf_addr, size_t len, size_t period_len, 608 enum dma_transfer_direction direction, 609 unsigned long flags) 610 { 611 struct mmp_pdma_chan *chan; 612 struct mmp_pdma_desc_sw *first = NULL, *prev = NULL, *new; 613 dma_addr_t dma_src, dma_dst; 614 615 if (!dchan || !len || !period_len) 616 return NULL; 617 618 /* the buffer length must be a multiple of period_len */ 619 if (len % period_len != 0) 620 return NULL; 621 622 if (period_len > PDMA_MAX_DESC_BYTES) 623 return NULL; 624 625 chan = to_mmp_pdma_chan(dchan); 626 mmp_pdma_config_write(dchan, &chan->slave_config, direction); 627 628 switch (direction) { 629 case DMA_MEM_TO_DEV: 630 dma_src = buf_addr; 631 dma_dst = chan->dev_addr; 632 break; 633 case DMA_DEV_TO_MEM: 634 dma_dst = buf_addr; 635 dma_src = chan->dev_addr; 636 break; 637 default: 638 dev_err(chan->dev, "Unsupported direction for cyclic DMA\n"); 639 return NULL; 640 } 641 642 chan->dir = direction; 643 644 do { 645 /* Allocate the link descriptor from DMA pool */ 646 new = mmp_pdma_alloc_descriptor(chan); 647 if (!new) { 648 dev_err(chan->dev, "no memory for desc\n"); 649 goto fail; 650 } 651 652 new->desc.dcmd = (chan->dcmd | DCMD_ENDIRQEN | 653 (DCMD_LENGTH & period_len)); 654 new->desc.dsadr = dma_src; 655 new->desc.dtadr = dma_dst; 656 657 if (!first) 658 first = new; 659 else 660 prev->desc.ddadr = new->async_tx.phys; 661 662 new->async_tx.cookie = 0; 663 async_tx_ack(&new->async_tx); 664 665 prev = new; 666 len -= period_len; 667 668 if (chan->dir == DMA_MEM_TO_DEV) 669 dma_src += period_len; 670 else 671 dma_dst += period_len; 672 673 /* Insert the link descriptor to the LD ring */ 674 list_add_tail(&new->node, &first->tx_list); 675 } while (len); 676 677 first->async_tx.flags = flags; /* client is in control of this ack */ 678 first->async_tx.cookie = -EBUSY; 679 680 /* make the cyclic link */ 681 new->desc.ddadr = first->async_tx.phys; 682 chan->cyclic_first = first; 683 684 return &first->async_tx; 685 686 fail: 687 if (first) 688 mmp_pdma_free_desc_list(chan, &first->tx_list); 689 return NULL; 690 } 691 692 static int mmp_pdma_config_write(struct dma_chan *dchan, 693 struct dma_slave_config *cfg, 694 enum dma_transfer_direction direction) 695 { 696 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 697 u32 maxburst = 0, addr = 0; 698 enum dma_slave_buswidth width = DMA_SLAVE_BUSWIDTH_UNDEFINED; 699 700 if (!dchan) 701 return -EINVAL; 702 703 if (direction == DMA_DEV_TO_MEM) { 704 chan->dcmd = DCMD_INCTRGADDR | DCMD_FLOWSRC; 705 maxburst = cfg->src_maxburst; 706 width = cfg->src_addr_width; 707 addr = cfg->src_addr; 708 } else if (direction == DMA_MEM_TO_DEV) { 709 chan->dcmd = DCMD_INCSRCADDR | DCMD_FLOWTRG; 710 maxburst = cfg->dst_maxburst; 711 width = cfg->dst_addr_width; 712 addr = cfg->dst_addr; 713 } 714 715 if (width == DMA_SLAVE_BUSWIDTH_1_BYTE) 716 chan->dcmd |= DCMD_WIDTH1; 717 else if (width == DMA_SLAVE_BUSWIDTH_2_BYTES) 718 chan->dcmd |= DCMD_WIDTH2; 719 else if (width == DMA_SLAVE_BUSWIDTH_4_BYTES) 720 chan->dcmd |= DCMD_WIDTH4; 721 722 if (maxburst == 8) 723 chan->dcmd |= DCMD_BURST8; 724 else if (maxburst == 16) 725 chan->dcmd |= DCMD_BURST16; 726 else if (maxburst == 32) 727 chan->dcmd |= DCMD_BURST32; 728 729 chan->dir = direction; 730 chan->dev_addr = addr; 731 /* FIXME: drivers should be ported over to use the filter 732 * function. Once that's done, the following two lines can 733 * be removed. 734 */ 735 if (cfg->slave_id) 736 chan->drcmr = cfg->slave_id; 737 738 return 0; 739 } 740 741 static int mmp_pdma_config(struct dma_chan *dchan, 742 struct dma_slave_config *cfg) 743 { 744 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 745 746 memcpy(&chan->slave_config, cfg, sizeof(*cfg)); 747 return 0; 748 } 749 750 static int mmp_pdma_terminate_all(struct dma_chan *dchan) 751 { 752 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 753 unsigned long flags; 754 755 if (!dchan) 756 return -EINVAL; 757 758 disable_chan(chan->phy); 759 mmp_pdma_free_phy(chan); 760 spin_lock_irqsave(&chan->desc_lock, flags); 761 mmp_pdma_free_desc_list(chan, &chan->chain_pending); 762 mmp_pdma_free_desc_list(chan, &chan->chain_running); 763 spin_unlock_irqrestore(&chan->desc_lock, flags); 764 chan->idle = true; 765 766 return 0; 767 } 768 769 static unsigned int mmp_pdma_residue(struct mmp_pdma_chan *chan, 770 dma_cookie_t cookie) 771 { 772 struct mmp_pdma_desc_sw *sw; 773 u32 curr, residue = 0; 774 bool passed = false; 775 bool cyclic = chan->cyclic_first != NULL; 776 777 /* 778 * If the channel does not have a phy pointer anymore, it has already 779 * been completed. Therefore, its residue is 0. 780 */ 781 if (!chan->phy) 782 return 0; 783 784 if (chan->dir == DMA_DEV_TO_MEM) 785 curr = readl(chan->phy->base + DTADR(chan->phy->idx)); 786 else 787 curr = readl(chan->phy->base + DSADR(chan->phy->idx)); 788 789 list_for_each_entry(sw, &chan->chain_running, node) { 790 u32 start, end, len; 791 792 if (chan->dir == DMA_DEV_TO_MEM) 793 start = sw->desc.dtadr; 794 else 795 start = sw->desc.dsadr; 796 797 len = sw->desc.dcmd & DCMD_LENGTH; 798 end = start + len; 799 800 /* 801 * 'passed' will be latched once we found the descriptor which 802 * lies inside the boundaries of the curr pointer. All 803 * descriptors that occur in the list _after_ we found that 804 * partially handled descriptor are still to be processed and 805 * are hence added to the residual bytes counter. 806 */ 807 808 if (passed) { 809 residue += len; 810 } else if (curr >= start && curr <= end) { 811 residue += end - curr; 812 passed = true; 813 } 814 815 /* 816 * Descriptors that have the ENDIRQEN bit set mark the end of a 817 * transaction chain, and the cookie assigned with it has been 818 * returned previously from mmp_pdma_tx_submit(). 819 * 820 * In case we have multiple transactions in the running chain, 821 * and the cookie does not match the one the user asked us 822 * about, reset the state variables and start over. 823 * 824 * This logic does not apply to cyclic transactions, where all 825 * descriptors have the ENDIRQEN bit set, and for which we 826 * can't have multiple transactions on one channel anyway. 827 */ 828 if (cyclic || !(sw->desc.dcmd & DCMD_ENDIRQEN)) 829 continue; 830 831 if (sw->async_tx.cookie == cookie) { 832 return residue; 833 } else { 834 residue = 0; 835 passed = false; 836 } 837 } 838 839 /* We should only get here in case of cyclic transactions */ 840 return residue; 841 } 842 843 static enum dma_status mmp_pdma_tx_status(struct dma_chan *dchan, 844 dma_cookie_t cookie, 845 struct dma_tx_state *txstate) 846 { 847 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 848 enum dma_status ret; 849 850 ret = dma_cookie_status(dchan, cookie, txstate); 851 if (likely(ret != DMA_ERROR)) 852 dma_set_residue(txstate, mmp_pdma_residue(chan, cookie)); 853 854 return ret; 855 } 856 857 /** 858 * mmp_pdma_issue_pending - Issue the DMA start command 859 * pending list ==> running list 860 */ 861 static void mmp_pdma_issue_pending(struct dma_chan *dchan) 862 { 863 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 864 unsigned long flags; 865 866 spin_lock_irqsave(&chan->desc_lock, flags); 867 start_pending_queue(chan); 868 spin_unlock_irqrestore(&chan->desc_lock, flags); 869 } 870 871 /* 872 * dma_do_tasklet 873 * Do call back 874 * Start pending list 875 */ 876 static void dma_do_tasklet(unsigned long data) 877 { 878 struct mmp_pdma_chan *chan = (struct mmp_pdma_chan *)data; 879 struct mmp_pdma_desc_sw *desc, *_desc; 880 LIST_HEAD(chain_cleanup); 881 unsigned long flags; 882 struct dmaengine_desc_callback cb; 883 884 if (chan->cyclic_first) { 885 spin_lock_irqsave(&chan->desc_lock, flags); 886 desc = chan->cyclic_first; 887 dmaengine_desc_get_callback(&desc->async_tx, &cb); 888 spin_unlock_irqrestore(&chan->desc_lock, flags); 889 890 dmaengine_desc_callback_invoke(&cb, NULL); 891 892 return; 893 } 894 895 /* submit pending list; callback for each desc; free desc */ 896 spin_lock_irqsave(&chan->desc_lock, flags); 897 898 list_for_each_entry_safe(desc, _desc, &chan->chain_running, node) { 899 /* 900 * move the descriptors to a temporary list so we can drop 901 * the lock during the entire cleanup operation 902 */ 903 list_move(&desc->node, &chain_cleanup); 904 905 /* 906 * Look for the first list entry which has the ENDIRQEN flag 907 * set. That is the descriptor we got an interrupt for, so 908 * complete that transaction and its cookie. 909 */ 910 if (desc->desc.dcmd & DCMD_ENDIRQEN) { 911 dma_cookie_t cookie = desc->async_tx.cookie; 912 dma_cookie_complete(&desc->async_tx); 913 dev_dbg(chan->dev, "completed_cookie=%d\n", cookie); 914 break; 915 } 916 } 917 918 /* 919 * The hardware is idle and ready for more when the 920 * chain_running list is empty. 921 */ 922 chan->idle = list_empty(&chan->chain_running); 923 924 /* Start any pending transactions automatically */ 925 start_pending_queue(chan); 926 spin_unlock_irqrestore(&chan->desc_lock, flags); 927 928 /* Run the callback for each descriptor, in order */ 929 list_for_each_entry_safe(desc, _desc, &chain_cleanup, node) { 930 struct dma_async_tx_descriptor *txd = &desc->async_tx; 931 932 /* Remove from the list of transactions */ 933 list_del(&desc->node); 934 /* Run the link descriptor callback function */ 935 dmaengine_desc_get_callback(txd, &cb); 936 dmaengine_desc_callback_invoke(&cb, NULL); 937 938 dma_pool_free(chan->desc_pool, desc, txd->phys); 939 } 940 } 941 942 static int mmp_pdma_remove(struct platform_device *op) 943 { 944 struct mmp_pdma_device *pdev = platform_get_drvdata(op); 945 struct mmp_pdma_phy *phy; 946 int i, irq = 0, irq_num = 0; 947 948 if (op->dev.of_node) 949 of_dma_controller_free(op->dev.of_node); 950 951 for (i = 0; i < pdev->dma_channels; i++) { 952 if (platform_get_irq(op, i) > 0) 953 irq_num++; 954 } 955 956 if (irq_num != pdev->dma_channels) { 957 irq = platform_get_irq(op, 0); 958 devm_free_irq(&op->dev, irq, pdev); 959 } else { 960 for (i = 0; i < pdev->dma_channels; i++) { 961 phy = &pdev->phy[i]; 962 irq = platform_get_irq(op, i); 963 devm_free_irq(&op->dev, irq, phy); 964 } 965 } 966 967 dma_async_device_unregister(&pdev->device); 968 return 0; 969 } 970 971 static int mmp_pdma_chan_init(struct mmp_pdma_device *pdev, int idx, int irq) 972 { 973 struct mmp_pdma_phy *phy = &pdev->phy[idx]; 974 struct mmp_pdma_chan *chan; 975 int ret; 976 977 chan = devm_kzalloc(pdev->dev, sizeof(*chan), GFP_KERNEL); 978 if (chan == NULL) 979 return -ENOMEM; 980 981 phy->idx = idx; 982 phy->base = pdev->base; 983 984 if (irq) { 985 ret = devm_request_irq(pdev->dev, irq, mmp_pdma_chan_handler, 986 IRQF_SHARED, "pdma", phy); 987 if (ret) { 988 dev_err(pdev->dev, "channel request irq fail!\n"); 989 return ret; 990 } 991 } 992 993 spin_lock_init(&chan->desc_lock); 994 chan->dev = pdev->dev; 995 chan->chan.device = &pdev->device; 996 tasklet_init(&chan->tasklet, dma_do_tasklet, (unsigned long)chan); 997 INIT_LIST_HEAD(&chan->chain_pending); 998 INIT_LIST_HEAD(&chan->chain_running); 999 1000 /* register virt channel to dma engine */ 1001 list_add_tail(&chan->chan.device_node, &pdev->device.channels); 1002 1003 return 0; 1004 } 1005 1006 static const struct of_device_id mmp_pdma_dt_ids[] = { 1007 { .compatible = "marvell,pdma-1.0", }, 1008 {} 1009 }; 1010 MODULE_DEVICE_TABLE(of, mmp_pdma_dt_ids); 1011 1012 static struct dma_chan *mmp_pdma_dma_xlate(struct of_phandle_args *dma_spec, 1013 struct of_dma *ofdma) 1014 { 1015 struct mmp_pdma_device *d = ofdma->of_dma_data; 1016 struct dma_chan *chan; 1017 1018 chan = dma_get_any_slave_channel(&d->device); 1019 if (!chan) 1020 return NULL; 1021 1022 to_mmp_pdma_chan(chan)->drcmr = dma_spec->args[0]; 1023 1024 return chan; 1025 } 1026 1027 static int mmp_pdma_probe(struct platform_device *op) 1028 { 1029 struct mmp_pdma_device *pdev; 1030 const struct of_device_id *of_id; 1031 struct mmp_dma_platdata *pdata = dev_get_platdata(&op->dev); 1032 struct resource *iores; 1033 int i, ret, irq = 0; 1034 int dma_channels = 0, irq_num = 0; 1035 const enum dma_slave_buswidth widths = 1036 DMA_SLAVE_BUSWIDTH_1_BYTE | DMA_SLAVE_BUSWIDTH_2_BYTES | 1037 DMA_SLAVE_BUSWIDTH_4_BYTES; 1038 1039 pdev = devm_kzalloc(&op->dev, sizeof(*pdev), GFP_KERNEL); 1040 if (!pdev) 1041 return -ENOMEM; 1042 1043 pdev->dev = &op->dev; 1044 1045 spin_lock_init(&pdev->phy_lock); 1046 1047 iores = platform_get_resource(op, IORESOURCE_MEM, 0); 1048 pdev->base = devm_ioremap_resource(pdev->dev, iores); 1049 if (IS_ERR(pdev->base)) 1050 return PTR_ERR(pdev->base); 1051 1052 of_id = of_match_device(mmp_pdma_dt_ids, pdev->dev); 1053 if (of_id) 1054 of_property_read_u32(pdev->dev->of_node, "#dma-channels", 1055 &dma_channels); 1056 else if (pdata && pdata->dma_channels) 1057 dma_channels = pdata->dma_channels; 1058 else 1059 dma_channels = 32; /* default 32 channel */ 1060 pdev->dma_channels = dma_channels; 1061 1062 for (i = 0; i < dma_channels; i++) { 1063 if (platform_get_irq(op, i) > 0) 1064 irq_num++; 1065 } 1066 1067 pdev->phy = devm_kcalloc(pdev->dev, dma_channels, sizeof(*pdev->phy), 1068 GFP_KERNEL); 1069 if (pdev->phy == NULL) 1070 return -ENOMEM; 1071 1072 INIT_LIST_HEAD(&pdev->device.channels); 1073 1074 if (irq_num != dma_channels) { 1075 /* all chan share one irq, demux inside */ 1076 irq = platform_get_irq(op, 0); 1077 ret = devm_request_irq(pdev->dev, irq, mmp_pdma_int_handler, 1078 IRQF_SHARED, "pdma", pdev); 1079 if (ret) 1080 return ret; 1081 } 1082 1083 for (i = 0; i < dma_channels; i++) { 1084 irq = (irq_num != dma_channels) ? 0 : platform_get_irq(op, i); 1085 ret = mmp_pdma_chan_init(pdev, i, irq); 1086 if (ret) 1087 return ret; 1088 } 1089 1090 dma_cap_set(DMA_SLAVE, pdev->device.cap_mask); 1091 dma_cap_set(DMA_MEMCPY, pdev->device.cap_mask); 1092 dma_cap_set(DMA_CYCLIC, pdev->device.cap_mask); 1093 dma_cap_set(DMA_PRIVATE, pdev->device.cap_mask); 1094 pdev->device.dev = &op->dev; 1095 pdev->device.device_alloc_chan_resources = mmp_pdma_alloc_chan_resources; 1096 pdev->device.device_free_chan_resources = mmp_pdma_free_chan_resources; 1097 pdev->device.device_tx_status = mmp_pdma_tx_status; 1098 pdev->device.device_prep_dma_memcpy = mmp_pdma_prep_memcpy; 1099 pdev->device.device_prep_slave_sg = mmp_pdma_prep_slave_sg; 1100 pdev->device.device_prep_dma_cyclic = mmp_pdma_prep_dma_cyclic; 1101 pdev->device.device_issue_pending = mmp_pdma_issue_pending; 1102 pdev->device.device_config = mmp_pdma_config; 1103 pdev->device.device_terminate_all = mmp_pdma_terminate_all; 1104 pdev->device.copy_align = DMAENGINE_ALIGN_8_BYTES; 1105 pdev->device.src_addr_widths = widths; 1106 pdev->device.dst_addr_widths = widths; 1107 pdev->device.directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM); 1108 pdev->device.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; 1109 1110 if (pdev->dev->coherent_dma_mask) 1111 dma_set_mask(pdev->dev, pdev->dev->coherent_dma_mask); 1112 else 1113 dma_set_mask(pdev->dev, DMA_BIT_MASK(64)); 1114 1115 ret = dma_async_device_register(&pdev->device); 1116 if (ret) { 1117 dev_err(pdev->device.dev, "unable to register\n"); 1118 return ret; 1119 } 1120 1121 if (op->dev.of_node) { 1122 /* Device-tree DMA controller registration */ 1123 ret = of_dma_controller_register(op->dev.of_node, 1124 mmp_pdma_dma_xlate, pdev); 1125 if (ret < 0) { 1126 dev_err(&op->dev, "of_dma_controller_register failed\n"); 1127 return ret; 1128 } 1129 } 1130 1131 platform_set_drvdata(op, pdev); 1132 dev_info(pdev->device.dev, "initialized %d channels\n", dma_channels); 1133 return 0; 1134 } 1135 1136 static const struct platform_device_id mmp_pdma_id_table[] = { 1137 { "mmp-pdma", }, 1138 { }, 1139 }; 1140 1141 static struct platform_driver mmp_pdma_driver = { 1142 .driver = { 1143 .name = "mmp-pdma", 1144 .of_match_table = mmp_pdma_dt_ids, 1145 }, 1146 .id_table = mmp_pdma_id_table, 1147 .probe = mmp_pdma_probe, 1148 .remove = mmp_pdma_remove, 1149 }; 1150 1151 bool mmp_pdma_filter_fn(struct dma_chan *chan, void *param) 1152 { 1153 struct mmp_pdma_chan *c = to_mmp_pdma_chan(chan); 1154 1155 if (chan->device->dev->driver != &mmp_pdma_driver.driver) 1156 return false; 1157 1158 c->drcmr = *(unsigned int *)param; 1159 1160 return true; 1161 } 1162 EXPORT_SYMBOL_GPL(mmp_pdma_filter_fn); 1163 1164 module_platform_driver(mmp_pdma_driver); 1165 1166 MODULE_DESCRIPTION("MARVELL MMP Peripheral DMA Driver"); 1167 MODULE_AUTHOR("Marvell International Ltd."); 1168 MODULE_LICENSE("GPL v2"); 1169