1 /*- 2 * Copyright (c) 2018 Ruslan Bukin <br@bsdpad.com> 3 * All rights reserved. 4 * 5 * This software was developed by SRI International and the University of 6 * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237 7 * ("CTSRD"), as part of the DARPA CRASH research programme. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include "opt_platform.h" 35 #include <sys/param.h> 36 #include <sys/conf.h> 37 #include <sys/bus.h> 38 #include <sys/kernel.h> 39 #include <sys/malloc.h> 40 #include <sys/mbuf.h> 41 #include <sys/sx.h> 42 43 #include <machine/bus.h> 44 45 #ifdef FDT 46 #include <dev/fdt/fdt_common.h> 47 #include <dev/ofw/ofw_bus.h> 48 #include <dev/ofw/ofw_bus_subr.h> 49 #endif 50 51 #include <dev/xdma/xdma.h> 52 53 #include <xdma_if.h> 54 55 struct seg_load_request { 56 struct bus_dma_segment *seg; 57 uint32_t nsegs; 58 uint32_t error; 59 }; 60 61 static int 62 _xchan_bufs_alloc(xdma_channel_t *xchan) 63 { 64 xdma_controller_t *xdma; 65 struct xdma_request *xr; 66 int i; 67 68 xdma = xchan->xdma; 69 70 for (i = 0; i < xchan->xr_num; i++) { 71 xr = &xchan->xr_mem[i]; 72 xr->buf.cbuf = contigmalloc(xchan->maxsegsize, 73 M_XDMA, 0, 0, ~0, PAGE_SIZE, 0); 74 if (xr->buf.cbuf == NULL) { 75 device_printf(xdma->dev, 76 "%s: Can't allocate contiguous kernel" 77 " physical memory\n", __func__); 78 return (-1); 79 } 80 } 81 82 return (0); 83 } 84 85 static int 86 _xchan_bufs_alloc_busdma(xdma_channel_t *xchan) 87 { 88 xdma_controller_t *xdma; 89 struct xdma_request *xr; 90 int err; 91 int i; 92 93 xdma = xchan->xdma; 94 95 /* Create bus_dma tag */ 96 err = bus_dma_tag_create( 97 bus_get_dma_tag(xdma->dev), /* Parent tag. */ 98 xchan->alignment, /* alignment */ 99 xchan->boundary, /* boundary */ 100 xchan->lowaddr, /* lowaddr */ 101 xchan->highaddr, /* highaddr */ 102 NULL, NULL, /* filter, filterarg */ 103 xchan->maxsegsize * xchan->maxnsegs, /* maxsize */ 104 xchan->maxnsegs, /* nsegments */ 105 xchan->maxsegsize, /* maxsegsize */ 106 0, /* flags */ 107 NULL, NULL, /* lockfunc, lockarg */ 108 &xchan->dma_tag_bufs); 109 if (err != 0) { 110 device_printf(xdma->dev, 111 "%s: Can't create bus_dma tag.\n", __func__); 112 return (-1); 113 } 114 115 for (i = 0; i < xchan->xr_num; i++) { 116 xr = &xchan->xr_mem[i]; 117 err = bus_dmamap_create(xchan->dma_tag_bufs, 0, 118 &xr->buf.map); 119 if (err != 0) { 120 device_printf(xdma->dev, 121 "%s: Can't create buf DMA map.\n", __func__); 122 123 /* Cleanup. */ 124 bus_dma_tag_destroy(xchan->dma_tag_bufs); 125 126 return (-1); 127 } 128 } 129 130 return (0); 131 } 132 133 static int 134 xchan_bufs_alloc(xdma_channel_t *xchan) 135 { 136 xdma_controller_t *xdma; 137 int ret; 138 139 xdma = xchan->xdma; 140 141 if (xdma == NULL) { 142 device_printf(xdma->dev, 143 "%s: Channel was not allocated properly.\n", __func__); 144 return (-1); 145 } 146 147 if (xchan->caps & XCHAN_CAP_BUSDMA) 148 ret = _xchan_bufs_alloc_busdma(xchan); 149 else 150 ret = _xchan_bufs_alloc(xchan); 151 if (ret != 0) { 152 device_printf(xdma->dev, 153 "%s: Can't allocate bufs.\n", __func__); 154 return (-1); 155 } 156 157 xchan->flags |= XCHAN_BUFS_ALLOCATED; 158 159 return (0); 160 } 161 162 static int 163 xchan_bufs_free(xdma_channel_t *xchan) 164 { 165 struct xdma_request *xr; 166 struct xchan_buf *b; 167 int i; 168 169 if ((xchan->flags & XCHAN_BUFS_ALLOCATED) == 0) 170 return (-1); 171 172 if (xchan->caps & XCHAN_CAP_BUSDMA) { 173 for (i = 0; i < xchan->xr_num; i++) { 174 xr = &xchan->xr_mem[i]; 175 b = &xr->buf; 176 bus_dmamap_destroy(xchan->dma_tag_bufs, b->map); 177 } 178 bus_dma_tag_destroy(xchan->dma_tag_bufs); 179 } else { 180 for (i = 0; i < xchan->xr_num; i++) { 181 xr = &xchan->xr_mem[i]; 182 contigfree(xr->buf.cbuf, xchan->maxsegsize, M_XDMA); 183 } 184 } 185 186 xchan->flags &= ~XCHAN_BUFS_ALLOCATED; 187 188 return (0); 189 } 190 191 void 192 xdma_channel_free_sg(xdma_channel_t *xchan) 193 { 194 195 xchan_bufs_free(xchan); 196 xchan_sglist_free(xchan); 197 xchan_bank_free(xchan); 198 } 199 200 /* 201 * Prepare xchan for a scatter-gather transfer. 202 * xr_num - xdma requests queue size, 203 * maxsegsize - maximum allowed scatter-gather list element size in bytes 204 */ 205 int 206 xdma_prep_sg(xdma_channel_t *xchan, uint32_t xr_num, 207 bus_size_t maxsegsize, bus_size_t maxnsegs, 208 bus_size_t alignment, bus_addr_t boundary, 209 bus_addr_t lowaddr, bus_addr_t highaddr) 210 { 211 xdma_controller_t *xdma; 212 int ret; 213 214 xdma = xchan->xdma; 215 216 KASSERT(xdma != NULL, ("xdma is NULL")); 217 218 if (xchan->flags & XCHAN_CONFIGURED) { 219 device_printf(xdma->dev, 220 "%s: Channel is already configured.\n", __func__); 221 return (-1); 222 } 223 224 xchan->xr_num = xr_num; 225 xchan->maxsegsize = maxsegsize; 226 xchan->maxnsegs = maxnsegs; 227 xchan->alignment = alignment; 228 xchan->boundary = boundary; 229 xchan->lowaddr = lowaddr; 230 xchan->highaddr = highaddr; 231 232 if (xchan->maxnsegs > XDMA_MAX_SEG) { 233 device_printf(xdma->dev, "%s: maxnsegs is too big\n", 234 __func__); 235 return (-1); 236 } 237 238 xchan_bank_init(xchan); 239 240 /* Allocate sglist. */ 241 ret = xchan_sglist_alloc(xchan); 242 if (ret != 0) { 243 device_printf(xdma->dev, 244 "%s: Can't allocate sglist.\n", __func__); 245 return (-1); 246 } 247 248 /* Allocate bufs. */ 249 ret = xchan_bufs_alloc(xchan); 250 if (ret != 0) { 251 device_printf(xdma->dev, 252 "%s: Can't allocate bufs.\n", __func__); 253 254 /* Cleanup */ 255 xchan_sglist_free(xchan); 256 xchan_bank_free(xchan); 257 258 return (-1); 259 } 260 261 xchan->flags |= (XCHAN_CONFIGURED | XCHAN_TYPE_SG); 262 263 XCHAN_LOCK(xchan); 264 ret = XDMA_CHANNEL_PREP_SG(xdma->dma_dev, xchan); 265 if (ret != 0) { 266 device_printf(xdma->dev, 267 "%s: Can't prepare SG transfer.\n", __func__); 268 XCHAN_UNLOCK(xchan); 269 270 return (-1); 271 } 272 XCHAN_UNLOCK(xchan); 273 274 return (0); 275 } 276 277 void 278 xchan_seg_done(xdma_channel_t *xchan, 279 struct xdma_transfer_status *st) 280 { 281 struct xdma_request *xr; 282 xdma_controller_t *xdma; 283 struct xchan_buf *b; 284 285 xdma = xchan->xdma; 286 287 xr = TAILQ_FIRST(&xchan->processing); 288 if (xr == NULL) 289 panic("request not found\n"); 290 291 b = &xr->buf; 292 293 atomic_subtract_int(&b->nsegs_left, 1); 294 295 if (b->nsegs_left == 0) { 296 if (xchan->caps & XCHAN_CAP_BUSDMA) { 297 if (xr->direction == XDMA_MEM_TO_DEV) 298 bus_dmamap_sync(xchan->dma_tag_bufs, b->map, 299 BUS_DMASYNC_POSTWRITE); 300 else 301 bus_dmamap_sync(xchan->dma_tag_bufs, b->map, 302 BUS_DMASYNC_POSTREAD); 303 bus_dmamap_unload(xchan->dma_tag_bufs, b->map); 304 } 305 xr->status.error = st->error; 306 xr->status.transferred = st->transferred; 307 308 QUEUE_PROC_LOCK(xchan); 309 TAILQ_REMOVE(&xchan->processing, xr, xr_next); 310 QUEUE_PROC_UNLOCK(xchan); 311 312 QUEUE_OUT_LOCK(xchan); 313 TAILQ_INSERT_TAIL(&xchan->queue_out, xr, xr_next); 314 QUEUE_OUT_UNLOCK(xchan); 315 } 316 } 317 318 static void 319 xdma_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error) 320 { 321 struct seg_load_request *slr; 322 struct bus_dma_segment *seg; 323 int i; 324 325 slr = arg; 326 seg = slr->seg; 327 328 if (error != 0) { 329 slr->error = error; 330 return; 331 } 332 333 slr->nsegs = nsegs; 334 335 for (i = 0; i < nsegs; i++) { 336 seg[i].ds_addr = segs[i].ds_addr; 337 seg[i].ds_len = segs[i].ds_len; 338 } 339 } 340 341 static int 342 _xdma_load_data_busdma(xdma_channel_t *xchan, struct xdma_request *xr, 343 struct bus_dma_segment *seg) 344 { 345 xdma_controller_t *xdma; 346 struct seg_load_request slr; 347 uint32_t nsegs; 348 void *addr; 349 int error; 350 351 xdma = xchan->xdma; 352 353 error = 0; 354 nsegs = 0; 355 356 switch (xr->req_type) { 357 case XR_TYPE_MBUF: 358 error = bus_dmamap_load_mbuf_sg(xchan->dma_tag_bufs, 359 xr->buf.map, xr->m, seg, &nsegs, BUS_DMA_NOWAIT); 360 break; 361 case XR_TYPE_BIO: 362 slr.nsegs = 0; 363 slr.error = 0; 364 slr.seg = seg; 365 error = bus_dmamap_load_bio(xchan->dma_tag_bufs, 366 xr->buf.map, xr->bp, xdma_dmamap_cb, &slr, BUS_DMA_NOWAIT); 367 if (slr.error != 0) { 368 device_printf(xdma->dma_dev, 369 "%s: bus_dmamap_load failed, err %d\n", 370 __func__, slr.error); 371 return (0); 372 } 373 nsegs = slr.nsegs; 374 break; 375 case XR_TYPE_VIRT: 376 switch (xr->direction) { 377 case XDMA_MEM_TO_DEV: 378 addr = (void *)xr->src_addr; 379 break; 380 case XDMA_DEV_TO_MEM: 381 addr = (void *)xr->dst_addr; 382 break; 383 default: 384 device_printf(xdma->dma_dev, 385 "%s: Direction is not supported\n", __func__); 386 return (0); 387 } 388 slr.nsegs = 0; 389 slr.error = 0; 390 slr.seg = seg; 391 error = bus_dmamap_load(xchan->dma_tag_bufs, xr->buf.map, 392 addr, (xr->block_len * xr->block_num), 393 xdma_dmamap_cb, &slr, BUS_DMA_NOWAIT); 394 if (slr.error != 0) { 395 device_printf(xdma->dma_dev, 396 "%s: bus_dmamap_load failed, err %d\n", 397 __func__, slr.error); 398 return (0); 399 } 400 nsegs = slr.nsegs; 401 break; 402 default: 403 break; 404 } 405 406 if (error != 0) { 407 if (error == ENOMEM) { 408 /* 409 * Out of memory. Try again later. 410 * TODO: count errors. 411 */ 412 } else 413 device_printf(xdma->dma_dev, 414 "%s: bus_dmamap_load failed with err %d\n", 415 __func__, error); 416 return (0); 417 } 418 419 if (xr->direction == XDMA_MEM_TO_DEV) 420 bus_dmamap_sync(xchan->dma_tag_bufs, xr->buf.map, 421 BUS_DMASYNC_PREWRITE); 422 else 423 bus_dmamap_sync(xchan->dma_tag_bufs, xr->buf.map, 424 BUS_DMASYNC_PREREAD); 425 426 return (nsegs); 427 } 428 429 static int 430 _xdma_load_data(xdma_channel_t *xchan, struct xdma_request *xr, 431 struct bus_dma_segment *seg) 432 { 433 xdma_controller_t *xdma; 434 struct mbuf *m; 435 uint32_t nsegs; 436 437 xdma = xchan->xdma; 438 439 m = xr->m; 440 441 nsegs = 1; 442 443 switch (xr->req_type) { 444 case XR_TYPE_MBUF: 445 if (xr->direction == XDMA_MEM_TO_DEV) { 446 m_copydata(m, 0, m->m_pkthdr.len, xr->buf.cbuf); 447 seg[0].ds_addr = (bus_addr_t)xr->buf.cbuf; 448 seg[0].ds_len = m->m_pkthdr.len; 449 } else { 450 seg[0].ds_addr = mtod(m, bus_addr_t); 451 seg[0].ds_len = m->m_pkthdr.len; 452 } 453 break; 454 case XR_TYPE_BIO: 455 case XR_TYPE_VIRT: 456 default: 457 panic("implement me\n"); 458 } 459 460 return (nsegs); 461 } 462 463 static int 464 xdma_load_data(xdma_channel_t *xchan, 465 struct xdma_request *xr, struct bus_dma_segment *seg) 466 { 467 xdma_controller_t *xdma; 468 int error; 469 int nsegs; 470 471 xdma = xchan->xdma; 472 473 error = 0; 474 nsegs = 0; 475 476 if (xchan->caps & XCHAN_CAP_BUSDMA) 477 nsegs = _xdma_load_data_busdma(xchan, xr, seg); 478 else 479 nsegs = _xdma_load_data(xchan, xr, seg); 480 if (nsegs == 0) 481 return (0); /* Try again later. */ 482 483 xr->buf.nsegs = nsegs; 484 xr->buf.nsegs_left = nsegs; 485 486 return (nsegs); 487 } 488 489 static int 490 xdma_process(xdma_channel_t *xchan, 491 struct xdma_sglist *sg) 492 { 493 struct bus_dma_segment seg[XDMA_MAX_SEG]; 494 struct xdma_request *xr; 495 struct xdma_request *xr_tmp; 496 xdma_controller_t *xdma; 497 uint32_t capacity; 498 uint32_t n; 499 uint32_t c; 500 int nsegs; 501 int ret; 502 503 XCHAN_ASSERT_LOCKED(xchan); 504 505 xdma = xchan->xdma; 506 507 n = 0; 508 509 ret = XDMA_CHANNEL_CAPACITY(xdma->dma_dev, xchan, &capacity); 510 if (ret != 0) { 511 device_printf(xdma->dev, 512 "%s: Can't get DMA controller capacity.\n", __func__); 513 return (-1); 514 } 515 516 TAILQ_FOREACH_SAFE(xr, &xchan->queue_in, xr_next, xr_tmp) { 517 switch (xr->req_type) { 518 case XR_TYPE_MBUF: 519 c = xdma_mbuf_defrag(xchan, xr); 520 break; 521 case XR_TYPE_BIO: 522 case XR_TYPE_VIRT: 523 default: 524 c = 1; 525 } 526 527 if (capacity <= (c + n)) { 528 /* 529 * No space yet available for the entire 530 * request in the DMA engine. 531 */ 532 break; 533 } 534 535 if ((c + n + xchan->maxnsegs) >= XDMA_SGLIST_MAXLEN) { 536 /* Sglist is full. */ 537 break; 538 } 539 540 nsegs = xdma_load_data(xchan, xr, seg); 541 if (nsegs == 0) 542 break; 543 544 xdma_sglist_add(&sg[n], seg, nsegs, xr); 545 n += nsegs; 546 547 QUEUE_IN_LOCK(xchan); 548 TAILQ_REMOVE(&xchan->queue_in, xr, xr_next); 549 QUEUE_IN_UNLOCK(xchan); 550 551 QUEUE_PROC_LOCK(xchan); 552 TAILQ_INSERT_TAIL(&xchan->processing, xr, xr_next); 553 QUEUE_PROC_UNLOCK(xchan); 554 } 555 556 return (n); 557 } 558 559 int 560 xdma_queue_submit_sg(xdma_channel_t *xchan) 561 { 562 struct xdma_sglist *sg; 563 xdma_controller_t *xdma; 564 uint32_t sg_n; 565 int ret; 566 567 xdma = xchan->xdma; 568 KASSERT(xdma != NULL, ("xdma is NULL")); 569 570 XCHAN_ASSERT_LOCKED(xchan); 571 572 sg = xchan->sg; 573 574 if ((xchan->flags & XCHAN_BUFS_ALLOCATED) == 0) { 575 device_printf(xdma->dev, 576 "%s: Can't submit a transfer: no bufs\n", 577 __func__); 578 return (-1); 579 } 580 581 sg_n = xdma_process(xchan, sg); 582 if (sg_n == 0) 583 return (0); /* Nothing to submit */ 584 585 /* Now submit sglist to DMA engine driver. */ 586 ret = XDMA_CHANNEL_SUBMIT_SG(xdma->dma_dev, xchan, sg, sg_n); 587 if (ret != 0) { 588 device_printf(xdma->dev, 589 "%s: Can't submit an sglist.\n", __func__); 590 return (-1); 591 } 592 593 return (0); 594 } 595