1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2012 Oleksandr Tymoshenko <gonzo@freebsd.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 */ 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/bus.h> 35 #include <sys/conf.h> 36 #include <sys/kernel.h> 37 #include <sys/lock.h> 38 #include <sys/malloc.h> 39 #include <sys/module.h> 40 #include <sys/mutex.h> 41 #include <sys/rman.h> 42 #include <sys/sysctl.h> 43 #include <sys/taskqueue.h> 44 45 #include <machine/bus.h> 46 47 #include <dev/ofw/ofw_bus.h> 48 #include <dev/ofw/ofw_bus_subr.h> 49 50 #include <dev/mmc/bridge.h> 51 #include <dev/mmc/mmcreg.h> 52 #include <dev/mmc/mmc_fdt_helpers.h> 53 54 #include <dev/sdhci/sdhci.h> 55 56 #include "mmcbr_if.h" 57 #include "sdhci_if.h" 58 59 #include "opt_mmccam.h" 60 61 #include "bcm2835_dma.h" 62 #include <arm/broadcom/bcm2835/bcm2835_mbox_prop.h> 63 #ifdef NOTYET 64 #include <arm/broadcom/bcm2835/bcm2835_clkman.h> 65 #endif 66 #include <arm/broadcom/bcm2835/bcm2835_vcbus.h> 67 68 #define BCM2835_DEFAULT_SDHCI_FREQ 50 69 #define BCM2838_DEFAULT_SDHCI_FREQ 100 70 71 #define BCM_SDHCI_BUFFER_SIZE 512 72 /* 73 * NUM_DMA_SEGS is the number of DMA segments we want to accommodate on average. 74 * We add in a number of segments based on how much we may need to spill into 75 * another segment due to crossing page boundaries. e.g. up to PAGE_SIZE, an 76 * extra page is needed as we can cross a page boundary exactly once. 77 */ 78 #define NUM_DMA_SEGS 1 79 #define NUM_DMA_SPILL_SEGS \ 80 ((((NUM_DMA_SEGS * BCM_SDHCI_BUFFER_SIZE) - 1) / PAGE_SIZE) + 1) 81 #define ALLOCATED_DMA_SEGS (NUM_DMA_SEGS + NUM_DMA_SPILL_SEGS) 82 #define BCM_DMA_MAXSIZE (NUM_DMA_SEGS * BCM_SDHCI_BUFFER_SIZE) 83 84 #define BCM_SDHCI_SLOT_LEFT(slot) \ 85 ((slot)->curcmd->data->len - (slot)->offset) 86 87 #define BCM_SDHCI_SEGSZ_LEFT(slot) \ 88 min(BCM_DMA_MAXSIZE, \ 89 rounddown(BCM_SDHCI_SLOT_LEFT(slot), BCM_SDHCI_BUFFER_SIZE)) 90 91 #define DATA_PENDING_MASK (SDHCI_INT_DATA_AVAIL | SDHCI_INT_SPACE_AVAIL) 92 #define DATA_XFER_MASK (DATA_PENDING_MASK | SDHCI_INT_DATA_END) 93 94 #ifdef DEBUG 95 static int bcm2835_sdhci_debug = 0; 96 97 TUNABLE_INT("hw.bcm2835.sdhci.debug", &bcm2835_sdhci_debug); 98 SYSCTL_INT(_hw_sdhci, OID_AUTO, bcm2835_sdhci_debug, CTLFLAG_RWTUN, 99 &bcm2835_sdhci_debug, 0, "bcm2835 SDHCI debug level"); 100 101 #define dprintf(fmt, args...) \ 102 do { \ 103 if (bcm2835_sdhci_debug) \ 104 printf("%s: " fmt, __func__, ##args); \ 105 } while (0) 106 #else 107 #define dprintf(fmt, args...) 108 #endif 109 110 static int bcm2835_sdhci_hs = 1; 111 static int bcm2835_sdhci_pio_mode = 0; 112 113 struct bcm_mmc_conf { 114 int clock_id; 115 int clock_src; 116 int default_freq; 117 int quirks; 118 int emmc_dreq; 119 }; 120 121 struct bcm_mmc_conf bcm2835_sdhci_conf = { 122 .clock_id = BCM2835_MBOX_CLOCK_ID_EMMC, 123 .clock_src = -1, 124 .default_freq = BCM2835_DEFAULT_SDHCI_FREQ, 125 .quirks = SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK | 126 SDHCI_QUIRK_BROKEN_TIMEOUT_VAL | SDHCI_QUIRK_DONT_SET_HISPD_BIT | 127 SDHCI_QUIRK_MISSING_CAPS, 128 .emmc_dreq = BCM_DMA_DREQ_EMMC, 129 }; 130 131 struct bcm_mmc_conf bcm2838_emmc2_conf = { 132 .clock_id = BCM2838_MBOX_CLOCK_ID_EMMC2, 133 .clock_src = -1, 134 .default_freq = BCM2838_DEFAULT_SDHCI_FREQ, 135 .quirks = 0, 136 .emmc_dreq = BCM_DMA_DREQ_NONE, 137 }; 138 139 static struct ofw_compat_data compat_data[] = { 140 {"broadcom,bcm2835-sdhci", (uintptr_t)&bcm2835_sdhci_conf}, 141 {"brcm,bcm2835-sdhci", (uintptr_t)&bcm2835_sdhci_conf}, 142 {"brcm,bcm2835-mmc", (uintptr_t)&bcm2835_sdhci_conf}, 143 {"brcm,bcm2711-emmc2", (uintptr_t)&bcm2838_emmc2_conf}, 144 {"brcm,bcm2838-emmc2", (uintptr_t)&bcm2838_emmc2_conf}, 145 {NULL, 0} 146 }; 147 148 TUNABLE_INT("hw.bcm2835.sdhci.hs", &bcm2835_sdhci_hs); 149 TUNABLE_INT("hw.bcm2835.sdhci.pio_mode", &bcm2835_sdhci_pio_mode); 150 151 struct bcm_sdhci_softc { 152 device_t sc_dev; 153 struct resource * sc_mem_res; 154 struct resource * sc_irq_res; 155 bus_space_tag_t sc_bst; 156 bus_space_handle_t sc_bsh; 157 void * sc_intrhand; 158 struct mmc_request * sc_req; 159 struct sdhci_slot sc_slot; 160 struct mmc_helper sc_mmc_helper; 161 int sc_dma_ch; 162 bus_dma_tag_t sc_dma_tag; 163 bus_dmamap_t sc_dma_map; 164 vm_paddr_t sc_sdhci_buffer_phys; 165 bus_addr_t dmamap_seg_addrs[ALLOCATED_DMA_SEGS]; 166 bus_size_t dmamap_seg_sizes[ALLOCATED_DMA_SEGS]; 167 int dmamap_seg_count; 168 int dmamap_seg_index; 169 int dmamap_status; 170 uint32_t blksz_and_count; 171 uint32_t cmd_and_mode; 172 bool need_update_blk; 173 #ifdef NOTYET 174 device_t clkman; 175 #endif 176 struct bcm_mmc_conf * conf; 177 }; 178 179 static int bcm_sdhci_probe(device_t); 180 static int bcm_sdhci_attach(device_t); 181 static int bcm_sdhci_detach(device_t); 182 static void bcm_sdhci_intr(void *); 183 184 static int bcm_sdhci_get_ro(device_t, device_t); 185 static void bcm_sdhci_dma_intr(int ch, void *arg); 186 static void bcm_sdhci_start_dma(struct sdhci_slot *slot); 187 188 static void 189 bcm_sdhci_dmacb(void *arg, bus_dma_segment_t *segs, int nseg, int err) 190 { 191 struct bcm_sdhci_softc *sc = arg; 192 int i; 193 194 /* Sanity check: we can only ever have one mapping at a time. */ 195 KASSERT(sc->dmamap_seg_count == 0, ("leaked DMA segment")); 196 sc->dmamap_status = err; 197 sc->dmamap_seg_count = nseg; 198 199 /* Note nseg is guaranteed to be zero if err is non-zero. */ 200 for (i = 0; i < nseg; i++) { 201 sc->dmamap_seg_addrs[i] = segs[i].ds_addr; 202 sc->dmamap_seg_sizes[i] = segs[i].ds_len; 203 } 204 } 205 206 static int 207 bcm_sdhci_probe(device_t dev) 208 { 209 210 if (!ofw_bus_status_okay(dev)) 211 return (ENXIO); 212 213 if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0) 214 return (ENXIO); 215 216 device_set_desc(dev, "Broadcom 2708 SDHCI controller"); 217 218 return (BUS_PROBE_DEFAULT); 219 } 220 221 static int 222 bcm_sdhci_attach(device_t dev) 223 { 224 struct bcm_sdhci_softc *sc = device_get_softc(dev); 225 int rid, err; 226 phandle_t node; 227 pcell_t cell; 228 u_int default_freq; 229 230 sc->sc_dev = dev; 231 sc->sc_req = NULL; 232 233 sc->conf = (struct bcm_mmc_conf *)ofw_bus_search_compatible(dev, 234 compat_data)->ocd_data; 235 if (sc->conf == 0) 236 return (ENXIO); 237 238 err = bcm2835_mbox_set_power_state(BCM2835_MBOX_POWER_ID_EMMC, TRUE); 239 if (err != 0) { 240 if (bootverbose) 241 device_printf(dev, "Unable to enable the power\n"); 242 return (err); 243 } 244 245 default_freq = 0; 246 err = bcm2835_mbox_get_clock_rate(sc->conf->clock_id, &default_freq); 247 if (err == 0) { 248 /* Convert to MHz */ 249 default_freq /= 1000000; 250 } 251 if (default_freq == 0) { 252 node = ofw_bus_get_node(sc->sc_dev); 253 if ((OF_getencprop(node, "clock-frequency", &cell, 254 sizeof(cell))) > 0) 255 default_freq = cell / 1000000; 256 } 257 if (default_freq == 0) 258 default_freq = sc->conf->default_freq; 259 260 if (bootverbose) 261 device_printf(dev, "SDHCI frequency: %dMHz\n", default_freq); 262 #ifdef NOTYET 263 if (sc->conf->clock_src > 0) { 264 uint32_t f; 265 sc->clkman = devclass_get_device( 266 devclass_find("bcm2835_clkman"), 0); 267 if (sc->clkman == NULL) { 268 device_printf(dev, "cannot find Clock Manager\n"); 269 return (ENXIO); 270 } 271 272 f = bcm2835_clkman_set_frequency(sc->clkman, 273 sc->conf->clock_src, default_freq); 274 if (f == 0) 275 return (EINVAL); 276 277 if (bootverbose) 278 device_printf(dev, "Clock source frequency: %dMHz\n", 279 f); 280 } 281 #endif 282 283 rid = 0; 284 sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 285 RF_ACTIVE); 286 if (!sc->sc_mem_res) { 287 device_printf(dev, "cannot allocate memory window\n"); 288 err = ENXIO; 289 goto fail; 290 } 291 292 sc->sc_bst = rman_get_bustag(sc->sc_mem_res); 293 sc->sc_bsh = rman_get_bushandle(sc->sc_mem_res); 294 295 rid = 0; 296 sc->sc_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 297 RF_ACTIVE | RF_SHAREABLE); 298 if (!sc->sc_irq_res) { 299 device_printf(dev, "cannot allocate interrupt\n"); 300 err = ENXIO; 301 goto fail; 302 } 303 304 if (bus_setup_intr(dev, sc->sc_irq_res, INTR_TYPE_BIO | INTR_MPSAFE, 305 NULL, bcm_sdhci_intr, sc, &sc->sc_intrhand)) { 306 device_printf(dev, "cannot setup interrupt handler\n"); 307 err = ENXIO; 308 goto fail; 309 } 310 311 if (!bcm2835_sdhci_pio_mode) 312 sc->sc_slot.opt = SDHCI_PLATFORM_TRANSFER; 313 314 sc->sc_slot.caps = SDHCI_CAN_VDD_330 | SDHCI_CAN_VDD_180; 315 if (bcm2835_sdhci_hs) 316 sc->sc_slot.caps |= SDHCI_CAN_DO_HISPD; 317 sc->sc_slot.caps |= (default_freq << SDHCI_CLOCK_BASE_SHIFT); 318 sc->sc_slot.quirks = sc->conf->quirks; 319 320 sdhci_init_slot(dev, &sc->sc_slot, 0); 321 mmc_fdt_parse(dev, 0, &sc->sc_mmc_helper, &sc->sc_slot.host); 322 323 sc->sc_dma_ch = bcm_dma_allocate(BCM_DMA_CH_ANY); 324 if (sc->sc_dma_ch == BCM_DMA_CH_INVALID) 325 goto fail; 326 327 err = bcm_dma_setup_intr(sc->sc_dma_ch, bcm_sdhci_dma_intr, sc); 328 if (err != 0) { 329 device_printf(dev, 330 "cannot setup dma interrupt handler\n"); 331 err = ENXIO; 332 goto fail; 333 } 334 335 /* Allocate bus_dma resources. */ 336 err = bus_dma_tag_create(bus_get_dma_tag(dev), 337 1, 0, bcm283x_dmabus_peripheral_lowaddr(), 338 BUS_SPACE_MAXADDR, NULL, NULL, 339 BCM_DMA_MAXSIZE, ALLOCATED_DMA_SEGS, BCM_SDHCI_BUFFER_SIZE, 340 BUS_DMA_ALLOCNOW, NULL, NULL, 341 &sc->sc_dma_tag); 342 343 if (err) { 344 device_printf(dev, "failed allocate DMA tag"); 345 goto fail; 346 } 347 348 err = bus_dmamap_create(sc->sc_dma_tag, 0, &sc->sc_dma_map); 349 if (err) { 350 device_printf(dev, "bus_dmamap_create failed\n"); 351 goto fail; 352 } 353 354 /* FIXME: Fix along with other BUS_SPACE_PHYSADDR instances */ 355 sc->sc_sdhci_buffer_phys = rman_get_start(sc->sc_mem_res) + 356 SDHCI_BUFFER; 357 358 bus_generic_probe(dev); 359 bus_generic_attach(dev); 360 361 sdhci_start_slot(&sc->sc_slot); 362 363 /* Seed our copies. */ 364 sc->blksz_and_count = SDHCI_READ_4(dev, &sc->sc_slot, SDHCI_BLOCK_SIZE); 365 sc->cmd_and_mode = SDHCI_READ_4(dev, &sc->sc_slot, SDHCI_TRANSFER_MODE); 366 367 return (0); 368 369 fail: 370 if (sc->sc_intrhand) 371 bus_teardown_intr(dev, sc->sc_irq_res, sc->sc_intrhand); 372 if (sc->sc_irq_res) 373 bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq_res); 374 if (sc->sc_mem_res) 375 bus_release_resource(dev, SYS_RES_MEMORY, 0, sc->sc_mem_res); 376 377 return (err); 378 } 379 380 static int 381 bcm_sdhci_detach(device_t dev) 382 { 383 384 return (EBUSY); 385 } 386 387 static void 388 bcm_sdhci_intr(void *arg) 389 { 390 struct bcm_sdhci_softc *sc = arg; 391 392 sdhci_generic_intr(&sc->sc_slot); 393 } 394 395 static int 396 bcm_sdhci_update_ios(device_t bus, device_t child) 397 { 398 struct bcm_sdhci_softc *sc; 399 struct mmc_ios *ios; 400 int rv; 401 402 sc = device_get_softc(bus); 403 ios = &sc->sc_slot.host.ios; 404 405 if (ios->power_mode == power_up) { 406 if (sc->sc_mmc_helper.vmmc_supply) 407 regulator_enable(sc->sc_mmc_helper.vmmc_supply); 408 if (sc->sc_mmc_helper.vqmmc_supply) 409 regulator_enable(sc->sc_mmc_helper.vqmmc_supply); 410 } 411 412 rv = sdhci_generic_update_ios(bus, child); 413 if (rv != 0) 414 return (rv); 415 416 if (ios->power_mode == power_off) { 417 if (sc->sc_mmc_helper.vmmc_supply) 418 regulator_disable(sc->sc_mmc_helper.vmmc_supply); 419 if (sc->sc_mmc_helper.vqmmc_supply) 420 regulator_disable(sc->sc_mmc_helper.vqmmc_supply); 421 } 422 423 return (0); 424 } 425 426 static int 427 bcm_sdhci_get_ro(device_t bus, device_t child) 428 { 429 430 return (0); 431 } 432 433 static inline uint32_t 434 RD4(struct bcm_sdhci_softc *sc, bus_size_t off) 435 { 436 uint32_t val = bus_space_read_4(sc->sc_bst, sc->sc_bsh, off); 437 return val; 438 } 439 440 static inline void 441 WR4(struct bcm_sdhci_softc *sc, bus_size_t off, uint32_t val) 442 { 443 444 bus_space_write_4(sc->sc_bst, sc->sc_bsh, off, val); 445 /* 446 * The Arasan HC has a bug where it may lose the content of 447 * consecutive writes to registers that are within two SD-card 448 * clock cycles of each other (a clock domain crossing problem). 449 */ 450 if (sc->sc_slot.clock > 0) 451 DELAY(((2 * 1000000) / sc->sc_slot.clock) + 1); 452 } 453 454 static uint8_t 455 bcm_sdhci_read_1(device_t dev, struct sdhci_slot *slot, bus_size_t off) 456 { 457 struct bcm_sdhci_softc *sc = device_get_softc(dev); 458 uint32_t val = RD4(sc, off & ~3); 459 460 return ((val >> (off & 3)*8) & 0xff); 461 } 462 463 static uint16_t 464 bcm_sdhci_read_2(device_t dev, struct sdhci_slot *slot, bus_size_t off) 465 { 466 struct bcm_sdhci_softc *sc = device_get_softc(dev); 467 uint32_t val32; 468 469 /* 470 * Standard 32-bit handling of command and transfer mode, as 471 * well as block size and count. 472 */ 473 if ((off == SDHCI_BLOCK_SIZE || off == SDHCI_BLOCK_COUNT) && 474 sc->need_update_blk) 475 val32 = sc->blksz_and_count; 476 else if (off == SDHCI_TRANSFER_MODE || off == SDHCI_COMMAND_FLAGS) 477 val32 = sc->cmd_and_mode; 478 else 479 val32 = RD4(sc, off & ~3); 480 481 return ((val32 >> (off & 3)*8) & 0xffff); 482 } 483 484 static uint32_t 485 bcm_sdhci_read_4(device_t dev, struct sdhci_slot *slot, bus_size_t off) 486 { 487 struct bcm_sdhci_softc *sc = device_get_softc(dev); 488 489 return RD4(sc, off); 490 } 491 492 static void 493 bcm_sdhci_read_multi_4(device_t dev, struct sdhci_slot *slot, bus_size_t off, 494 uint32_t *data, bus_size_t count) 495 { 496 struct bcm_sdhci_softc *sc = device_get_softc(dev); 497 498 bus_space_read_multi_4(sc->sc_bst, sc->sc_bsh, off, data, count); 499 } 500 501 static void 502 bcm_sdhci_write_1(device_t dev, struct sdhci_slot *slot, bus_size_t off, 503 uint8_t val) 504 { 505 struct bcm_sdhci_softc *sc = device_get_softc(dev); 506 uint32_t val32 = RD4(sc, off & ~3); 507 val32 &= ~(0xff << (off & 3)*8); 508 val32 |= (val << (off & 3)*8); 509 WR4(sc, off & ~3, val32); 510 } 511 512 static void 513 bcm_sdhci_write_2(device_t dev, struct sdhci_slot *slot, bus_size_t off, 514 uint16_t val) 515 { 516 struct bcm_sdhci_softc *sc = device_get_softc(dev); 517 uint32_t val32; 518 519 /* 520 * If we have a queued up 16bit value for blk size or count, use and 521 * update the saved value rather than doing any real register access. 522 * If we did not touch either since the last write, then read from 523 * register as at least block count can change. 524 * Similarly, if we are about to issue a command, always use the saved 525 * value for transfer mode as we can never write that without issuing 526 * a command. 527 */ 528 if ((off == SDHCI_BLOCK_SIZE || off == SDHCI_BLOCK_COUNT) && 529 sc->need_update_blk) 530 val32 = sc->blksz_and_count; 531 else if (off == SDHCI_COMMAND_FLAGS) 532 val32 = sc->cmd_and_mode; 533 else 534 val32 = RD4(sc, off & ~3); 535 536 val32 &= ~(0xffff << (off & 3)*8); 537 val32 |= (val << (off & 3)*8); 538 539 if (off == SDHCI_TRANSFER_MODE) 540 sc->cmd_and_mode = val32; 541 else if (off == SDHCI_BLOCK_SIZE || off == SDHCI_BLOCK_COUNT) { 542 sc->blksz_and_count = val32; 543 sc->need_update_blk = true; 544 } else { 545 if (off == SDHCI_COMMAND_FLAGS) { 546 /* If we saved blk writes, do them now before cmd. */ 547 if (sc->need_update_blk) { 548 WR4(sc, SDHCI_BLOCK_SIZE, sc->blksz_and_count); 549 sc->need_update_blk = false; 550 } 551 /* Always save cmd and mode registers. */ 552 sc->cmd_and_mode = val32; 553 } 554 WR4(sc, off & ~3, val32); 555 } 556 } 557 558 static void 559 bcm_sdhci_write_4(device_t dev, struct sdhci_slot *slot, bus_size_t off, 560 uint32_t val) 561 { 562 struct bcm_sdhci_softc *sc = device_get_softc(dev); 563 WR4(sc, off, val); 564 } 565 566 static void 567 bcm_sdhci_write_multi_4(device_t dev, struct sdhci_slot *slot, bus_size_t off, 568 uint32_t *data, bus_size_t count) 569 { 570 struct bcm_sdhci_softc *sc = device_get_softc(dev); 571 572 bus_space_write_multi_4(sc->sc_bst, sc->sc_bsh, off, data, count); 573 } 574 575 static void 576 bcm_sdhci_start_dma_seg(struct bcm_sdhci_softc *sc) 577 { 578 struct sdhci_slot *slot; 579 vm_paddr_t pdst, psrc; 580 int err, idx, len, sync_op, width; 581 582 slot = &sc->sc_slot; 583 mtx_assert(&slot->mtx, MA_OWNED); 584 idx = sc->dmamap_seg_index++; 585 len = sc->dmamap_seg_sizes[idx]; 586 slot->offset += len; 587 width = (len & 0xf ? BCM_DMA_32BIT : BCM_DMA_128BIT); 588 589 if (slot->curcmd->data->flags & MMC_DATA_READ) { 590 /* 591 * Peripherals on the AXI bus do not need DREQ pacing for reads 592 * from the ARM core, so we can safely set this to NONE. 593 */ 594 bcm_dma_setup_src(sc->sc_dma_ch, BCM_DMA_DREQ_NONE, 595 BCM_DMA_SAME_ADDR, BCM_DMA_32BIT); 596 bcm_dma_setup_dst(sc->sc_dma_ch, BCM_DMA_DREQ_NONE, 597 BCM_DMA_INC_ADDR, width); 598 psrc = sc->sc_sdhci_buffer_phys; 599 pdst = sc->dmamap_seg_addrs[idx]; 600 sync_op = BUS_DMASYNC_PREREAD; 601 } else { 602 /* 603 * The ordering here is important, because the last write to 604 * dst/src in the dma control block writes the real dreq value. 605 */ 606 bcm_dma_setup_src(sc->sc_dma_ch, BCM_DMA_DREQ_NONE, 607 BCM_DMA_INC_ADDR, width); 608 bcm_dma_setup_dst(sc->sc_dma_ch, sc->conf->emmc_dreq, 609 BCM_DMA_SAME_ADDR, BCM_DMA_32BIT); 610 psrc = sc->dmamap_seg_addrs[idx]; 611 pdst = sc->sc_sdhci_buffer_phys; 612 sync_op = BUS_DMASYNC_PREWRITE; 613 } 614 615 /* 616 * When starting a new DMA operation do the busdma sync operation, and 617 * disable SDCHI data interrrupts because we'll be driven by DMA 618 * interrupts (or SDHCI error interrupts) until the IO is done. 619 */ 620 if (idx == 0) { 621 bus_dmamap_sync(sc->sc_dma_tag, sc->sc_dma_map, sync_op); 622 623 slot->intmask &= ~DATA_XFER_MASK; 624 bcm_sdhci_write_4(sc->sc_dev, slot, SDHCI_SIGNAL_ENABLE, 625 slot->intmask); 626 } 627 628 /* 629 * Start the DMA transfer. Only programming errors (like failing to 630 * allocate a channel) cause a non-zero return from bcm_dma_start(). 631 */ 632 err = bcm_dma_start(sc->sc_dma_ch, psrc, pdst, len); 633 KASSERT((err == 0), ("bcm2835_sdhci: failed DMA start")); 634 } 635 636 static void 637 bcm_sdhci_dma_exit(struct bcm_sdhci_softc *sc) 638 { 639 struct sdhci_slot *slot = &sc->sc_slot; 640 641 mtx_assert(&slot->mtx, MA_OWNED); 642 643 /* Re-enable interrupts */ 644 slot->intmask |= DATA_XFER_MASK; 645 bcm_sdhci_write_4(slot->bus, slot, SDHCI_SIGNAL_ENABLE, 646 slot->intmask); 647 } 648 649 static void 650 bcm_sdhci_dma_unload(struct bcm_sdhci_softc *sc) 651 { 652 struct sdhci_slot *slot = &sc->sc_slot; 653 654 if (sc->dmamap_seg_count == 0) 655 return; 656 if ((slot->curcmd->data->flags & MMC_DATA_READ) != 0) 657 bus_dmamap_sync(sc->sc_dma_tag, sc->sc_dma_map, 658 BUS_DMASYNC_POSTREAD); 659 else 660 bus_dmamap_sync(sc->sc_dma_tag, sc->sc_dma_map, 661 BUS_DMASYNC_POSTWRITE); 662 bus_dmamap_unload(sc->sc_dma_tag, sc->sc_dma_map); 663 664 sc->dmamap_seg_count = 0; 665 sc->dmamap_seg_index = 0; 666 } 667 668 static void 669 bcm_sdhci_dma_intr(int ch, void *arg) 670 { 671 struct bcm_sdhci_softc *sc = (struct bcm_sdhci_softc *)arg; 672 struct sdhci_slot *slot = &sc->sc_slot; 673 uint32_t reg; 674 675 mtx_lock(&slot->mtx); 676 if (slot->curcmd == NULL) 677 goto out; 678 /* 679 * If there are more segments for the current dma, start the next one. 680 * Otherwise unload the dma map and decide what to do next based on the 681 * status of the sdhci controller and whether there's more data left. 682 */ 683 if (sc->dmamap_seg_index < sc->dmamap_seg_count) { 684 bcm_sdhci_start_dma_seg(sc); 685 goto out; 686 } 687 688 bcm_sdhci_dma_unload(sc); 689 690 /* 691 * If we had no further segments pending, we need to determine how to 692 * proceed next. If the 'data/space pending' bit is already set and we 693 * can continue via DMA, do so. Otherwise, re-enable interrupts and 694 * return. 695 */ 696 reg = bcm_sdhci_read_4(slot->bus, slot, SDHCI_INT_STATUS) & 697 DATA_XFER_MASK; 698 if ((reg & DATA_PENDING_MASK) != 0 && 699 BCM_SDHCI_SEGSZ_LEFT(slot) >= BCM_SDHCI_BUFFER_SIZE) { 700 /* ACK any pending interrupts */ 701 bcm_sdhci_write_4(slot->bus, slot, SDHCI_INT_STATUS, 702 DATA_PENDING_MASK); 703 704 bcm_sdhci_start_dma(slot); 705 if (slot->curcmd->error != 0) { 706 /* We won't recover from this error for this command. */ 707 bcm_sdhci_dma_unload(sc); 708 bcm_sdhci_dma_exit(sc); 709 sdhci_finish_data(slot); 710 } 711 } else if ((reg & SDHCI_INT_DATA_END) != 0) { 712 bcm_sdhci_dma_exit(sc); 713 bcm_sdhci_write_4(slot->bus, slot, SDHCI_INT_STATUS, 714 reg); 715 slot->flags &= ~PLATFORM_DATA_STARTED; 716 sdhci_finish_data(slot); 717 } else { 718 bcm_sdhci_dma_exit(sc); 719 } 720 out: 721 mtx_unlock(&slot->mtx); 722 } 723 724 static void 725 bcm_sdhci_start_dma(struct sdhci_slot *slot) 726 { 727 struct bcm_sdhci_softc *sc = device_get_softc(slot->bus); 728 uint8_t *buf; 729 size_t left; 730 731 mtx_assert(&slot->mtx, MA_OWNED); 732 733 left = BCM_SDHCI_SEGSZ_LEFT(slot); 734 buf = (uint8_t *)slot->curcmd->data->data + slot->offset; 735 KASSERT(left != 0, 736 ("%s: DMA handling incorrectly indicated", __func__)); 737 738 /* 739 * No need to check segment count here; if we've not yet unloaded 740 * previous segments, we'll catch that in bcm_sdhci_dmacb. 741 */ 742 if (bus_dmamap_load(sc->sc_dma_tag, sc->sc_dma_map, buf, left, 743 bcm_sdhci_dmacb, sc, BUS_DMA_NOWAIT) != 0 || 744 sc->dmamap_status != 0) { 745 slot->curcmd->error = MMC_ERR_NO_MEMORY; 746 return; 747 } 748 749 /* DMA start */ 750 bcm_sdhci_start_dma_seg(sc); 751 } 752 753 static int 754 bcm_sdhci_will_handle_transfer(device_t dev, struct sdhci_slot *slot) 755 { 756 #ifdef INVARIANTS 757 struct bcm_sdhci_softc *sc = device_get_softc(slot->bus); 758 #endif 759 760 /* 761 * We don't want to perform DMA in this context -- interrupts are 762 * disabled, and a transaction may already be in progress. 763 */ 764 if (dumping) 765 return (0); 766 767 /* 768 * This indicates that we somehow let a data interrupt slip by into the 769 * SDHCI framework, when it should not have. This really needs to be 770 * caught and fixed ASAP, as it really shouldn't happen. 771 */ 772 KASSERT(sc->dmamap_seg_count == 0, 773 ("data pending interrupt pushed through SDHCI framework")); 774 775 /* 776 * Do not use DMA for transfers less than our block size. Checking 777 * alignment serves little benefit, as we round transfer sizes down to 778 * a multiple of the block size and push the transfer back to 779 * SDHCI-driven PIO once we're below the block size. 780 */ 781 if (BCM_SDHCI_SEGSZ_LEFT(slot) < BCM_DMA_BLOCK_SIZE) 782 return (0); 783 784 return (1); 785 } 786 787 static void 788 bcm_sdhci_start_transfer(device_t dev, struct sdhci_slot *slot, 789 uint32_t *intmask) 790 { 791 792 /* DMA transfer FIFO 1KB */ 793 bcm_sdhci_start_dma(slot); 794 } 795 796 static void 797 bcm_sdhci_finish_transfer(device_t dev, struct sdhci_slot *slot) 798 { 799 struct bcm_sdhci_softc *sc = device_get_softc(slot->bus); 800 801 /* 802 * Clean up. Interrupts are clearly enabled, because we received an 803 * SDHCI_INT_DATA_END to get this far -- just make sure we don't leave 804 * anything laying around. 805 */ 806 if (sc->dmamap_seg_count != 0) { 807 /* 808 * Our segment math should have worked out such that we would 809 * never finish the transfer without having used up all of the 810 * segments. If we haven't, that means we must have erroneously 811 * regressed to SDHCI-driven PIO to finish the operation and 812 * this is certainly caused by developer-error. 813 */ 814 bcm_sdhci_dma_unload(sc); 815 } 816 817 sdhci_finish_data(slot); 818 } 819 820 static device_method_t bcm_sdhci_methods[] = { 821 /* Device interface */ 822 DEVMETHOD(device_probe, bcm_sdhci_probe), 823 DEVMETHOD(device_attach, bcm_sdhci_attach), 824 DEVMETHOD(device_detach, bcm_sdhci_detach), 825 826 /* Bus interface */ 827 DEVMETHOD(bus_read_ivar, sdhci_generic_read_ivar), 828 DEVMETHOD(bus_write_ivar, sdhci_generic_write_ivar), 829 DEVMETHOD(bus_add_child, bus_generic_add_child), 830 831 /* MMC bridge interface */ 832 DEVMETHOD(mmcbr_update_ios, bcm_sdhci_update_ios), 833 DEVMETHOD(mmcbr_request, sdhci_generic_request), 834 DEVMETHOD(mmcbr_get_ro, bcm_sdhci_get_ro), 835 DEVMETHOD(mmcbr_acquire_host, sdhci_generic_acquire_host), 836 DEVMETHOD(mmcbr_release_host, sdhci_generic_release_host), 837 838 /* Platform transfer methods */ 839 DEVMETHOD(sdhci_platform_will_handle, bcm_sdhci_will_handle_transfer), 840 DEVMETHOD(sdhci_platform_start_transfer, bcm_sdhci_start_transfer), 841 DEVMETHOD(sdhci_platform_finish_transfer, bcm_sdhci_finish_transfer), 842 /* SDHCI registers accessors */ 843 DEVMETHOD(sdhci_read_1, bcm_sdhci_read_1), 844 DEVMETHOD(sdhci_read_2, bcm_sdhci_read_2), 845 DEVMETHOD(sdhci_read_4, bcm_sdhci_read_4), 846 DEVMETHOD(sdhci_read_multi_4, bcm_sdhci_read_multi_4), 847 DEVMETHOD(sdhci_write_1, bcm_sdhci_write_1), 848 DEVMETHOD(sdhci_write_2, bcm_sdhci_write_2), 849 DEVMETHOD(sdhci_write_4, bcm_sdhci_write_4), 850 DEVMETHOD(sdhci_write_multi_4, bcm_sdhci_write_multi_4), 851 852 DEVMETHOD_END 853 }; 854 855 static devclass_t bcm_sdhci_devclass; 856 857 static driver_t bcm_sdhci_driver = { 858 "sdhci_bcm", 859 bcm_sdhci_methods, 860 sizeof(struct bcm_sdhci_softc), 861 }; 862 863 DRIVER_MODULE(sdhci_bcm, simplebus, bcm_sdhci_driver, bcm_sdhci_devclass, 864 NULL, NULL); 865 #ifdef NOTYET 866 MODULE_DEPEND(sdhci_bcm, bcm2835_clkman, 1, 1, 1); 867 #endif 868 SDHCI_DEPEND(sdhci_bcm); 869 #ifndef MMCCAM 870 MMC_DECLARE_BRIDGE(sdhci_bcm); 871 #endif 872