1 /* $OpenBSD: ufshci.c,v 1.33 2024/05/27 10:27:58 mglocker Exp $ */ 2 3 /* 4 * Copyright (c) 2022 Marcus Glocker <mglocker@openbsd.org> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 /* 20 * Universal Flash Storage Host Controller Interface (UFSHCI) 2.1 driver based 21 * on the JEDEC JESD223C.pdf and JESD220C-2_1.pdf specifications. 22 */ 23 24 #include <sys/param.h> 25 #include <sys/systm.h> 26 #include <sys/buf.h> 27 #include <sys/kernel.h> 28 #include <sys/malloc.h> 29 #include <sys/device.h> 30 #include <sys/queue.h> 31 #include <sys/mutex.h> 32 #include <sys/pool.h> 33 34 #include <sys/atomic.h> 35 36 #include <machine/bus.h> 37 38 #include <scsi/scsi_all.h> 39 #include <scsi/scsi_disk.h> 40 #include <scsi/scsiconf.h> 41 42 #include <dev/ic/ufshcivar.h> 43 #include <dev/ic/ufshcireg.h> 44 45 #ifdef UFSHCI_DEBUG 46 int ufshci_dbglvl = 1; 47 #define DPRINTF(l, x...) do { if ((l) <= ufshci_dbglvl) printf(x); } \ 48 while (0) 49 #else 50 #define DPRINTF(l, x...) 51 #endif 52 53 struct cfdriver ufshci_cd = { 54 NULL, "ufshci", DV_DULL 55 }; 56 57 int ufshci_reset(struct ufshci_softc *); 58 int ufshci_is_poll(struct ufshci_softc *, uint32_t); 59 struct ufshci_dmamem *ufshci_dmamem_alloc(struct ufshci_softc *, size_t); 60 void ufshci_dmamem_free(struct ufshci_softc *, 61 struct ufshci_dmamem *); 62 int ufshci_init(struct ufshci_softc *); 63 int ufshci_doorbell_read(struct ufshci_softc *); 64 void ufshci_doorbell_write(struct ufshci_softc *, int); 65 int ufshci_doorbell_poll(struct ufshci_softc *, int, 66 uint32_t); 67 int ufshci_utr_cmd_nop(struct ufshci_softc *, 68 struct ufshci_ccb *, struct scsi_xfer *); 69 int ufshci_utr_cmd_lun(struct ufshci_softc *, 70 struct ufshci_ccb *, struct scsi_xfer *); 71 int ufshci_utr_cmd_inquiry(struct ufshci_softc *, 72 struct ufshci_ccb *, struct scsi_xfer *); 73 int ufshci_utr_cmd_capacity16(struct ufshci_softc *, 74 struct ufshci_ccb *, struct scsi_xfer *); 75 int ufshci_utr_cmd_capacity(struct ufshci_softc *, 76 struct ufshci_ccb *, struct scsi_xfer *); 77 int ufshci_utr_cmd_io(struct ufshci_softc *, 78 struct ufshci_ccb *, struct scsi_xfer *, int); 79 int ufshci_utr_cmd_sync(struct ufshci_softc *, 80 struct ufshci_ccb *, struct scsi_xfer *, 81 uint32_t, uint16_t); 82 int ufshci_xfer_complete(struct ufshci_softc *); 83 int ufshci_powerdown(struct ufshci_softc *); 84 int ufshci_resume(struct ufshci_softc *); 85 86 /* SCSI */ 87 int ufshci_ccb_alloc(struct ufshci_softc *, int); 88 void *ufshci_ccb_get(void *); 89 void ufshci_ccb_put(void *, void *); 90 void ufshci_ccb_free(struct ufshci_softc*, int); 91 92 void ufshci_scsi_cmd(struct scsi_xfer *); 93 void ufshci_minphys(struct buf *, struct scsi_link *); 94 int ufshci_scsi_probe(struct scsi_link *); 95 void ufshci_scsi_free(struct scsi_link *); 96 97 void ufshci_scsi_inquiry(struct scsi_xfer *); 98 void ufshci_scsi_capacity16(struct scsi_xfer *); 99 void ufshci_scsi_capacity(struct scsi_xfer *); 100 void ufshci_scsi_sync(struct scsi_xfer *); 101 void ufshci_scsi_io(struct scsi_xfer *, int); 102 void ufshci_scsi_io_done(struct ufshci_softc *, 103 struct ufshci_ccb *); 104 void ufshci_scsi_done(struct ufshci_softc *, 105 struct ufshci_ccb *); 106 107 const struct scsi_adapter ufshci_switch = { 108 ufshci_scsi_cmd, NULL, NULL, NULL, NULL 109 }; 110 111 int 112 ufshci_intr(void *arg) 113 { 114 struct ufshci_softc *sc = arg; 115 uint32_t status, hcs; 116 int handled = 0; 117 118 status = UFSHCI_READ_4(sc, UFSHCI_REG_IS); 119 DPRINTF(3, "%s: status=0x%08x\n", __func__, status); 120 121 if (status == 0) 122 return 0; 123 124 if (status & UFSHCI_REG_IS_UCCS) { 125 DPRINTF(3, "%s: UCCS interrupt\n", __func__); 126 handled = 1; 127 } 128 if (status & UFSHCI_REG_IS_UTRCS) { 129 DPRINTF(3, "%s: UTRCS interrupt\n", __func__); 130 131 ufshci_xfer_complete(sc); 132 133 handled = 1; 134 } 135 /* If Auto-Hibernate raises an interrupt, it's to yield an error. */ 136 if (status & UFSHCI_REG_IS_UHES) { 137 hcs = UFSHCI_READ_4(sc, UFSHCI_REG_HCS); 138 printf("%s: Auto-Hibernate enter error UPMCRS=0x%x\n", 139 __func__, UFSHCI_REG_HCS_UPMCRS(hcs)); 140 } 141 if (status & UFSHCI_REG_IS_UHXS) { 142 hcs = UFSHCI_READ_4(sc, UFSHCI_REG_HCS); 143 printf("%s: Auto-Hibernate exit error UPMCRS=0x%x\n", 144 __func__, UFSHCI_REG_HCS_UPMCRS(hcs)); 145 } 146 147 if (handled == 0) { 148 printf("%s: UNKNOWN interrupt, status=0x%08x\n", 149 sc->sc_dev.dv_xname, status); 150 } 151 152 /* ACK interrupt */ 153 UFSHCI_WRITE_4(sc, UFSHCI_REG_IS, status); 154 155 return 1; 156 } 157 158 int 159 ufshci_attach(struct ufshci_softc *sc) 160 { 161 struct scsibus_attach_args saa; 162 163 mtx_init(&sc->sc_cmd_mtx, IPL_BIO); 164 mtx_init(&sc->sc_ccb_mtx, IPL_BIO); 165 SIMPLEQ_INIT(&sc->sc_ccb_list); 166 scsi_iopool_init(&sc->sc_iopool, sc, ufshci_ccb_get, ufshci_ccb_put); 167 168 ufshci_reset(sc); 169 170 sc->sc_ver = UFSHCI_READ_4(sc, UFSHCI_REG_VER); 171 printf(", UFSHCI %d.%d%d\n", 172 UFSHCI_REG_VER_MAJOR(sc->sc_ver), 173 UFSHCI_REG_VER_MINOR(sc->sc_ver), 174 UFSHCI_REG_VER_SUFFIX(sc->sc_ver)); 175 176 sc->sc_cap = UFSHCI_READ_4(sc, UFSHCI_REG_CAP); 177 sc->sc_hcpid = UFSHCI_READ_4(sc, UFSHCI_REG_HCPID); 178 sc->sc_hcmid = UFSHCI_READ_4(sc, UFSHCI_REG_HCMID); 179 sc->sc_nutmrs = UFSHCI_REG_CAP_NUTMRS(sc->sc_cap) + 1; 180 sc->sc_rtt = UFSHCI_REG_CAP_RTT(sc->sc_cap) + 1; 181 sc->sc_nutrs = UFSHCI_REG_CAP_NUTRS(sc->sc_cap) + 1; 182 183 DPRINTF(1, "Capabilities (0x%08x):\n", sc->sc_cap); 184 DPRINTF(1, "CS=%d\n", sc->sc_cap & UFSHCI_REG_CAP_CS ? 1 : 0); 185 DPRINTF(1, "UICDMETMS=%d\n", 186 sc->sc_cap & UFSHCI_REG_CAP_UICDMETMS ? 1 : 0); 187 DPRINTF(1, "OODDS=%d\n", sc->sc_cap & UFSHCI_REG_CAP_OODDS ? 1 : 0); 188 DPRINTF(1, "64AS=%d\n", sc->sc_cap & UFSHCI_REG_CAP_64AS ? 1 : 0); 189 DPRINTF(1, "AUTOH8=%d\n", sc->sc_cap & UFSHCI_REG_AUTOH8 ? 1 : 0); 190 DPRINTF(1, "NUTMRS=%d\n", sc->sc_nutmrs); 191 DPRINTF(1, "RTT=%d\n", sc->sc_rtt); 192 DPRINTF(1, "NUTRS=%d\n", sc->sc_nutrs); 193 DPRINTF(1, "HCPID=0x%08x\n", sc->sc_hcpid); 194 DPRINTF(1, "HCMID (0x%08x):\n", sc->sc_hcmid); 195 DPRINTF(1, " BI=0x%04x\n", UFSHCI_REG_HCMID_BI(sc->sc_hcmid)); 196 DPRINTF(1, " MIC=0x%04x\n", UFSHCI_REG_HCMID_MIC(sc->sc_hcmid)); 197 198 if (sc->sc_nutrs < UFSHCI_SLOTS_MIN || 199 sc->sc_nutrs > UFSHCI_SLOTS_MAX) { 200 printf("%s: Invalid NUTRS value %d (must be %d-%d)!\n", 201 sc->sc_dev.dv_xname, sc->sc_nutrs, 202 UFSHCI_SLOTS_MIN, UFSHCI_SLOTS_MAX); 203 return 1; 204 } 205 if (sc->sc_nutrs == UFSHCI_SLOTS_MAX) 206 sc->sc_iacth = UFSHCI_INTR_AGGR_COUNT_MAX; 207 else 208 sc->sc_iacth = sc->sc_nutrs; 209 DPRINTF(1, "Intr. aggr. counter threshold:\nIACTH=%d\n", sc->sc_iacth); 210 211 /* 212 * XXX: 213 * At the moment normal interrupts work better for us than interrupt 214 * aggregation, because: 215 * 216 * 1. With interrupt aggregation enabled, the I/O performance 217 * isn't better, but even slightly worse depending on the 218 * UFS controller and architecture. 219 * 2. With interrupt aggregation enabled we currently see 220 * intermittent SCSI command stalling. Probably there is a 221 * race condition where new SCSI commands are getting 222 * scheduled, while we miss to reset the interrupt aggregation 223 * counter/timer, which leaves us with no more interrupts 224 * triggered. This needs to be fixed, but I couldn't figure 225 * out yet how. 226 */ 227 #if 0 228 sc->sc_flags |= UFSHCI_FLAGS_AGGR_INTR; /* Enable intr. aggregation */ 229 #endif 230 ufshci_init(sc); 231 232 if (ufshci_ccb_alloc(sc, sc->sc_nutrs) != 0) { 233 printf("%s: %s: Can't allocate CCBs\n", 234 sc->sc_dev.dv_xname, __func__); 235 return 1; 236 } 237 238 /* Enable Auto-Hibernate Idle Timer (AHIT) and set it to 150ms. */ 239 if (sc->sc_cap & UFSHCI_REG_AUTOH8) { 240 UFSHCI_WRITE_4(sc, UFSHCI_REG_AHIT, 241 UFSHCI_REG_AHIT_TS(UFSHCI_REG_AHIT_TS_1MS) | 150); 242 } 243 244 /* Attach to SCSI layer */ 245 saa.saa_adapter = &ufshci_switch; 246 saa.saa_adapter_softc = sc; 247 saa.saa_adapter_buswidth = 2; /* XXX: What's the right value? */ 248 saa.saa_luns = 1; /* XXX: Should we use ufshci_utr_cmd_lun() */ 249 saa.saa_adapter_target = 0; 250 saa.saa_openings = sc->sc_nutrs; 251 saa.saa_pool = &sc->sc_iopool; 252 saa.saa_quirks = saa.saa_flags = 0; 253 saa.saa_wwpn = saa.saa_wwnn = 0; 254 255 config_found(&sc->sc_dev, &saa, scsiprint); 256 257 return 0; 258 } 259 260 int 261 ufshci_reset(struct ufshci_softc *sc) 262 { 263 int i; 264 int retry = 10; 265 uint32_t hce; 266 267 /* 268 * 7.1.1 Host Controller Initialization: 2) 269 * Reset and enable host controller 270 */ 271 UFSHCI_WRITE_4(sc, UFSHCI_REG_HCE, UFSHCI_REG_HCE_HCE); 272 273 /* 7.1.1 Host Controller Initialization: 3) */ 274 for (i = 0; i < retry; i++) { 275 hce = UFSHCI_READ_4(sc, UFSHCI_REG_HCE); 276 if (hce == 1) 277 break; 278 delay(1); 279 } 280 if (i == retry) { 281 printf("%s: Enabling Host Controller failed!\n", 282 sc->sc_dev.dv_xname); 283 return -1; 284 } 285 286 DPRINTF(2, "\n%s: Host Controller enabled (i=%d)\n", __func__, i); 287 288 return 0; 289 } 290 291 int 292 ufshci_is_poll(struct ufshci_softc *sc, uint32_t type) 293 { 294 uint32_t status; 295 int i, retry = 25; 296 297 DPRINTF(3, "%s\n", __func__); 298 299 for (i = 0; i < retry; i++) { 300 status = UFSHCI_READ_4(sc, UFSHCI_REG_IS); 301 if (status & type) 302 break; 303 delay(10); 304 } 305 if (i == retry) { 306 printf("%s: %s: timeout\n", sc->sc_dev.dv_xname, __func__); 307 return -1; 308 } 309 DPRINTF(3, "%s: completed after %d retries\n", __func__, i); 310 311 /* ACK interrupt */ 312 UFSHCI_WRITE_4(sc, UFSHCI_REG_IS, status); 313 314 return 0; 315 } 316 317 struct ufshci_dmamem * 318 ufshci_dmamem_alloc(struct ufshci_softc *sc, size_t size) 319 { 320 struct ufshci_dmamem *udm; 321 int nsegs; 322 323 udm = malloc(sizeof(*udm), M_DEVBUF, M_WAITOK | M_ZERO); 324 if (udm == NULL) 325 return NULL; 326 327 udm->udm_size = size; 328 329 if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, 330 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW | 331 (sc->sc_cap & UFSHCI_REG_CAP_64AS) ? BUS_DMA_64BIT : 0, 332 &udm->udm_map) != 0) 333 goto udmfree; 334 335 if (bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, &udm->udm_seg, 336 1, &nsegs, BUS_DMA_WAITOK | BUS_DMA_ZERO) != 0) 337 goto destroy; 338 339 if (bus_dmamem_map(sc->sc_dmat, &udm->udm_seg, nsegs, size, 340 &udm->udm_kva, BUS_DMA_WAITOK) != 0) 341 goto free; 342 343 if (bus_dmamap_load(sc->sc_dmat, udm->udm_map, udm->udm_kva, size, 344 NULL, BUS_DMA_WAITOK) != 0) 345 goto unmap; 346 347 DPRINTF(2, "%s: size=%lu, page_size=%d, nsegs=%d\n", 348 __func__, size, PAGE_SIZE, nsegs); 349 350 return udm; 351 352 unmap: 353 bus_dmamem_unmap(sc->sc_dmat, udm->udm_kva, size); 354 free: 355 bus_dmamem_free(sc->sc_dmat, &udm->udm_seg, 1); 356 destroy: 357 bus_dmamap_destroy(sc->sc_dmat, udm->udm_map); 358 udmfree: 359 free(udm, M_DEVBUF, sizeof(*udm)); 360 361 return NULL; 362 } 363 364 void 365 ufshci_dmamem_free(struct ufshci_softc *sc, struct ufshci_dmamem *udm) 366 { 367 bus_dmamap_unload(sc->sc_dmat, udm->udm_map); 368 bus_dmamem_unmap(sc->sc_dmat, udm->udm_kva, udm->udm_size); 369 bus_dmamem_free(sc->sc_dmat, &udm->udm_seg, 1); 370 bus_dmamap_destroy(sc->sc_dmat, udm->udm_map); 371 free(udm, M_DEVBUF, sizeof(*udm)); 372 } 373 374 int 375 ufshci_init(struct ufshci_softc *sc) 376 { 377 uint32_t reg; 378 uint64_t dva; 379 380 /* 381 * 7.1.1 Host Controller Initialization: 4) 382 * TODO: Do we need to set DME_SET? 383 */ 384 385 /* 7.1.1 Host Controller Initialization: 5) */ 386 if (sc->sc_cap & UFSHCI_REG_AUTOH8) { 387 UFSHCI_WRITE_4(sc, UFSHCI_REG_IE, 388 UFSHCI_REG_IE_UTRCE | UFSHCI_REG_IE_UTMRCE | 389 UFSHCI_REG_IS_UHES | UFSHCI_REG_IS_UHXS); 390 } else { 391 UFSHCI_WRITE_4(sc, UFSHCI_REG_IE, 392 UFSHCI_REG_IE_UTRCE | UFSHCI_REG_IE_UTMRCE); 393 } 394 395 /* 7.1.1 Host Controller Initialization: 6) */ 396 UFSHCI_WRITE_4(sc, UFSHCI_REG_UICCMD, 397 UFSHCI_REG_UICCMD_CMDOP_DME_LINKSTARTUP); 398 if (ufshci_is_poll(sc, UFSHCI_REG_IS_UCCS) != 0) 399 return -1; 400 401 /* 402 * 7.1.1 Host Controller Initialization: 7), 8), 9) 403 * TODO: Implement retry in case UFSHCI_REG_HCS returns 0 404 */ 405 reg = UFSHCI_READ_4(sc, UFSHCI_REG_HCS); 406 if (reg & UFSHCI_REG_HCS_DP) 407 DPRINTF(2, "%s: Device Presence SET\n", __func__); 408 else 409 DPRINTF(2, "%s: Device Presence NOT SET\n", __func__); 410 411 /* 412 * 7.1.1 Host Controller Initialization: 10) 413 * TODO: Enable additional interrupt on the IE register 414 */ 415 416 /* 7.1.1 Host Controller Initialization: 11) */ 417 if (sc->sc_flags & UFSHCI_FLAGS_AGGR_INTR) { 418 UFSHCI_WRITE_4(sc, UFSHCI_REG_UTRIACR, 419 UFSHCI_REG_UTRIACR_IAEN | 420 UFSHCI_REG_UTRIACR_IAPWEN | 421 UFSHCI_REG_UTRIACR_CTR | 422 UFSHCI_REG_UTRIACR_IACTH(sc->sc_iacth) | 423 UFSHCI_REG_UTRIACR_IATOVAL(UFSHCI_INTR_AGGR_TIMEOUT)); 424 } else { 425 UFSHCI_WRITE_4(sc, UFSHCI_REG_UTRIACR, 0); 426 } 427 428 /* 429 * 7.1.1 Host Controller Initialization: 12) 430 * TODO: More UIC commands to issue? 431 */ 432 433 /* 7.1.1 Host Controller Initialization: 13) */ 434 sc->sc_dmamem_utmrd = ufshci_dmamem_alloc(sc, 435 sizeof(struct ufshci_utmrd) * sc->sc_nutmrs); 436 if (sc->sc_dmamem_utmrd == NULL) { 437 printf("%s: Can't allocate DMA memory for UTMRD\n", 438 sc->sc_dev.dv_xname); 439 return -1; 440 } 441 /* 7.1.1 Host Controller Initialization: 14) */ 442 dva = UFSHCI_DMA_DVA(sc->sc_dmamem_utmrd); 443 DPRINTF(2, "%s: utmrd dva=%llu\n", __func__, dva); 444 UFSHCI_WRITE_4(sc, UFSHCI_REG_UTMRLBA, (uint32_t)dva); 445 UFSHCI_WRITE_4(sc, UFSHCI_REG_UTMRLBAU, (uint32_t)(dva >> 32)); 446 447 /* 7.1.1 Host Controller Initialization: 15) */ 448 sc->sc_dmamem_utrd = ufshci_dmamem_alloc(sc, 449 sizeof(struct ufshci_utrd) * sc->sc_nutrs); 450 if (sc->sc_dmamem_utrd == NULL) { 451 printf("%s: Can't allocate DMA memory for UTRD\n", 452 sc->sc_dev.dv_xname); 453 return -1; 454 } 455 /* 7.1.1 Host Controller Initialization: 16) */ 456 dva = UFSHCI_DMA_DVA(sc->sc_dmamem_utrd); 457 DPRINTF(2, "%s: utrd dva=%llu\n", __func__, dva); 458 UFSHCI_WRITE_4(sc, UFSHCI_REG_UTRLBA, (uint32_t)dva); 459 UFSHCI_WRITE_4(sc, UFSHCI_REG_UTRLBAU, (uint32_t)(dva >> 32)); 460 461 462 /* Allocate UCDs. */ 463 sc->sc_dmamem_ucd = ufshci_dmamem_alloc(sc, 464 sizeof(struct ufshci_ucd) * sc->sc_nutrs); 465 if (sc->sc_dmamem_ucd == NULL) { 466 printf("%s: Can't allocate DMA memory for UCD\n", 467 sc->sc_dev.dv_xname); 468 return -1; 469 } 470 471 /* 7.1.1 Host Controller Initialization: 17) */ 472 UFSHCI_WRITE_4(sc, UFSHCI_REG_UTMRLRSR, UFSHCI_REG_UTMRLRSR_START); 473 474 /* 7.1.1 Host Controller Initialization: 18) */ 475 UFSHCI_WRITE_4(sc, UFSHCI_REG_UTRLRSR, UFSHCI_REG_UTRLRSR_START); 476 477 /* 7.1.1 Host Controller Initialization: 19) */ 478 /* TODO: bMaxNumOfRTT will be set as the minimum value of 479 * bDeviceRTTCap and NORTT. ??? 480 */ 481 482 return 0; 483 } 484 485 int 486 ufshci_doorbell_read(struct ufshci_softc *sc) 487 { 488 uint32_t reg; 489 490 reg = UFSHCI_READ_4(sc, UFSHCI_REG_UTRLDBR); 491 492 return reg; 493 } 494 495 void 496 ufshci_doorbell_write(struct ufshci_softc *sc, int slot) 497 { 498 uint32_t reg; 499 500 reg = (1U << slot); 501 502 UFSHCI_WRITE_4(sc, UFSHCI_REG_UTRLDBR, reg); 503 } 504 505 int 506 ufshci_doorbell_poll(struct ufshci_softc *sc, int slot, uint32_t timeout_ms) 507 { 508 uint32_t reg; 509 uint64_t timeout_us; 510 511 DPRINTF(3, "%s\n", __func__); 512 513 for (timeout_us = timeout_ms * 1000; timeout_us != 0; 514 timeout_us -= 1000) { 515 reg = UFSHCI_READ_4(sc, UFSHCI_REG_UTRLDBR); 516 if ((reg & (1U << slot)) == 0) 517 break; 518 delay(1000); 519 } 520 if (timeout_us == 0) { 521 printf("%s: %s: timeout\n", sc->sc_dev.dv_xname, __func__); 522 return -1; 523 } 524 525 return 0; 526 } 527 528 int 529 ufshci_utr_cmd_nop(struct ufshci_softc *sc, struct ufshci_ccb *ccb, 530 struct scsi_xfer *xs) 531 { 532 int slot, off, len; 533 uint64_t dva; 534 struct ufshci_utrd *utrd; 535 struct ufshci_ucd *ucd; 536 537 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 1) */ 538 slot = ccb->ccb_slot; 539 utrd = UFSHCI_DMA_KVA(sc->sc_dmamem_utrd); 540 utrd += slot; 541 memset(utrd, 0, sizeof(*utrd)); 542 DPRINTF(3, "%s: slot=%d\n", __func__, slot); 543 544 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2a) */ 545 utrd->dw0 = UFSHCI_UTRD_DW0_CT_UFS; 546 547 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2b) */ 548 utrd->dw0 |= UFSHCI_UTRD_DW0_DD_NO; 549 550 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2c) */ 551 if (sc->sc_flags & UFSHCI_FLAGS_AGGR_INTR) 552 utrd->dw0 |= UFSHCI_UTRD_DW0_I_REG; 553 else 554 utrd->dw0 |= UFSHCI_UTRD_DW0_I_INT; 555 556 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2d) */ 557 utrd->dw2 = UFSHCI_UTRD_DW2_OCS_IOV; 558 559 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2e) */ 560 ucd = UFSHCI_DMA_KVA(sc->sc_dmamem_ucd); 561 ucd += slot; 562 memset(ucd, 0, sizeof(*ucd)); 563 564 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2f) */ 565 ucd->cmd.hdr.tc = UPIU_TC_I2T_NOP_OUT; 566 ucd->cmd.hdr.flags = 0; 567 ucd->cmd.hdr.lun = 0; 568 ucd->cmd.hdr.task_tag = slot; 569 ucd->cmd.hdr.cmd_set_type = 0; /* SCSI command */ 570 ucd->cmd.hdr.query = 0; 571 ucd->cmd.hdr.response = 0; 572 ucd->cmd.hdr.status = 0; 573 ucd->cmd.hdr.ehs_len = 0; 574 ucd->cmd.hdr.device_info = 0; 575 ucd->cmd.hdr.ds_len = 0; 576 577 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2g) */ 578 /* Already done with above memset */ 579 580 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 3) */ 581 dva = UFSHCI_DMA_DVA(sc->sc_dmamem_ucd); 582 DPRINTF(3, "%s: ucd dva=%llu\n", __func__, dva); 583 utrd->dw4 = (uint32_t)dva; 584 utrd->dw5 = (uint32_t)(dva >> 32); 585 586 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 4) */ 587 off = sizeof(struct upiu_command) / 4; /* DWORD offset */ 588 utrd->dw6 = UFSHCI_UTRD_DW6_RUO(off); 589 590 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 5) */ 591 len = sizeof(struct upiu_response) / 4; /* DWORD length */ 592 utrd->dw6 |= UFSHCI_UTRD_DW6_RUL(len); 593 594 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 6) */ 595 off = (sizeof(struct upiu_command) + sizeof(struct upiu_response)) / 4; 596 utrd->dw7 = UFSHCI_UTRD_DW7_PRDTO(off); 597 598 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 7) */ 599 utrd->dw7 |= UFSHCI_UTRD_DW7_PRDTL(0); /* No data xfer */ 600 601 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 9) */ 602 if (UFSHCI_READ_4(sc, UFSHCI_REG_UTRLRSR) != 1) { 603 printf("%s: %s: UTRLRSR not set\n", 604 sc->sc_dev.dv_xname, __func__); 605 return -1; 606 } 607 608 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_utrd), 609 sizeof(*utrd) * slot, sizeof(*utrd), BUS_DMASYNC_PREWRITE); 610 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_ucd), 611 sizeof(*ucd) * slot, sizeof(*ucd), BUS_DMASYNC_PREWRITE); 612 613 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 14) */ 614 ccb->ccb_status = CCB_STATUS_INPROGRESS; 615 ufshci_doorbell_write(sc, slot); 616 617 return 0; 618 } 619 620 int 621 ufshci_utr_cmd_lun(struct ufshci_softc *sc, struct ufshci_ccb *ccb, 622 struct scsi_xfer *xs) 623 { 624 int slot, off, len, i; 625 uint64_t dva; 626 struct ufshci_utrd *utrd; 627 struct ufshci_ucd *ucd; 628 bus_dmamap_t dmap = ccb->ccb_dmamap; 629 630 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 1) */ 631 slot = ccb->ccb_slot; 632 utrd = UFSHCI_DMA_KVA(sc->sc_dmamem_utrd); 633 utrd += slot; 634 memset(utrd, 0, sizeof(*utrd)); 635 DPRINTF(3, "%s: slot=%d\n", __func__, slot); 636 637 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2a) */ 638 utrd->dw0 = UFSHCI_UTRD_DW0_CT_UFS; 639 640 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2b) */ 641 utrd->dw0 |= UFSHCI_UTRD_DW0_DD_T2I; 642 643 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2c) */ 644 if (sc->sc_flags & UFSHCI_FLAGS_AGGR_INTR) 645 utrd->dw0 |= UFSHCI_UTRD_DW0_I_REG; 646 else 647 utrd->dw0 |= UFSHCI_UTRD_DW0_I_INT; 648 649 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2d) */ 650 utrd->dw2 = UFSHCI_UTRD_DW2_OCS_IOV; 651 652 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2e) */ 653 ucd = UFSHCI_DMA_KVA(sc->sc_dmamem_ucd); 654 ucd += slot; 655 memset(ucd, 0, sizeof(*ucd)); 656 657 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2f) */ 658 ucd->cmd.hdr.tc = UPIU_TC_I2T_COMMAND; 659 ucd->cmd.hdr.flags = (1 << 6); /* Bit-5 = Write, Bit-6 = Read */ 660 ucd->cmd.hdr.lun = 0; 661 ucd->cmd.hdr.task_tag = slot; 662 ucd->cmd.hdr.cmd_set_type = 0; /* SCSI command */ 663 ucd->cmd.hdr.query = 0; 664 ucd->cmd.hdr.response = 0; 665 ucd->cmd.hdr.status = 0; 666 ucd->cmd.hdr.ehs_len = 0; 667 ucd->cmd.hdr.device_info = 0; 668 ucd->cmd.hdr.ds_len = 0; 669 670 ucd->cmd.expected_xfer_len = htobe32(xs->datalen); 671 672 ucd->cmd.cdb[0] = REPORT_LUNS; 673 ucd->cmd.cdb[6] = 0; 674 ucd->cmd.cdb[7] = 0; 675 ucd->cmd.cdb[8] = 0; 676 ucd->cmd.cdb[9] = xs->datalen; 677 678 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2g) */ 679 /* Already done with above memset */ 680 681 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 3) */ 682 dva = UFSHCI_DMA_DVA(sc->sc_dmamem_ucd); 683 DPRINTF(3, "%s: ucd dva=%llu\n", __func__, dva); 684 utrd->dw4 = (uint32_t)dva; 685 utrd->dw5 = (uint32_t)(dva >> 32); 686 687 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 4) */ 688 off = sizeof(struct upiu_command) / 4; /* DWORD offset */ 689 utrd->dw6 = UFSHCI_UTRD_DW6_RUO(off); 690 691 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 5) */ 692 len = sizeof(struct upiu_response) / 4; /* DWORD length */ 693 utrd->dw6 |= UFSHCI_UTRD_DW6_RUL(len); 694 695 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 6) */ 696 off = (sizeof(struct upiu_command) + sizeof(struct upiu_response)) / 4; 697 utrd->dw7 = UFSHCI_UTRD_DW7_PRDTO(off); 698 699 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 7) */ 700 utrd->dw7 |= UFSHCI_UTRD_DW7_PRDTL(dmap->dm_nsegs); 701 702 /* Build PRDT data segment. */ 703 for (i = 0; i < dmap->dm_nsegs; i++) { 704 dva = dmap->dm_segs[i].ds_addr; 705 ucd->prdt[i].dw0 = (uint32_t)dva; 706 ucd->prdt[i].dw1 = (uint32_t)(dva >> 32); 707 ucd->prdt[i].dw2 = 0; 708 ucd->prdt[i].dw3 = dmap->dm_segs[i].ds_len - 1; 709 } 710 711 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 9) */ 712 if (UFSHCI_READ_4(sc, UFSHCI_REG_UTRLRSR) != 1) { 713 printf("%s: %s: UTRLRSR not set\n", 714 sc->sc_dev.dv_xname, __func__); 715 return -1; 716 } 717 718 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_utrd), 719 sizeof(*utrd) * slot, sizeof(*utrd), BUS_DMASYNC_PREWRITE); 720 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_ucd), 721 sizeof(*ucd) * slot, sizeof(*ucd), BUS_DMASYNC_PREWRITE); 722 723 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 14) */ 724 ccb->ccb_status = CCB_STATUS_INPROGRESS; 725 ufshci_doorbell_write(sc, slot); 726 727 return 0; 728 } 729 730 int 731 ufshci_utr_cmd_inquiry(struct ufshci_softc *sc, struct ufshci_ccb *ccb, 732 struct scsi_xfer *xs) 733 { 734 int slot, off, len, i; 735 uint64_t dva; 736 struct ufshci_utrd *utrd; 737 struct ufshci_ucd *ucd; 738 bus_dmamap_t dmap = ccb->ccb_dmamap; 739 740 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 1) */ 741 slot = ccb->ccb_slot; 742 utrd = UFSHCI_DMA_KVA(sc->sc_dmamem_utrd); 743 utrd += slot; 744 memset(utrd, 0, sizeof(*utrd)); 745 DPRINTF(3, "%s: slot=%d\n", __func__, slot); 746 747 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2a) */ 748 utrd->dw0 = UFSHCI_UTRD_DW0_CT_UFS; 749 750 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2b) */ 751 utrd->dw0 |= UFSHCI_UTRD_DW0_DD_T2I; 752 753 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2c) */ 754 if (sc->sc_flags & UFSHCI_FLAGS_AGGR_INTR) 755 utrd->dw0 |= UFSHCI_UTRD_DW0_I_REG; 756 else 757 utrd->dw0 |= UFSHCI_UTRD_DW0_I_INT; 758 759 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2d) */ 760 utrd->dw2 = UFSHCI_UTRD_DW2_OCS_IOV; 761 762 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2e) */ 763 ucd = UFSHCI_DMA_KVA(sc->sc_dmamem_ucd); 764 ucd += slot; 765 memset(ucd, 0, sizeof(*ucd)); 766 767 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2f) */ 768 ucd->cmd.hdr.tc = UPIU_TC_I2T_COMMAND; 769 ucd->cmd.hdr.flags = (1 << 6); /* Bit-5 = Write, Bit-6 = Read */ 770 ucd->cmd.hdr.lun = 0; 771 ucd->cmd.hdr.task_tag = slot; 772 ucd->cmd.hdr.cmd_set_type = 0; /* SCSI command */ 773 ucd->cmd.hdr.query = 0; 774 ucd->cmd.hdr.response = 0; 775 ucd->cmd.hdr.status = 0; 776 ucd->cmd.hdr.ehs_len = 0; 777 ucd->cmd.hdr.device_info = 0; 778 ucd->cmd.hdr.ds_len = 0; 779 780 ucd->cmd.expected_xfer_len = htobe32(xs->datalen); 781 782 ucd->cmd.cdb[0] = INQUIRY; /* 0x12 */ 783 ucd->cmd.cdb[3] = 0; 784 ucd->cmd.cdb[4] = xs->datalen; 785 786 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2g) */ 787 /* Already done with above memset */ 788 789 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 3) */ 790 dva = UFSHCI_DMA_DVA(sc->sc_dmamem_ucd) + (sizeof(*ucd) * slot); 791 DPRINTF(3, "%s: ucd dva=%llu\n", __func__, dva); 792 utrd->dw4 = (uint32_t)dva; 793 utrd->dw5 = (uint32_t)(dva >> 32); 794 795 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 4) */ 796 off = sizeof(struct upiu_command) / 4; /* DWORD offset */ 797 utrd->dw6 = UFSHCI_UTRD_DW6_RUO(off); 798 799 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 5) */ 800 len = sizeof(struct upiu_response) / 4; /* DWORD length */ 801 utrd->dw6 |= UFSHCI_UTRD_DW6_RUL(len); 802 803 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 6) */ 804 off = (sizeof(struct upiu_command) + sizeof(struct upiu_response)) / 4; 805 utrd->dw7 = UFSHCI_UTRD_DW7_PRDTO(off); 806 807 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 7) */ 808 utrd->dw7 |= UFSHCI_UTRD_DW7_PRDTL(dmap->dm_nsegs); 809 810 /* Build PRDT data segment. */ 811 for (i = 0; i < dmap->dm_nsegs; i++) { 812 dva = dmap->dm_segs[i].ds_addr; 813 ucd->prdt[i].dw0 = (uint32_t)dva; 814 ucd->prdt[i].dw1 = (uint32_t)(dva >> 32); 815 ucd->prdt[i].dw2 = 0; 816 ucd->prdt[i].dw3 = dmap->dm_segs[i].ds_len - 1; 817 } 818 819 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 9) */ 820 if (UFSHCI_READ_4(sc, UFSHCI_REG_UTRLRSR) != 1) { 821 printf("%s: %s: UTRLRSR not set\n", 822 sc->sc_dev.dv_xname, __func__); 823 return -1; 824 } 825 826 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_utrd), 827 sizeof(*utrd) * slot, sizeof(*utrd), BUS_DMASYNC_PREWRITE); 828 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_ucd), 829 sizeof(*ucd) * slot, sizeof(*ucd), BUS_DMASYNC_PREWRITE); 830 831 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 14) */ 832 ccb->ccb_status = CCB_STATUS_INPROGRESS; 833 ufshci_doorbell_write(sc, slot); 834 835 return slot; 836 } 837 838 int 839 ufshci_utr_cmd_capacity16(struct ufshci_softc *sc, struct ufshci_ccb *ccb, 840 struct scsi_xfer *xs) 841 { 842 int slot, off, len, i; 843 uint64_t dva; 844 struct ufshci_utrd *utrd; 845 struct ufshci_ucd *ucd; 846 bus_dmamap_t dmap = ccb->ccb_dmamap; 847 848 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 1) */ 849 slot = ccb->ccb_slot; 850 utrd = UFSHCI_DMA_KVA(sc->sc_dmamem_utrd); 851 utrd += slot; 852 memset(utrd, 0, sizeof(*utrd)); 853 DPRINTF(3, "%s: slot=%d\n", __func__, slot); 854 855 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2a) */ 856 utrd->dw0 = UFSHCI_UTRD_DW0_CT_UFS; 857 858 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2b) */ 859 utrd->dw0 |= UFSHCI_UTRD_DW0_DD_T2I; 860 861 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2c) */ 862 if (sc->sc_flags & UFSHCI_FLAGS_AGGR_INTR) 863 utrd->dw0 |= UFSHCI_UTRD_DW0_I_REG; 864 else 865 utrd->dw0 |= UFSHCI_UTRD_DW0_I_INT; 866 867 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2d) */ 868 utrd->dw2 = UFSHCI_UTRD_DW2_OCS_IOV; 869 870 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2e) */ 871 ucd = UFSHCI_DMA_KVA(sc->sc_dmamem_ucd); 872 ucd += slot; 873 memset(ucd, 0, sizeof(*ucd)); 874 875 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2f) */ 876 ucd->cmd.hdr.tc = UPIU_TC_I2T_COMMAND; 877 ucd->cmd.hdr.flags = (1 << 6); /* Bit-5 = Write, Bit-6 = Read */ 878 ucd->cmd.hdr.lun = 0; 879 ucd->cmd.hdr.task_tag = slot; 880 ucd->cmd.hdr.cmd_set_type = 0; /* SCSI command */ 881 ucd->cmd.hdr.query = 0; 882 ucd->cmd.hdr.response = 0; 883 ucd->cmd.hdr.status = 0; 884 ucd->cmd.hdr.ehs_len = 0; 885 ucd->cmd.hdr.device_info = 0; 886 ucd->cmd.hdr.ds_len = 0; 887 888 ucd->cmd.expected_xfer_len = htobe32(xs->datalen); 889 890 ucd->cmd.cdb[0] = READ_CAPACITY_16; /* 0x9e */ 891 ucd->cmd.cdb[1] = 0x10; /* Service Action */ 892 /* Logical Block Address = 0 for UFS */ 893 ucd->cmd.cdb[10] = 0; 894 ucd->cmd.cdb[11] = 0; 895 ucd->cmd.cdb[12] = 0; 896 ucd->cmd.cdb[13] = xs->datalen; 897 898 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2g) */ 899 /* Already done with above memset */ 900 901 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 3) */ 902 dva = UFSHCI_DMA_DVA(sc->sc_dmamem_ucd) + (sizeof(*ucd) * slot); 903 DPRINTF(3, "%s: ucd dva=%llu\n", __func__, dva); 904 utrd->dw4 = (uint32_t)dva; 905 utrd->dw5 = (uint32_t)(dva >> 32); 906 907 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 4) */ 908 off = sizeof(struct upiu_command) / 4; /* DWORD offset */ 909 utrd->dw6 = UFSHCI_UTRD_DW6_RUO(off); 910 911 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 5) */ 912 len = sizeof(struct upiu_response) / 4; /* DWORD length */ 913 utrd->dw6 |= UFSHCI_UTRD_DW6_RUL(len); 914 915 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 6) */ 916 off = (sizeof(struct upiu_command) + sizeof(struct upiu_response)) / 4; 917 utrd->dw7 = UFSHCI_UTRD_DW7_PRDTO(off); 918 919 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 7) */ 920 utrd->dw7 |= UFSHCI_UTRD_DW7_PRDTL(dmap->dm_nsegs); 921 922 /* Build PRDT data segment. */ 923 for (i = 0; i < dmap->dm_nsegs; i++) { 924 dva = dmap->dm_segs[i].ds_addr; 925 ucd->prdt[i].dw0 = (uint32_t)dva; 926 ucd->prdt[i].dw1 = (uint32_t)(dva >> 32); 927 ucd->prdt[i].dw2 = 0; 928 ucd->prdt[i].dw3 = dmap->dm_segs[i].ds_len - 1; 929 } 930 931 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 9) */ 932 if (UFSHCI_READ_4(sc, UFSHCI_REG_UTRLRSR) != 1) { 933 printf("%s: %s: UTRLRSR not set\n", 934 sc->sc_dev.dv_xname, __func__); 935 return -1; 936 } 937 938 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_utrd), 939 sizeof(*utrd) * slot, sizeof(*utrd), BUS_DMASYNC_PREWRITE); 940 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_ucd), 941 sizeof(*ucd) * slot, sizeof(*ucd), BUS_DMASYNC_PREWRITE); 942 943 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 14) */ 944 ccb->ccb_status = CCB_STATUS_INPROGRESS; 945 ufshci_doorbell_write(sc, slot); 946 947 return slot; 948 } 949 950 int 951 ufshci_utr_cmd_capacity(struct ufshci_softc *sc, struct ufshci_ccb *ccb, 952 struct scsi_xfer *xs) 953 { 954 int slot, off, len, i; 955 uint64_t dva; 956 struct ufshci_utrd *utrd; 957 struct ufshci_ucd *ucd; 958 bus_dmamap_t dmap = ccb->ccb_dmamap; 959 960 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 1) */ 961 slot = ccb->ccb_slot; 962 utrd = UFSHCI_DMA_KVA(sc->sc_dmamem_utrd); 963 utrd += slot; 964 memset(utrd, 0, sizeof(*utrd)); 965 DPRINTF(3, "%s: slot=%d\n", __func__, slot); 966 967 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2a) */ 968 utrd->dw0 = UFSHCI_UTRD_DW0_CT_UFS; 969 970 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2b) */ 971 utrd->dw0 |= UFSHCI_UTRD_DW0_DD_T2I; 972 973 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2c) */ 974 if (sc->sc_flags & UFSHCI_FLAGS_AGGR_INTR) 975 utrd->dw0 |= UFSHCI_UTRD_DW0_I_REG; 976 else 977 utrd->dw0 |= UFSHCI_UTRD_DW0_I_INT; 978 979 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2d) */ 980 utrd->dw2 = UFSHCI_UTRD_DW2_OCS_IOV; 981 982 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2e) */ 983 ucd = UFSHCI_DMA_KVA(sc->sc_dmamem_ucd); 984 ucd += slot; 985 memset(ucd, 0, sizeof(*ucd)); 986 987 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2f) */ 988 ucd->cmd.hdr.tc = UPIU_TC_I2T_COMMAND; 989 ucd->cmd.hdr.flags = (1 << 6); /* Bit-5 = Write, Bit-6 = Read */ 990 ucd->cmd.hdr.lun = 0; 991 ucd->cmd.hdr.task_tag = slot; 992 ucd->cmd.hdr.cmd_set_type = 0; /* SCSI command */ 993 ucd->cmd.hdr.query = 0; 994 ucd->cmd.hdr.response = 0; 995 ucd->cmd.hdr.status = 0; 996 ucd->cmd.hdr.ehs_len = 0; 997 ucd->cmd.hdr.device_info = 0; 998 ucd->cmd.hdr.ds_len = 0; 999 1000 ucd->cmd.expected_xfer_len = htobe32(xs->datalen); 1001 1002 ucd->cmd.cdb[0] = READ_CAPACITY; /* 0x25 */ 1003 /* Logical Block Address = 0 for UFS */ 1004 ucd->cmd.cdb[2] = 0; 1005 ucd->cmd.cdb[3] = 0; 1006 ucd->cmd.cdb[4] = 0; 1007 ucd->cmd.cdb[5] = 0; 1008 1009 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2g) */ 1010 /* Already done with above memset */ 1011 1012 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 3) */ 1013 dva = UFSHCI_DMA_DVA(sc->sc_dmamem_ucd) + (sizeof(*ucd) * slot); 1014 DPRINTF(3, "%s: ucd dva=%llu\n", __func__, dva); 1015 utrd->dw4 = (uint32_t)dva; 1016 utrd->dw5 = (uint32_t)(dva >> 32); 1017 1018 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 4) */ 1019 off = sizeof(struct upiu_command) / 4; /* DWORD offset */ 1020 utrd->dw6 = UFSHCI_UTRD_DW6_RUO(off); 1021 1022 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 5) */ 1023 len = sizeof(struct upiu_response) / 4; /* DWORD length */ 1024 utrd->dw6 |= UFSHCI_UTRD_DW6_RUL(len); 1025 1026 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 6) */ 1027 off = (sizeof(struct upiu_command) + sizeof(struct upiu_response)) / 4; 1028 utrd->dw7 = UFSHCI_UTRD_DW7_PRDTO(off); 1029 1030 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 7) */ 1031 utrd->dw7 |= UFSHCI_UTRD_DW7_PRDTL(dmap->dm_nsegs); 1032 1033 /* Build PRDT data segment. */ 1034 for (i = 0; i < dmap->dm_nsegs; i++) { 1035 dva = dmap->dm_segs[i].ds_addr; 1036 ucd->prdt[i].dw0 = (uint32_t)dva; 1037 ucd->prdt[i].dw1 = (uint32_t)(dva >> 32); 1038 ucd->prdt[i].dw2 = 0; 1039 ucd->prdt[i].dw3 = dmap->dm_segs[i].ds_len - 1; 1040 } 1041 1042 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 9) */ 1043 if (UFSHCI_READ_4(sc, UFSHCI_REG_UTRLRSR) != 1) { 1044 printf("%s: %s: UTRLRSR not set\n", 1045 sc->sc_dev.dv_xname, __func__); 1046 return -1; 1047 } 1048 1049 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_utrd), 1050 sizeof(*utrd) * slot, sizeof(*utrd), BUS_DMASYNC_PREWRITE); 1051 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_ucd), 1052 sizeof(*ucd) * slot, sizeof(*ucd), BUS_DMASYNC_PREWRITE); 1053 1054 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 14) */ 1055 ccb->ccb_status = CCB_STATUS_INPROGRESS; 1056 ufshci_doorbell_write(sc, slot); 1057 1058 return slot; 1059 } 1060 1061 int 1062 ufshci_utr_cmd_io(struct ufshci_softc *sc, struct ufshci_ccb *ccb, 1063 struct scsi_xfer *xs, int dir) 1064 { 1065 int slot, off, len, i; 1066 uint64_t dva; 1067 struct ufshci_utrd *utrd; 1068 struct ufshci_ucd *ucd; 1069 bus_dmamap_t dmap = ccb->ccb_dmamap; 1070 uint32_t blocks; 1071 uint64_t lba; 1072 1073 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 1) */ 1074 slot = ccb->ccb_slot; 1075 utrd = UFSHCI_DMA_KVA(sc->sc_dmamem_utrd); 1076 utrd += slot; 1077 memset(utrd, 0, sizeof(*utrd)); 1078 DPRINTF(3, "%s: slot=%d\n", __func__, slot); 1079 1080 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2a) */ 1081 utrd->dw0 = UFSHCI_UTRD_DW0_CT_UFS; 1082 1083 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2b) */ 1084 if (dir == SCSI_DATA_IN) 1085 utrd->dw0 |= UFSHCI_UTRD_DW0_DD_T2I; 1086 else 1087 utrd->dw0 |= UFSHCI_UTRD_DW0_DD_I2T; 1088 1089 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2c) */ 1090 if (sc->sc_flags & UFSHCI_FLAGS_AGGR_INTR) 1091 utrd->dw0 |= UFSHCI_UTRD_DW0_I_REG; 1092 else 1093 utrd->dw0 |= UFSHCI_UTRD_DW0_I_INT; 1094 1095 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2d) */ 1096 utrd->dw2 = UFSHCI_UTRD_DW2_OCS_IOV; 1097 1098 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2e) */ 1099 ucd = UFSHCI_DMA_KVA(sc->sc_dmamem_ucd); 1100 ucd += slot; 1101 memset(ucd, 0, sizeof(*ucd)); 1102 1103 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2f) */ 1104 ucd->cmd.hdr.tc = UPIU_TC_I2T_COMMAND; 1105 if (dir == SCSI_DATA_IN) 1106 ucd->cmd.hdr.flags = (1 << 6); /* Bit-6 = Read */ 1107 else 1108 ucd->cmd.hdr.flags = (1 << 5); /* Bit-5 = Write */ 1109 ucd->cmd.hdr.lun = 0; 1110 ucd->cmd.hdr.task_tag = slot; 1111 ucd->cmd.hdr.cmd_set_type = 0; /* SCSI command */ 1112 ucd->cmd.hdr.query = 0; 1113 ucd->cmd.hdr.response = 0; 1114 ucd->cmd.hdr.status = 0; 1115 ucd->cmd.hdr.ehs_len = 0; 1116 ucd->cmd.hdr.device_info = 0; 1117 ucd->cmd.hdr.ds_len = 0; 1118 1119 /* 1120 * JESD220C-2_1.pdf, page 88, d) Expected Data Transfer Length: 1121 * "When the COMMAND UPIU encodes a SCSI WRITE or SCSI READ command 1122 * (specifically WRITE (6), READ (6), WRITE (10), READ (10), 1123 * WRITE (16), or READ (16)), the value of this field shall be the 1124 * product of the Logical Block Size (bLogicalBlockSize) and the 1125 * TRANSFER LENGTH field of the CDB." 1126 */ 1127 scsi_cmd_rw_decode(&xs->cmd, &lba, &blocks); 1128 ucd->cmd.expected_xfer_len = htobe32(UFSHCI_LBS * blocks); 1129 1130 memcpy(ucd->cmd.cdb, &xs->cmd, sizeof(ucd->cmd.cdb)); 1131 1132 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2g) */ 1133 /* Already done with above memset */ 1134 1135 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 3) */ 1136 dva = UFSHCI_DMA_DVA(sc->sc_dmamem_ucd) + (sizeof(*ucd) * slot); 1137 DPRINTF(3, "%s: ucd dva=%llu\n", __func__, dva); 1138 utrd->dw4 = (uint32_t)dva; 1139 utrd->dw5 = (uint32_t)(dva >> 32); 1140 1141 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 4) */ 1142 off = sizeof(struct upiu_command) / 4; /* DWORD offset */ 1143 utrd->dw6 = UFSHCI_UTRD_DW6_RUO(off); 1144 1145 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 5) */ 1146 len = sizeof(struct upiu_response) / 4; /* DWORD length */ 1147 utrd->dw6 |= UFSHCI_UTRD_DW6_RUL(len); 1148 1149 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 6) */ 1150 off = (sizeof(struct upiu_command) + sizeof(struct upiu_response)) / 4; 1151 utrd->dw7 = UFSHCI_UTRD_DW7_PRDTO(off); 1152 1153 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 7) */ 1154 utrd->dw7 |= UFSHCI_UTRD_DW7_PRDTL(dmap->dm_nsegs); 1155 1156 /* Build PRDT data segment. */ 1157 for (i = 0; i < dmap->dm_nsegs; i++) { 1158 dva = dmap->dm_segs[i].ds_addr; 1159 ucd->prdt[i].dw0 = (uint32_t)dva; 1160 ucd->prdt[i].dw1 = (uint32_t)(dva >> 32); 1161 ucd->prdt[i].dw2 = 0; 1162 ucd->prdt[i].dw3 = dmap->dm_segs[i].ds_len - 1; 1163 } 1164 1165 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 9) */ 1166 if (UFSHCI_READ_4(sc, UFSHCI_REG_UTRLRSR) != 1) { 1167 printf("%s: %s: UTRLRSR not set\n", 1168 sc->sc_dev.dv_xname, __func__); 1169 return -1; 1170 } 1171 1172 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_utrd), 1173 sizeof(*utrd) * slot, sizeof(*utrd), BUS_DMASYNC_PREWRITE); 1174 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_ucd), 1175 sizeof(*ucd) * slot, sizeof(*ucd), BUS_DMASYNC_PREWRITE); 1176 1177 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 14) */ 1178 ccb->ccb_status = CCB_STATUS_INPROGRESS; 1179 ufshci_doorbell_write(sc, slot); 1180 1181 return slot; 1182 } 1183 1184 int 1185 ufshci_utr_cmd_sync(struct ufshci_softc *sc, struct ufshci_ccb *ccb, 1186 struct scsi_xfer *xs, uint32_t lba, uint16_t blocks) 1187 { 1188 int slot, off, len; 1189 uint64_t dva; 1190 struct ufshci_utrd *utrd; 1191 struct ufshci_ucd *ucd; 1192 1193 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 1) */ 1194 slot = ccb->ccb_slot; 1195 utrd = UFSHCI_DMA_KVA(sc->sc_dmamem_utrd); 1196 utrd += slot; 1197 memset(utrd, 0, sizeof(*utrd)); 1198 DPRINTF(3, "%s: slot=%d\n", __func__, slot); 1199 1200 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2a) */ 1201 utrd->dw0 = UFSHCI_UTRD_DW0_CT_UFS; 1202 1203 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2b) */ 1204 utrd->dw0 |= UFSHCI_UTRD_DW0_DD_I2T; 1205 1206 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2c) */ 1207 if (sc->sc_flags & UFSHCI_FLAGS_AGGR_INTR) 1208 utrd->dw0 |= UFSHCI_UTRD_DW0_I_REG; 1209 else 1210 utrd->dw0 |= UFSHCI_UTRD_DW0_I_INT; 1211 1212 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2d) */ 1213 utrd->dw2 = UFSHCI_UTRD_DW2_OCS_IOV; 1214 1215 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2e) */ 1216 ucd = UFSHCI_DMA_KVA(sc->sc_dmamem_ucd); 1217 ucd += slot; 1218 memset(ucd, 0, sizeof(*ucd)); 1219 1220 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2f) */ 1221 ucd->cmd.hdr.tc = UPIU_TC_I2T_COMMAND; 1222 ucd->cmd.hdr.flags = 0; /* No data transfer */ 1223 ucd->cmd.hdr.lun = 0; 1224 ucd->cmd.hdr.task_tag = slot; 1225 ucd->cmd.hdr.cmd_set_type = 0; /* SCSI command */ 1226 ucd->cmd.hdr.query = 0; 1227 ucd->cmd.hdr.response = 0; 1228 ucd->cmd.hdr.status = 0; 1229 ucd->cmd.hdr.ehs_len = 0; 1230 ucd->cmd.hdr.device_info = 0; 1231 ucd->cmd.hdr.ds_len = 0; 1232 1233 ucd->cmd.expected_xfer_len = htobe32(0); /* No data transfer */ 1234 1235 ucd->cmd.cdb[0] = SYNCHRONIZE_CACHE; /* 0x35 */ 1236 ucd->cmd.cdb[2] = (lba >> 24) & 0xff; 1237 ucd->cmd.cdb[3] = (lba >> 16) & 0xff; 1238 ucd->cmd.cdb[4] = (lba >> 8) & 0xff; 1239 ucd->cmd.cdb[5] = (lba >> 0) & 0xff; 1240 ucd->cmd.cdb[7] = (blocks >> 8) & 0xff; 1241 ucd->cmd.cdb[8] = (blocks >> 0) & 0xff; 1242 1243 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2g) */ 1244 /* Already done with above memset */ 1245 1246 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 3) */ 1247 dva = UFSHCI_DMA_DVA(sc->sc_dmamem_ucd) + (sizeof(*ucd) * slot); 1248 DPRINTF(3, "%s: ucd dva=%llu\n", __func__, dva); 1249 utrd->dw4 = (uint32_t)dva; 1250 utrd->dw5 = (uint32_t)(dva >> 32); 1251 1252 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 4) */ 1253 off = sizeof(struct upiu_command) / 4; /* DWORD offset */ 1254 utrd->dw6 = UFSHCI_UTRD_DW6_RUO(off); 1255 1256 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 5) */ 1257 len = sizeof(struct upiu_response) / 4; /* DWORD length */ 1258 utrd->dw6 |= UFSHCI_UTRD_DW6_RUL(len); 1259 1260 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 6) */ 1261 off = (sizeof(struct upiu_command) + sizeof(struct upiu_response)) / 4; 1262 utrd->dw7 = UFSHCI_UTRD_DW7_PRDTO(off); 1263 1264 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 7) */ 1265 utrd->dw7 |= UFSHCI_UTRD_DW7_PRDTL(0); /* No data xfer */ 1266 1267 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 9) */ 1268 if (UFSHCI_READ_4(sc, UFSHCI_REG_UTRLRSR) != 1) { 1269 printf("%s: %s: UTRLRSR not set\n", 1270 sc->sc_dev.dv_xname, __func__); 1271 return -1; 1272 } 1273 1274 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_utrd), 1275 sizeof(*utrd) * slot, sizeof(*utrd), BUS_DMASYNC_PREWRITE); 1276 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_ucd), 1277 sizeof(*ucd) * slot, sizeof(*ucd), BUS_DMASYNC_PREWRITE); 1278 1279 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 14) */ 1280 ccb->ccb_status = CCB_STATUS_INPROGRESS; 1281 ufshci_doorbell_write(sc, slot); 1282 1283 return slot; 1284 } 1285 1286 int 1287 ufshci_xfer_complete(struct ufshci_softc *sc) 1288 { 1289 struct ufshci_ccb *ccb; 1290 uint32_t reg; 1291 int i, timeout; 1292 1293 mtx_enter(&sc->sc_cmd_mtx); 1294 1295 /* Wait for all commands to complete. */ 1296 for (timeout = 5000; timeout != 0; timeout--) { 1297 reg = ufshci_doorbell_read(sc); 1298 if (reg == 0) 1299 break; 1300 delay(10); 1301 } 1302 if (timeout == 0) 1303 printf("%s: timeout (reg=0x%x)\n", __func__, reg); 1304 1305 for (i = 0; i < sc->sc_nutrs; i++) { 1306 ccb = &sc->sc_ccbs[i]; 1307 1308 /* Skip unused CCBs. */ 1309 if (ccb->ccb_status != CCB_STATUS_INPROGRESS) 1310 continue; 1311 1312 if (ccb->ccb_done == NULL) 1313 panic("ccb done wasn't defined"); 1314 1315 /* 7.2.3: Clear completion notification 3b) */ 1316 UFSHCI_WRITE_4(sc, UFSHCI_REG_UTRLCNR, (1U << i)); 1317 1318 /* 7.2.3: Mark software slot for re-use 3c) */ 1319 ccb->ccb_status = CCB_STATUS_READY2FREE; 1320 1321 DPRINTF(3, "slot %d completed\n", i); 1322 } 1323 1324 /* 7.2.3: Reset Interrupt Aggregation Counter and Timer 4) */ 1325 if (sc->sc_flags & UFSHCI_FLAGS_AGGR_INTR) { 1326 UFSHCI_WRITE_4(sc, UFSHCI_REG_UTRIACR, 1327 UFSHCI_REG_UTRIACR_IAEN | UFSHCI_REG_UTRIACR_CTR); 1328 } 1329 1330 mtx_leave(&sc->sc_cmd_mtx); 1331 1332 /* 1333 * Complete the CCB, which will re-schedule new transfers if any are 1334 * pending. 1335 */ 1336 for (i = 0; i < sc->sc_nutrs; i++) { 1337 ccb = &sc->sc_ccbs[i]; 1338 1339 /* 7.2.3: Process the transfer by higher OS layer 3a) */ 1340 if (ccb->ccb_status == CCB_STATUS_READY2FREE) 1341 ccb->ccb_done(sc, ccb); 1342 } 1343 1344 return 0; 1345 } 1346 1347 int 1348 ufshci_activate(struct ufshci_softc *sc, int act) 1349 { 1350 int rv = 0; 1351 1352 switch (act) { 1353 case DVACT_POWERDOWN: 1354 DPRINTF(1, "%s: POWERDOWN\n", __func__); 1355 rv = config_activate_children(&sc->sc_dev, act); 1356 ufshci_powerdown(sc); 1357 break; 1358 case DVACT_RESUME: 1359 DPRINTF(1, "%s: RESUME\n", __func__); 1360 ufshci_resume(sc); 1361 if (rv == 0) 1362 rv = config_activate_children(&sc->sc_dev, act); 1363 break; 1364 default: 1365 rv = config_activate_children(&sc->sc_dev, act); 1366 break; 1367 } 1368 1369 return rv; 1370 } 1371 1372 int 1373 ufshci_powerdown(struct ufshci_softc *sc) 1374 { 1375 uint32_t reg; 1376 1377 /* Send "hibernate enter" command. */ 1378 UFSHCI_WRITE_4(sc, UFSHCI_REG_UICCMD, 1379 UFSHCI_REG_UICCMD_CMDOP_DME_HIBERNATE_ENTER); 1380 if (ufshci_is_poll(sc, UFSHCI_REG_IS_UHES) != 0) { 1381 printf("%s: hibernate enter cmd failed\n", __func__); 1382 return 1; 1383 } 1384 1385 /* Check if "hibernate enter" command was executed successfully. */ 1386 reg = UFSHCI_READ_4(sc, UFSHCI_REG_HCS); 1387 DPRINTF(1, "%s: UPMCRS=0x%x\n", __func__, UFSHCI_REG_HCS_UPMCRS(reg)); 1388 if (UFSHCI_REG_HCS_UPMCRS(reg) > UFSHCI_REG_HCS_UPMCRS_PWR_REMTOTE) { 1389 printf("%s: hibernate enter cmd returned UPMCRS error=0x%x\n", 1390 __func__, UFSHCI_REG_HCS_UPMCRS(reg)); 1391 return 1; 1392 } 1393 1394 return 0; 1395 } 1396 1397 int 1398 ufshci_resume(struct ufshci_softc *sc) 1399 { 1400 uint32_t reg; 1401 1402 /* Send "hibernate exit" command. */ 1403 UFSHCI_WRITE_4(sc, UFSHCI_REG_UICCMD, 1404 UFSHCI_REG_UICCMD_CMDOP_DME_HIBERNATE_EXIT); 1405 if (ufshci_is_poll(sc, UFSHCI_REG_IS_UHXS) != 0) { 1406 printf("%s: hibernate exit command failed\n", __func__); 1407 return 1; 1408 } 1409 1410 /* Check if "hibernate exit" command was executed successfully. */ 1411 reg = UFSHCI_READ_4(sc, UFSHCI_REG_HCS); 1412 DPRINTF(1, "%s: UPMCRS=0x%x\n", __func__, UFSHCI_REG_HCS_UPMCRS(reg)); 1413 if (UFSHCI_REG_HCS_UPMCRS(reg) > UFSHCI_REG_HCS_UPMCRS_PWR_REMTOTE) { 1414 printf("%s: hibernate exit cmd returned UPMCRS error=0x%x\n", 1415 __func__, UFSHCI_REG_HCS_UPMCRS(reg)); 1416 return 1; 1417 } 1418 1419 return 0; 1420 } 1421 1422 /* SCSI */ 1423 1424 int 1425 ufshci_ccb_alloc(struct ufshci_softc *sc, int nccbs) 1426 { 1427 struct ufshci_ccb *ccb; 1428 int i; 1429 1430 DPRINTF(2, "%s: nccbs=%d, dma_size=%d, dma_nsegs=%d, " 1431 "dma_segmaxsize=%d\n", 1432 __func__, nccbs, UFSHCI_UCD_PRDT_MAX_XFER, UFSHCI_UCD_PRDT_MAX_SEGS, 1433 UFSHCI_UCD_PRDT_MAX_XFER); 1434 1435 sc->sc_ccbs = mallocarray(nccbs, sizeof(*ccb), M_DEVBUF, 1436 M_WAITOK | M_CANFAIL); 1437 if (sc->sc_ccbs == NULL) 1438 return 1; 1439 1440 for (i = 0; i < nccbs; i++) { 1441 ccb = &sc->sc_ccbs[i]; 1442 1443 if (bus_dmamap_create(sc->sc_dmat, UFSHCI_UCD_PRDT_MAX_XFER, 1444 UFSHCI_UCD_PRDT_MAX_SEGS, UFSHCI_UCD_PRDT_MAX_XFER, 0, 1445 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW | 1446 (sc->sc_cap & UFSHCI_REG_CAP_64AS) ? BUS_DMA_64BIT : 0, 1447 &ccb->ccb_dmamap) != 0) 1448 goto free_maps; 1449 1450 ccb->ccb_cookie = NULL; 1451 ccb->ccb_status = CCB_STATUS_FREE; 1452 ccb->ccb_slot = i; 1453 1454 SIMPLEQ_INSERT_TAIL(&sc->sc_ccb_list, ccb, ccb_entry); 1455 } 1456 1457 return 0; 1458 1459 free_maps: 1460 ufshci_ccb_free(sc, nccbs); 1461 return 1; 1462 } 1463 1464 void * 1465 ufshci_ccb_get(void *cookie) 1466 { 1467 struct ufshci_softc *sc = cookie; 1468 struct ufshci_ccb *ccb; 1469 1470 DPRINTF(3, "%s\n", __func__); 1471 1472 mtx_enter(&sc->sc_ccb_mtx); 1473 ccb = SIMPLEQ_FIRST(&sc->sc_ccb_list); 1474 if (ccb != NULL) 1475 SIMPLEQ_REMOVE_HEAD(&sc->sc_ccb_list, ccb_entry); 1476 mtx_leave(&sc->sc_ccb_mtx); 1477 1478 return ccb; 1479 } 1480 1481 void 1482 ufshci_ccb_put(void *cookie, void *io) 1483 { 1484 struct ufshci_softc *sc = cookie; 1485 struct ufshci_ccb *ccb = io; 1486 1487 DPRINTF(3, "%s\n", __func__); 1488 1489 mtx_enter(&sc->sc_ccb_mtx); 1490 SIMPLEQ_INSERT_HEAD(&sc->sc_ccb_list, ccb, ccb_entry); 1491 mtx_leave(&sc->sc_ccb_mtx); 1492 } 1493 1494 void 1495 ufshci_ccb_free(struct ufshci_softc *sc, int nccbs) 1496 { 1497 struct ufshci_ccb *ccb; 1498 1499 DPRINTF(3, "%s\n", __func__); 1500 1501 while ((ccb = SIMPLEQ_FIRST(&sc->sc_ccb_list)) != NULL) { 1502 SIMPLEQ_REMOVE_HEAD(&sc->sc_ccb_list, ccb_entry); 1503 bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap); 1504 } 1505 1506 ufshci_dmamem_free(sc, sc->sc_dmamem_utrd); 1507 free(sc->sc_ccbs, M_DEVBUF, nccbs * sizeof(*ccb)); 1508 } 1509 1510 void 1511 ufshci_scsi_cmd(struct scsi_xfer *xs) 1512 { 1513 struct scsi_link *link = xs->sc_link; 1514 struct ufshci_softc *sc = link->bus->sb_adapter_softc; 1515 1516 mtx_enter(&sc->sc_cmd_mtx); 1517 1518 DPRINTF(3, "%s: cmd=0x%x\n", __func__, xs->cmd.opcode); 1519 1520 switch (xs->cmd.opcode) { 1521 1522 case READ_COMMAND: 1523 case READ_10: 1524 case READ_12: 1525 case READ_16: 1526 DPRINTF(3, "io read\n"); 1527 ufshci_scsi_io(xs, SCSI_DATA_IN); 1528 break; 1529 case WRITE_COMMAND: 1530 case WRITE_10: 1531 case WRITE_12: 1532 case WRITE_16: 1533 DPRINTF(3, "io write\n"); 1534 ufshci_scsi_io(xs, SCSI_DATA_OUT); 1535 break; 1536 case SYNCHRONIZE_CACHE: 1537 DPRINTF(3, "sync\n"); 1538 ufshci_scsi_sync(xs); 1539 break; 1540 case INQUIRY: 1541 DPRINTF(3, "inquiry\n"); 1542 ufshci_scsi_inquiry(xs); 1543 break; 1544 case READ_CAPACITY_16: 1545 DPRINTF(3, "capacity16\n"); 1546 ufshci_scsi_capacity16(xs); 1547 break; 1548 case READ_CAPACITY: 1549 DPRINTF(3, "capacity\n"); 1550 ufshci_scsi_capacity(xs); 1551 break; 1552 case TEST_UNIT_READY: 1553 case PREVENT_ALLOW: 1554 case START_STOP: 1555 xs->error = XS_NOERROR; 1556 scsi_done(xs); 1557 break; 1558 default: 1559 DPRINTF(3, "%s: unhandled scsi command 0x%02x\n", 1560 __func__, xs->cmd.opcode); 1561 xs->error = XS_DRIVER_STUFFUP; 1562 scsi_done(xs); 1563 break; 1564 } 1565 1566 mtx_leave(&sc->sc_cmd_mtx); 1567 } 1568 1569 void 1570 ufshci_minphys(struct buf *bp, struct scsi_link *link) 1571 { 1572 DPRINTF(3, "%s\n", __func__); 1573 } 1574 1575 int 1576 ufshci_scsi_probe(struct scsi_link *link) 1577 { 1578 DPRINTF(3, "%s\n", __func__); 1579 1580 return 0; 1581 } 1582 1583 void 1584 ufshci_scsi_free(struct scsi_link *link) 1585 { 1586 DPRINTF(3, "%s\n", __func__); 1587 } 1588 1589 void 1590 ufshci_scsi_inquiry(struct scsi_xfer *xs) 1591 { 1592 struct scsi_link *link = xs->sc_link; 1593 struct ufshci_softc *sc = link->bus->sb_adapter_softc; 1594 struct ufshci_ccb *ccb = xs->io; 1595 bus_dmamap_t dmap = ccb->ccb_dmamap; 1596 int error; 1597 1598 DPRINTF(3, "%s: INQUIRY (%s)\n", 1599 __func__, ISSET(xs->flags, SCSI_POLL) ? "poll" : "no poll"); 1600 1601 if (xs->datalen > UPIU_SCSI_RSP_INQUIRY_SIZE) { 1602 DPRINTF(2, "%s: request len too large\n", __func__); 1603 goto error1; 1604 } 1605 1606 error = bus_dmamap_load(sc->sc_dmat, dmap, xs->data, xs->datalen, NULL, 1607 ISSET(xs->flags, SCSI_NOSLEEP) ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK); 1608 if (error != 0) { 1609 printf("%s: bus_dmamap_load error=%d\n", __func__, error); 1610 goto error1; 1611 } 1612 1613 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 1614 BUS_DMASYNC_PREREAD); 1615 1616 ccb->ccb_cookie = xs; 1617 ccb->ccb_done = ufshci_scsi_io_done; 1618 1619 /* Response length should be UPIU_SCSI_RSP_INQUIRY_SIZE. */ 1620 error = ufshci_utr_cmd_inquiry(sc, ccb, xs); 1621 if (error == -1) 1622 goto error2; 1623 1624 if (ISSET(xs->flags, SCSI_POLL)) { 1625 if (ufshci_doorbell_poll(sc, ccb->ccb_slot, xs->timeout) == 0) { 1626 ccb->ccb_done(sc, ccb); 1627 return; 1628 } 1629 goto error2; 1630 } 1631 1632 return; 1633 1634 error2: 1635 bus_dmamap_unload(sc->sc_dmat, dmap); 1636 ccb->ccb_cookie = NULL; 1637 ccb->ccb_status = CCB_STATUS_FREE; 1638 ccb->ccb_done = NULL; 1639 error1: 1640 xs->error = XS_DRIVER_STUFFUP; 1641 scsi_done(xs); 1642 } 1643 1644 void 1645 ufshci_scsi_capacity16(struct scsi_xfer *xs) 1646 { 1647 struct scsi_link *link = xs->sc_link; 1648 struct ufshci_softc *sc = link->bus->sb_adapter_softc; 1649 struct ufshci_ccb *ccb = xs->io; 1650 bus_dmamap_t dmap = ccb->ccb_dmamap; 1651 int error; 1652 1653 DPRINTF(3, "%s: CAPACITY16 (%s)\n", 1654 __func__, ISSET(xs->flags, SCSI_POLL) ? "poll" : "no poll"); 1655 1656 if (xs->datalen > UPIU_SCSI_RSP_CAPACITY16_SIZE) { 1657 DPRINTF(2, "%s: request len too large\n", __func__); 1658 goto error1; 1659 } 1660 1661 error = bus_dmamap_load(sc->sc_dmat, dmap, xs->data, xs->datalen, NULL, 1662 ISSET(xs->flags, SCSI_NOSLEEP) ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK); 1663 if (error != 0) { 1664 printf("%s: bus_dmamap_load error=%d\n", __func__, error); 1665 goto error1; 1666 } 1667 1668 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 1669 BUS_DMASYNC_PREREAD); 1670 1671 ccb->ccb_cookie = xs; 1672 ccb->ccb_done = ufshci_scsi_io_done; 1673 1674 /* Response length should be UPIU_SCSI_RSP_CAPACITY16_SIZE. */ 1675 error = ufshci_utr_cmd_capacity16(sc, ccb, xs); 1676 if (error == -1) 1677 goto error2; 1678 1679 if (ISSET(xs->flags, SCSI_POLL)) { 1680 if (ufshci_doorbell_poll(sc, ccb->ccb_slot, xs->timeout) == 0) { 1681 ccb->ccb_done(sc, ccb); 1682 return; 1683 } 1684 goto error2; 1685 } 1686 1687 return; 1688 1689 error2: 1690 bus_dmamap_unload(sc->sc_dmat, dmap); 1691 ccb->ccb_cookie = NULL; 1692 ccb->ccb_status = CCB_STATUS_FREE; 1693 ccb->ccb_done = NULL; 1694 error1: 1695 xs->error = XS_DRIVER_STUFFUP; 1696 scsi_done(xs); 1697 } 1698 1699 void 1700 ufshci_scsi_capacity(struct scsi_xfer *xs) 1701 { 1702 struct scsi_link *link = xs->sc_link; 1703 struct ufshci_softc *sc = link->bus->sb_adapter_softc; 1704 struct ufshci_ccb *ccb = xs->io; 1705 bus_dmamap_t dmap = ccb->ccb_dmamap; 1706 int error; 1707 1708 DPRINTF(3, "%s: CAPACITY (%s)\n", 1709 __func__, ISSET(xs->flags, SCSI_POLL) ? "poll" : "no poll"); 1710 1711 if (xs->datalen > UPIU_SCSI_RSP_CAPACITY_SIZE) { 1712 DPRINTF(2, "%s: request len too large\n", __func__); 1713 goto error1; 1714 } 1715 1716 error = bus_dmamap_load(sc->sc_dmat, dmap, xs->data, xs->datalen, NULL, 1717 ISSET(xs->flags, SCSI_NOSLEEP) ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK); 1718 if (error != 0) { 1719 printf("%s: bus_dmamap_load error=%d\n", __func__, error); 1720 goto error1; 1721 } 1722 1723 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 1724 BUS_DMASYNC_PREREAD); 1725 1726 ccb->ccb_cookie = xs; 1727 ccb->ccb_done = ufshci_scsi_io_done; 1728 1729 /* Response length should be UPIU_SCSI_RSP_CAPACITY_SIZE */ 1730 error = ufshci_utr_cmd_capacity(sc, ccb, xs); 1731 if (error == -1) 1732 goto error2; 1733 1734 if (ISSET(xs->flags, SCSI_POLL)) { 1735 if (ufshci_doorbell_poll(sc, ccb->ccb_slot, xs->timeout) == 0) { 1736 ccb->ccb_done(sc, ccb); 1737 return; 1738 } 1739 goto error2; 1740 } 1741 1742 return; 1743 1744 error2: 1745 bus_dmamap_unload(sc->sc_dmat, dmap); 1746 ccb->ccb_cookie = NULL; 1747 ccb->ccb_status = CCB_STATUS_FREE; 1748 ccb->ccb_done = NULL; 1749 error1: 1750 xs->error = XS_DRIVER_STUFFUP; 1751 scsi_done(xs); 1752 } 1753 1754 void 1755 ufshci_scsi_sync(struct scsi_xfer *xs) 1756 { 1757 struct scsi_link *link = xs->sc_link; 1758 struct ufshci_softc *sc = link->bus->sb_adapter_softc; 1759 struct ufshci_ccb *ccb = xs->io; 1760 uint64_t lba; 1761 uint32_t blocks; 1762 int error; 1763 1764 /* lba = 0, blocks = 0: Synchronize all logical blocks. */ 1765 lba = 0; blocks = 0; 1766 1767 DPRINTF(3, "%s: SYNC, lba=%llu, blocks=%u (%s)\n", 1768 __func__, lba, blocks, 1769 ISSET(xs->flags, SCSI_POLL) ? "poll" : "no poll"); 1770 1771 ccb->ccb_cookie = xs; 1772 ccb->ccb_done = ufshci_scsi_done; 1773 1774 error = ufshci_utr_cmd_sync(sc, ccb, xs, (uint32_t)lba, 1775 (uint16_t)blocks); 1776 if (error == -1) 1777 goto error; 1778 1779 if (ISSET(xs->flags, SCSI_POLL)) { 1780 if (ufshci_doorbell_poll(sc, ccb->ccb_slot, xs->timeout) == 0) { 1781 ccb->ccb_done(sc, ccb); 1782 return; 1783 } 1784 goto error; 1785 } 1786 1787 return; 1788 1789 error: 1790 ccb->ccb_cookie = NULL; 1791 ccb->ccb_status = CCB_STATUS_FREE; 1792 ccb->ccb_done = NULL; 1793 1794 xs->error = XS_DRIVER_STUFFUP; 1795 scsi_done(xs); 1796 } 1797 1798 void 1799 ufshci_scsi_io(struct scsi_xfer *xs, int dir) 1800 { 1801 struct scsi_link *link = xs->sc_link; 1802 struct ufshci_softc *sc = link->bus->sb_adapter_softc; 1803 struct ufshci_ccb *ccb = xs->io; 1804 bus_dmamap_t dmap = ccb->ccb_dmamap; 1805 int error; 1806 1807 if ((xs->flags & (SCSI_DATA_IN | SCSI_DATA_OUT)) != dir) 1808 goto error1; 1809 1810 DPRINTF(3, "%s: %s, datalen=%d (%s)\n", __func__, 1811 ISSET(xs->flags, SCSI_DATA_IN) ? "READ" : "WRITE", xs->datalen, 1812 ISSET(xs->flags, SCSI_POLL) ? "poll" : "no poll"); 1813 1814 error = bus_dmamap_load(sc->sc_dmat, dmap, xs->data, xs->datalen, NULL, 1815 ISSET(xs->flags, SCSI_NOSLEEP) ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK); 1816 if (error != 0) { 1817 printf("%s: bus_dmamap_load error=%d\n", __func__, error); 1818 goto error1; 1819 } 1820 1821 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 1822 ISSET(xs->flags, SCSI_DATA_IN) ? BUS_DMASYNC_PREREAD : 1823 BUS_DMASYNC_PREWRITE); 1824 1825 ccb->ccb_cookie = xs; 1826 ccb->ccb_done = ufshci_scsi_io_done; 1827 1828 if (dir == SCSI_DATA_IN) 1829 error = ufshci_utr_cmd_io(sc, ccb, xs, SCSI_DATA_IN); 1830 else 1831 error = ufshci_utr_cmd_io(sc, ccb, xs, SCSI_DATA_OUT); 1832 if (error == -1) 1833 goto error2; 1834 1835 if (ISSET(xs->flags, SCSI_POLL)) { 1836 if (ufshci_doorbell_poll(sc, ccb->ccb_slot, xs->timeout) == 0) { 1837 ccb->ccb_done(sc, ccb); 1838 return; 1839 } 1840 goto error2; 1841 } 1842 1843 return; 1844 1845 error2: 1846 bus_dmamap_unload(sc->sc_dmat, dmap); 1847 ccb->ccb_cookie = NULL; 1848 ccb->ccb_status = CCB_STATUS_FREE; 1849 ccb->ccb_done = NULL; 1850 error1: 1851 xs->error = XS_DRIVER_STUFFUP; 1852 scsi_done(xs); 1853 } 1854 1855 void 1856 ufshci_scsi_io_done(struct ufshci_softc *sc, struct ufshci_ccb *ccb) 1857 { 1858 struct scsi_xfer *xs = ccb->ccb_cookie; 1859 bus_dmamap_t dmap = ccb->ccb_dmamap; 1860 struct ufshci_ucd *ucd; 1861 struct ufshci_utrd *utrd; 1862 1863 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 1864 ISSET(xs->flags, SCSI_DATA_IN) ? BUS_DMASYNC_POSTREAD : 1865 BUS_DMASYNC_POSTWRITE); 1866 1867 bus_dmamap_unload(sc->sc_dmat, dmap); 1868 1869 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_ucd), 1870 sizeof(*ucd) * ccb->ccb_slot, sizeof(*ucd), 1871 BUS_DMASYNC_POSTWRITE); 1872 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_utrd), 1873 sizeof(*utrd) * ccb->ccb_slot, sizeof(*utrd), 1874 BUS_DMASYNC_POSTWRITE); 1875 1876 /* TODO: Do more checks on the Response UPIU in case of errors? */ 1877 utrd = UFSHCI_DMA_KVA(sc->sc_dmamem_utrd); 1878 utrd += ccb->ccb_slot; 1879 ucd = UFSHCI_DMA_KVA(sc->sc_dmamem_ucd); 1880 ucd += ccb->ccb_slot; 1881 if (utrd->dw2 != UFSHCI_UTRD_DW2_OCS_SUCCESS) { 1882 printf("%s: error: slot=%d, ocs=0x%x, rsp-tc=0x%x\n", 1883 __func__, ccb->ccb_slot, utrd->dw2, ucd->rsp.hdr.tc); 1884 } 1885 1886 ccb->ccb_cookie = NULL; 1887 ccb->ccb_status = CCB_STATUS_FREE; 1888 ccb->ccb_done = NULL; 1889 1890 xs->error = (utrd->dw2 == UFSHCI_UTRD_DW2_OCS_SUCCESS) ? 1891 XS_NOERROR : XS_DRIVER_STUFFUP; 1892 xs->status = SCSI_OK; 1893 xs->resid = 0; 1894 scsi_done(xs); 1895 } 1896 1897 void 1898 ufshci_scsi_done(struct ufshci_softc *sc, struct ufshci_ccb *ccb) 1899 { 1900 struct scsi_xfer *xs = ccb->ccb_cookie; 1901 struct ufshci_ucd *ucd; 1902 struct ufshci_utrd *utrd; 1903 1904 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_ucd), 1905 sizeof(*ucd) * ccb->ccb_slot, sizeof(*ucd), 1906 BUS_DMASYNC_POSTWRITE); 1907 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_utrd), 1908 sizeof(*utrd) * ccb->ccb_slot, sizeof(*utrd), 1909 BUS_DMASYNC_POSTWRITE); 1910 1911 /* TODO: Do more checks on the Response UPIU in case of errors? */ 1912 utrd = UFSHCI_DMA_KVA(sc->sc_dmamem_utrd); 1913 utrd += ccb->ccb_slot; 1914 ucd = UFSHCI_DMA_KVA(sc->sc_dmamem_ucd); 1915 ucd += ccb->ccb_slot; 1916 if (utrd->dw2 != UFSHCI_UTRD_DW2_OCS_SUCCESS) { 1917 printf("%s: error: slot=%d, ocs=0x%x, rsp-tc=0x%x\n", 1918 __func__, ccb->ccb_slot, utrd->dw2, ucd->rsp.hdr.tc); 1919 } 1920 1921 ccb->ccb_cookie = NULL; 1922 ccb->ccb_status = CCB_STATUS_FREE; 1923 ccb->ccb_done = NULL; 1924 1925 xs->error = (utrd->dw2 == UFSHCI_UTRD_DW2_OCS_SUCCESS) ? 1926 XS_NOERROR : XS_DRIVER_STUFFUP; 1927 xs->status = SCSI_OK; 1928 xs->resid = 0; 1929 scsi_done(xs); 1930 } 1931