1 /* $OpenBSD: ufshci.c,v 1.46 2025/01/18 19:42:39 mglocker Exp $ */
2
3 /*
4 * Copyright (c) 2022 Marcus Glocker <mglocker@openbsd.org>
5 *
6 * Permission to use, copy, modify, and distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18
19 /*
20 * Universal Flash Storage Host Controller Interface (UFSHCI) 2.1 driver based
21 * on the JEDEC JESD223C.pdf and JESD220C-2_1.pdf specifications.
22 */
23
24 #include "kstat.h"
25
26 #include <sys/param.h>
27 #include <sys/systm.h>
28 #include <sys/buf.h>
29 #include <sys/kernel.h>
30 #include <sys/malloc.h>
31 #include <sys/device.h>
32 #include <sys/queue.h>
33 #include <sys/mutex.h>
34 #include <sys/pool.h>
35 #include <sys/kstat.h>
36
37 #include <sys/atomic.h>
38
39 #include <machine/bus.h>
40
41 #include <scsi/scsi_all.h>
42 #include <scsi/scsi_disk.h>
43 #include <scsi/scsiconf.h>
44
45 #include <dev/ic/ufshcivar.h>
46 #include <dev/ic/ufshcireg.h>
47
48 #ifdef HIBERNATE
49 #include <uvm/uvm_extern.h>
50 #include <sys/hibernate.h>
51 #include <sys/disklabel.h>
52 #include <sys/disk.h>
53 #endif
54
55 #ifdef UFSHCI_DEBUG
56 int ufshci_debug = 1;
57 #endif
58
59 struct cfdriver ufshci_cd = {
60 NULL, "ufshci", DV_DULL
61 };
62
63 int ufshci_reset(struct ufshci_softc *);
64 int ufshci_is_poll(struct ufshci_softc *, uint32_t);
65 struct ufshci_dmamem *ufshci_dmamem_alloc(struct ufshci_softc *, size_t);
66 void ufshci_dmamem_free(struct ufshci_softc *,
67 struct ufshci_dmamem *);
68 int ufshci_alloc(struct ufshci_softc *);
69 int ufshci_init(struct ufshci_softc *);
70 void ufshci_disable(struct ufshci_softc *);
71 int ufshci_doorbell_read(struct ufshci_softc *);
72 void ufshci_doorbell_write(struct ufshci_softc *, int);
73 int ufshci_doorbell_poll(struct ufshci_softc *, int,
74 uint32_t);
75 int ufshci_utr_cmd_nop(struct ufshci_softc *,
76 struct ufshci_ccb *, struct scsi_xfer *);
77 int ufshci_utr_cmd_lun(struct ufshci_softc *,
78 struct ufshci_ccb *, struct scsi_xfer *);
79 int ufshci_utr_cmd_inquiry(struct ufshci_softc *,
80 struct ufshci_ccb *, struct scsi_xfer *);
81 int ufshci_utr_cmd_capacity16(struct ufshci_softc *,
82 struct ufshci_ccb *, struct scsi_xfer *);
83 int ufshci_utr_cmd_capacity(struct ufshci_softc *,
84 struct ufshci_ccb *, struct scsi_xfer *);
85 int ufshci_utr_cmd_io(struct ufshci_softc *,
86 struct ufshci_ccb *, struct scsi_xfer *, int);
87 int ufshci_utr_cmd_sync(struct ufshci_softc *,
88 struct ufshci_ccb *, struct scsi_xfer *,
89 uint32_t, uint16_t);
90 void ufshci_xfer_complete(struct ufshci_softc *);
91
92 /* SCSI */
93 int ufshci_ccb_alloc(struct ufshci_softc *, int);
94 void *ufshci_ccb_get(void *);
95 void ufshci_ccb_put(void *, void *);
96 void ufshci_ccb_free(struct ufshci_softc*, int);
97
98 void ufshci_scsi_cmd(struct scsi_xfer *);
99
100 void ufshci_scsi_inquiry(struct scsi_xfer *);
101 void ufshci_scsi_capacity16(struct scsi_xfer *);
102 void ufshci_scsi_capacity(struct scsi_xfer *);
103 void ufshci_scsi_sync(struct scsi_xfer *);
104 void ufshci_scsi_io(struct scsi_xfer *, int);
105 void ufshci_scsi_io_done(struct ufshci_softc *,
106 struct ufshci_ccb *);
107 void ufshci_scsi_done(struct ufshci_softc *,
108 struct ufshci_ccb *);
109
110 #ifdef HIBERNATE
111 int ufshci_hibernate_io(dev_t, daddr_t, vaddr_t, size_t,
112 int, void *);
113 #endif
114
115 #if NKSTAT > 0
116 void ufshci_kstat_attach(struct ufshci_softc *);
117 int ufshci_kstat_read_ccb(struct kstat *);
118 int ufshci_kstat_read_slot(struct kstat *);
119 #endif
120
121 const struct scsi_adapter ufshci_switch = {
122 ufshci_scsi_cmd, NULL, NULL, NULL, NULL
123 };
124
125 int
ufshci_intr(void * arg)126 ufshci_intr(void *arg)
127 {
128 struct ufshci_softc *sc = arg;
129 uint32_t status, hcs;
130 int handled = 0;
131
132 status = UFSHCI_READ_4(sc, UFSHCI_REG_IS);
133
134 if (status == 0)
135 return handled;
136
137 /* ACK interrupt */
138 UFSHCI_WRITE_4(sc, UFSHCI_REG_IS, status);
139
140 if (status & UFSHCI_REG_IS_UCCS) {
141 handled = 1;
142 }
143 if (status & UFSHCI_REG_IS_UTRCS) {
144 ufshci_xfer_complete(sc);
145
146 handled = 1;
147 }
148 /* If Auto-Hibernate raises an interrupt, it's to yield an error. */
149 if (status & UFSHCI_REG_IS_UHES) {
150 hcs = UFSHCI_READ_4(sc, UFSHCI_REG_HCS);
151 printf("%s: Auto-Hibernate enter error UPMCRS=0x%x\n",
152 __func__, UFSHCI_REG_HCS_UPMCRS(hcs));
153 handled = 1;
154 }
155 if (status & UFSHCI_REG_IS_UHXS) {
156 hcs = UFSHCI_READ_4(sc, UFSHCI_REG_HCS);
157 printf("%s: Auto-Hibernate exit error UPMCRS=0x%x\n",
158 __func__, UFSHCI_REG_HCS_UPMCRS(hcs));
159 handled = 1;
160 }
161
162 if (handled == 0) {
163 printf("%s: UNKNOWN interrupt, status=0x%08x\n",
164 sc->sc_dev.dv_xname, status);
165 }
166
167 return handled;
168 }
169
170 int
ufshci_attach(struct ufshci_softc * sc)171 ufshci_attach(struct ufshci_softc *sc)
172 {
173 struct scsibus_attach_args saa;
174
175 mtx_init(&sc->sc_cmd_mtx, IPL_BIO);
176 mtx_init(&sc->sc_ccb_mtx, IPL_BIO);
177 SIMPLEQ_INIT(&sc->sc_ccb_list);
178 scsi_iopool_init(&sc->sc_iopool, sc, ufshci_ccb_get, ufshci_ccb_put);
179
180 if (ufshci_reset(sc))
181 return 1;
182
183 sc->sc_ver = UFSHCI_READ_4(sc, UFSHCI_REG_VER);
184 printf(", UFSHCI %d.%d%d\n",
185 UFSHCI_REG_VER_MAJOR(sc->sc_ver),
186 UFSHCI_REG_VER_MINOR(sc->sc_ver),
187 UFSHCI_REG_VER_SUFFIX(sc->sc_ver));
188
189 sc->sc_cap = UFSHCI_READ_4(sc, UFSHCI_REG_CAP);
190 sc->sc_hcpid = UFSHCI_READ_4(sc, UFSHCI_REG_HCPID);
191 sc->sc_hcmid = UFSHCI_READ_4(sc, UFSHCI_REG_HCMID);
192 sc->sc_nutmrs = UFSHCI_REG_CAP_NUTMRS(sc->sc_cap) + 1;
193 sc->sc_rtt = UFSHCI_REG_CAP_RTT(sc->sc_cap) + 1;
194 sc->sc_nutrs = UFSHCI_REG_CAP_NUTRS(sc->sc_cap) + 1;
195
196 #ifdef UFSHCI_DEBUG
197 printf("Capabilities (0x%08x):\n", sc->sc_cap);
198 printf(" CS=%d\n", sc->sc_cap & UFSHCI_REG_CAP_CS ? 1 : 0);
199 printf(" UICDMETMS=%d\n", sc->sc_cap & UFSHCI_REG_CAP_UICDMETMS ? 1 :0);
200 printf(" OODDS=%d\n", sc->sc_cap & UFSHCI_REG_CAP_OODDS ? 1 : 0);
201 printf(" 64AS=%d\n", sc->sc_cap & UFSHCI_REG_CAP_64AS ? 1 : 0);
202 printf(" AUTOH8=%d\n", sc->sc_cap & UFSHCI_REG_AUTOH8 ? 1 : 0);
203 printf(" NUTMRS=%d\n", sc->sc_nutmrs);
204 printf(" RTT=%d\n", sc->sc_rtt);
205 printf(" NUTRS=%d\n", sc->sc_nutrs);
206 printf(" HCPID=0x%08x\n", sc->sc_hcpid);
207 printf("HCMID (0x%08x):\n", sc->sc_hcmid);
208 printf(" BI=0x%04x\n", UFSHCI_REG_HCMID_BI(sc->sc_hcmid));
209 printf(" MIC=0x%04x\n", UFSHCI_REG_HCMID_MIC(sc->sc_hcmid));
210 #endif
211 if (sc->sc_nutrs < UFSHCI_SLOTS_MIN ||
212 sc->sc_nutrs > UFSHCI_SLOTS_MAX) {
213 printf("%s: Invalid NUTRS value %d (must be %d-%d)!\n",
214 sc->sc_dev.dv_xname, sc->sc_nutrs,
215 UFSHCI_SLOTS_MIN, UFSHCI_SLOTS_MAX);
216 return 1;
217 }
218 if (sc->sc_nutrs == UFSHCI_SLOTS_MAX)
219 sc->sc_iacth = UFSHCI_INTR_AGGR_COUNT_MAX;
220 else
221 sc->sc_iacth = sc->sc_nutrs;
222 DPRINTF(1, "Intr. aggr. counter threshold:\nIACTH=%d\n", sc->sc_iacth);
223
224 /*
225 * XXX:
226 * At the moment normal interrupts work better for us than interrupt
227 * aggregation, because:
228 *
229 * 1. With interrupt aggregation enabled, the I/O performance
230 * isn't better, but even slightly worse depending on the
231 * UFS controller and architecture.
232 * 2. With interrupt aggregation enabled we currently see
233 * intermittent SCSI command stalling. Probably there is a
234 * race condition where new SCSI commands are getting
235 * scheduled, while we miss to reset the interrupt aggregation
236 * counter/timer, which leaves us with no more interrupts
237 * triggered. This needs to be fixed, but I couldn't figure
238 * out yet how.
239 */
240 #if 0
241 sc->sc_flags |= UFSHCI_FLAGS_AGGR_INTR; /* Enable intr. aggregation */
242 #endif
243 /* Allocate the DMA buffers and initialize the controller. */
244 if (ufshci_alloc(sc))
245 return 1;
246 if (ufshci_init(sc))
247 return 1;
248
249 if (ufshci_ccb_alloc(sc, sc->sc_nutrs) != 0) {
250 printf("%s: %s: Can't allocate CCBs\n",
251 sc->sc_dev.dv_xname, __func__);
252 return 1;
253 }
254
255 /* Enable Auto-Hibernate Idle Timer (AHIT) and set it to 150ms. */
256 if (sc->sc_cap & UFSHCI_REG_AUTOH8) {
257 UFSHCI_WRITE_4(sc, UFSHCI_REG_AHIT,
258 UFSHCI_REG_AHIT_TS(UFSHCI_REG_AHIT_TS_1MS) | 150);
259 }
260
261 /* Attach to SCSI layer */
262 saa.saa_adapter = &ufshci_switch;
263 saa.saa_adapter_softc = sc;
264 saa.saa_adapter_buswidth = UFSHCI_TARGETS_MAX + 1;
265 saa.saa_luns = 1;
266 saa.saa_adapter_target = 0;
267 saa.saa_openings = sc->sc_nutrs;
268 saa.saa_pool = &sc->sc_iopool;
269 saa.saa_quirks = saa.saa_flags = 0;
270 saa.saa_wwpn = saa.saa_wwnn = 0;
271 #if NKSTAT > 0
272 ufshci_kstat_attach(sc);
273 #endif
274 config_found(&sc->sc_dev, &saa, scsiprint);
275
276 return 0;
277 }
278
279 int
ufshci_reset(struct ufshci_softc * sc)280 ufshci_reset(struct ufshci_softc *sc)
281 {
282 int i;
283 int retry = 10;
284 uint32_t hce;
285
286 /*
287 * 7.1.1 Host Controller Initialization: 2)
288 * Reset and enable host controller
289 */
290 UFSHCI_WRITE_4(sc, UFSHCI_REG_HCE, UFSHCI_REG_HCE_HCE);
291
292 /* 7.1.1 Host Controller Initialization: 3) */
293 for (i = 0; i < retry; i++) {
294 hce = UFSHCI_READ_4(sc, UFSHCI_REG_HCE);
295 if (hce == 1)
296 break;
297 delay(1);
298 }
299 if (i == retry) {
300 printf("%s: Enabling Host Controller failed!\n",
301 sc->sc_dev.dv_xname);
302 return 1;
303 }
304
305 DPRINTF(2, "\n%s: Host Controller enabled (i=%d)\n", __func__, i);
306
307 return 0;
308 }
309
310 int
ufshci_is_poll(struct ufshci_softc * sc,uint32_t type)311 ufshci_is_poll(struct ufshci_softc *sc, uint32_t type)
312 {
313 uint32_t status;
314 int i, retry = 25;
315
316 for (i = 0; i < retry; i++) {
317 status = UFSHCI_READ_4(sc, UFSHCI_REG_IS);
318 if (status & type)
319 break;
320 delay(10);
321 }
322 if (i == retry) {
323 printf("%s: %s: timeout\n", sc->sc_dev.dv_xname, __func__);
324 return 1;
325 }
326 DPRINTF(3, "%s: completed after %d retries\n", __func__, i);
327
328 /* ACK interrupt */
329 UFSHCI_WRITE_4(sc, UFSHCI_REG_IS, status);
330
331 return 0;
332 }
333
334 struct ufshci_dmamem *
ufshci_dmamem_alloc(struct ufshci_softc * sc,size_t size)335 ufshci_dmamem_alloc(struct ufshci_softc *sc, size_t size)
336 {
337 struct ufshci_dmamem *udm;
338 int nsegs;
339
340 udm = malloc(sizeof(*udm), M_DEVBUF, M_WAITOK | M_ZERO);
341 if (udm == NULL)
342 return NULL;
343
344 udm->udm_size = size;
345
346 if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0,
347 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW |
348 ((sc->sc_cap & UFSHCI_REG_CAP_64AS) ? BUS_DMA_64BIT : 0),
349 &udm->udm_map) != 0)
350 goto udmfree;
351
352 if (bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, &udm->udm_seg,
353 1, &nsegs, BUS_DMA_WAITOK | BUS_DMA_ZERO) != 0)
354 goto destroy;
355
356 if (bus_dmamem_map(sc->sc_dmat, &udm->udm_seg, nsegs, size,
357 &udm->udm_kva, BUS_DMA_WAITOK) != 0)
358 goto free;
359
360 if (bus_dmamap_load(sc->sc_dmat, udm->udm_map, udm->udm_kva, size,
361 NULL, BUS_DMA_WAITOK) != 0)
362 goto unmap;
363
364 DPRINTF(2, "%s: size=%lu, page_size=%d, nsegs=%d\n",
365 __func__, size, PAGE_SIZE, nsegs);
366
367 return udm;
368
369 unmap:
370 bus_dmamem_unmap(sc->sc_dmat, udm->udm_kva, size);
371 free:
372 bus_dmamem_free(sc->sc_dmat, &udm->udm_seg, 1);
373 destroy:
374 bus_dmamap_destroy(sc->sc_dmat, udm->udm_map);
375 udmfree:
376 free(udm, M_DEVBUF, sizeof(*udm));
377
378 return NULL;
379 }
380
381 void
ufshci_dmamem_free(struct ufshci_softc * sc,struct ufshci_dmamem * udm)382 ufshci_dmamem_free(struct ufshci_softc *sc, struct ufshci_dmamem *udm)
383 {
384 bus_dmamap_unload(sc->sc_dmat, udm->udm_map);
385 bus_dmamem_unmap(sc->sc_dmat, udm->udm_kva, udm->udm_size);
386 bus_dmamem_free(sc->sc_dmat, &udm->udm_seg, 1);
387 bus_dmamap_destroy(sc->sc_dmat, udm->udm_map);
388 free(udm, M_DEVBUF, sizeof(*udm));
389 }
390
391 int
ufshci_alloc(struct ufshci_softc * sc)392 ufshci_alloc(struct ufshci_softc *sc)
393 {
394 /* 7.1.1 Host Controller Initialization: 13) */
395 sc->sc_dmamem_utmrd = ufshci_dmamem_alloc(sc,
396 sizeof(struct ufshci_utmrd) * sc->sc_nutmrs);
397 if (sc->sc_dmamem_utmrd == NULL) {
398 printf("%s: Can't allocate DMA memory for UTMRD\n",
399 sc->sc_dev.dv_xname);
400 return 1;
401 }
402
403 /* 7.1.1 Host Controller Initialization: 15) */
404 sc->sc_dmamem_utrd = ufshci_dmamem_alloc(sc,
405 sizeof(struct ufshci_utrd) * sc->sc_nutrs);
406 if (sc->sc_dmamem_utrd == NULL) {
407 printf("%s: Can't allocate DMA memory for UTRD\n",
408 sc->sc_dev.dv_xname);
409 return 1;
410 }
411
412 /* Allocate UCDs. */
413 sc->sc_dmamem_ucd = ufshci_dmamem_alloc(sc,
414 sizeof(struct ufshci_ucd) * sc->sc_nutrs);
415 if (sc->sc_dmamem_ucd == NULL) {
416 printf("%s: Can't allocate DMA memory for UCD\n",
417 sc->sc_dev.dv_xname);
418 return 1;
419 }
420
421 return 0;
422 }
423
424 int
ufshci_init(struct ufshci_softc * sc)425 ufshci_init(struct ufshci_softc *sc)
426 {
427 uint32_t reg;
428 uint64_t dva;
429
430 /*
431 * 7.1.1 Host Controller Initialization: 4)
432 * TODO: Do we need to set DME_SET?
433 */
434
435 /* 7.1.1 Host Controller Initialization: 5) */
436 if (sc->sc_cap & UFSHCI_REG_AUTOH8) {
437 UFSHCI_WRITE_4(sc, UFSHCI_REG_IE,
438 UFSHCI_REG_IE_UTRCE | UFSHCI_REG_IE_UTMRCE |
439 UFSHCI_REG_IS_UHES | UFSHCI_REG_IS_UHXS);
440 } else {
441 UFSHCI_WRITE_4(sc, UFSHCI_REG_IE,
442 UFSHCI_REG_IE_UTRCE | UFSHCI_REG_IE_UTMRCE);
443 }
444
445 /* 7.1.1 Host Controller Initialization: 6) */
446 UFSHCI_WRITE_4(sc, UFSHCI_REG_UICCMD,
447 UFSHCI_REG_UICCMD_CMDOP_DME_LINKSTARTUP);
448 if (ufshci_is_poll(sc, UFSHCI_REG_IS_UCCS))
449 return 1;
450
451 /*
452 * 7.1.1 Host Controller Initialization: 7), 8), 9)
453 * TODO: Implement retry in case UFSHCI_REG_HCS returns 0
454 */
455 reg = UFSHCI_READ_4(sc, UFSHCI_REG_HCS);
456 if (reg & UFSHCI_REG_HCS_DP)
457 DPRINTF(2, "%s: Device Presence SET\n", __func__);
458 else
459 DPRINTF(2, "%s: Device Presence NOT SET\n", __func__);
460
461 /*
462 * 7.1.1 Host Controller Initialization: 10)
463 * TODO: Enable additional interrupt on the IE register
464 */
465
466 /* 7.1.1 Host Controller Initialization: 11) */
467 if (sc->sc_flags & UFSHCI_FLAGS_AGGR_INTR) {
468 UFSHCI_WRITE_4(sc, UFSHCI_REG_UTRIACR,
469 UFSHCI_REG_UTRIACR_IAEN |
470 UFSHCI_REG_UTRIACR_IAPWEN |
471 UFSHCI_REG_UTRIACR_CTR |
472 UFSHCI_REG_UTRIACR_IACTH(sc->sc_iacth) |
473 UFSHCI_REG_UTRIACR_IATOVAL(UFSHCI_INTR_AGGR_TIMEOUT));
474 } else {
475 UFSHCI_WRITE_4(sc, UFSHCI_REG_UTRIACR, 0);
476 }
477
478 /*
479 * 7.1.1 Host Controller Initialization: 12)
480 * TODO: More UIC commands to issue?
481 */
482
483 /* 7.1.1 Host Controller Initialization: 14) */
484 dva = UFSHCI_DMA_DVA(sc->sc_dmamem_utmrd);
485 DPRINTF(2, "%s: utmrd dva=%llu\n", __func__, dva);
486 UFSHCI_WRITE_4(sc, UFSHCI_REG_UTMRLBA, (uint32_t)dva);
487 UFSHCI_WRITE_4(sc, UFSHCI_REG_UTMRLBAU, (uint32_t)(dva >> 32));
488
489 /* 7.1.1 Host Controller Initialization: 16) */
490 dva = UFSHCI_DMA_DVA(sc->sc_dmamem_utrd);
491 DPRINTF(2, "%s: utrd dva=%llu\n", __func__, dva);
492 UFSHCI_WRITE_4(sc, UFSHCI_REG_UTRLBA, (uint32_t)dva);
493 UFSHCI_WRITE_4(sc, UFSHCI_REG_UTRLBAU, (uint32_t)(dva >> 32));
494
495 /* 7.1.1 Host Controller Initialization: 17) */
496 UFSHCI_WRITE_4(sc, UFSHCI_REG_UTMRLRSR, UFSHCI_REG_UTMRLRSR_START);
497
498 /* 7.1.1 Host Controller Initialization: 18) */
499 UFSHCI_WRITE_4(sc, UFSHCI_REG_UTRLRSR, UFSHCI_REG_UTRLRSR_START);
500
501 /* 7.1.1 Host Controller Initialization: 19) */
502 /* TODO: bMaxNumOfRTT will be set as the minimum value of
503 * bDeviceRTTCap and NORTT. ???
504 */
505
506 return 0;
507 }
508
509 void
ufshci_disable(struct ufshci_softc * sc)510 ufshci_disable(struct ufshci_softc *sc)
511 {
512 /* Stop run queues. */
513 UFSHCI_WRITE_4(sc, UFSHCI_REG_UTMRLRSR, UFSHCI_REG_UTMRLRSR_STOP);
514 UFSHCI_WRITE_4(sc, UFSHCI_REG_UTRLRSR, UFSHCI_REG_UTRLRSR_STOP);
515
516 /* Disable interrupts. */
517 UFSHCI_WRITE_4(sc, UFSHCI_REG_IE, 0);
518 }
519
520 int
ufshci_doorbell_read(struct ufshci_softc * sc)521 ufshci_doorbell_read(struct ufshci_softc *sc)
522 {
523 uint32_t reg;
524
525 reg = UFSHCI_READ_4(sc, UFSHCI_REG_UTRLDBR);
526
527 return reg;
528 }
529
530 void
ufshci_doorbell_write(struct ufshci_softc * sc,int slot)531 ufshci_doorbell_write(struct ufshci_softc *sc, int slot)
532 {
533 uint32_t reg;
534
535 reg = (1U << slot);
536
537 UFSHCI_WRITE_4(sc, UFSHCI_REG_UTRLDBR, reg);
538 }
539
540 int
ufshci_doorbell_poll(struct ufshci_softc * sc,int slot,uint32_t timeout_ms)541 ufshci_doorbell_poll(struct ufshci_softc *sc, int slot, uint32_t timeout_ms)
542 {
543 uint32_t reg;
544 uint64_t timeout_us;
545
546 for (timeout_us = timeout_ms * 1000; timeout_us != 0;
547 timeout_us -= 1000) {
548 reg = UFSHCI_READ_4(sc, UFSHCI_REG_UTRLDBR);
549 if ((reg & (1U << slot)) == 0)
550 break;
551 delay(1000);
552 }
553 if (timeout_us == 0) {
554 printf("%s: %s: timeout\n", sc->sc_dev.dv_xname, __func__);
555 return 1;
556 }
557
558 return 0;
559 }
560
561 int
ufshci_utr_cmd_nop(struct ufshci_softc * sc,struct ufshci_ccb * ccb,struct scsi_xfer * xs)562 ufshci_utr_cmd_nop(struct ufshci_softc *sc, struct ufshci_ccb *ccb,
563 struct scsi_xfer *xs)
564 {
565 int slot, off, len;
566 uint64_t dva;
567 struct ufshci_utrd *utrd;
568 struct ufshci_ucd *ucd;
569
570 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 1) */
571 slot = ccb->ccb_slot;
572 utrd = UFSHCI_DMA_KVA(sc->sc_dmamem_utrd);
573 utrd += slot;
574 memset(utrd, 0, sizeof(*utrd));
575
576 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2a) */
577 utrd->dw0 = UFSHCI_UTRD_DW0_CT_UFS;
578
579 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2b) */
580 utrd->dw0 |= UFSHCI_UTRD_DW0_DD_NO;
581
582 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2c) */
583 if (sc->sc_flags & UFSHCI_FLAGS_AGGR_INTR)
584 utrd->dw0 |= UFSHCI_UTRD_DW0_I_REG;
585 else
586 utrd->dw0 |= UFSHCI_UTRD_DW0_I_INT;
587
588 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2d) */
589 utrd->dw2 = UFSHCI_UTRD_DW2_OCS_IOV;
590
591 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2e) */
592 ucd = UFSHCI_DMA_KVA(sc->sc_dmamem_ucd);
593 ucd += slot;
594 memset(ucd, 0, sizeof(*ucd));
595
596 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2f) */
597 ucd->cmd.hdr.tc = UPIU_TC_I2T_NOP_OUT;
598 ucd->cmd.hdr.flags = 0;
599 ucd->cmd.hdr.lun = 0;
600 ucd->cmd.hdr.task_tag = slot;
601 ucd->cmd.hdr.cmd_set_type = 0; /* SCSI command */
602 ucd->cmd.hdr.query = 0;
603 ucd->cmd.hdr.response = 0;
604 ucd->cmd.hdr.status = 0;
605 ucd->cmd.hdr.ehs_len = 0;
606 ucd->cmd.hdr.device_info = 0;
607 ucd->cmd.hdr.ds_len = 0;
608
609 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2g) */
610 /* Already done with above memset */
611
612 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 3) */
613 dva = UFSHCI_DMA_DVA(sc->sc_dmamem_ucd);
614 utrd->dw4 = (uint32_t)dva;
615 utrd->dw5 = (uint32_t)(dva >> 32);
616
617 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 4) */
618 off = sizeof(struct upiu_command) / 4; /* DWORD offset */
619 utrd->dw6 = UFSHCI_UTRD_DW6_RUO(off);
620
621 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 5) */
622 len = sizeof(struct upiu_response) / 4; /* DWORD length */
623 utrd->dw6 |= UFSHCI_UTRD_DW6_RUL(len);
624
625 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 6) */
626 off = (sizeof(struct upiu_command) + sizeof(struct upiu_response)) / 4;
627 utrd->dw7 = UFSHCI_UTRD_DW7_PRDTO(off);
628
629 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 7) */
630 utrd->dw7 |= UFSHCI_UTRD_DW7_PRDTL(0); /* No data xfer */
631
632 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 9) */
633 if (UFSHCI_READ_4(sc, UFSHCI_REG_UTRLRSR) != 1) {
634 printf("%s: %s: UTRLRSR not set\n",
635 sc->sc_dev.dv_xname, __func__);
636 return 1;
637 }
638
639 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_utrd),
640 sizeof(*utrd) * slot, sizeof(*utrd), BUS_DMASYNC_PREWRITE);
641 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_ucd),
642 sizeof(*ucd) * slot, sizeof(*ucd), BUS_DMASYNC_PREWRITE);
643
644 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 14) */
645 ccb->ccb_status = CCB_STATUS_INPROGRESS;
646 ufshci_doorbell_write(sc, slot);
647
648 return 0;
649 }
650
651 int
ufshci_utr_cmd_lun(struct ufshci_softc * sc,struct ufshci_ccb * ccb,struct scsi_xfer * xs)652 ufshci_utr_cmd_lun(struct ufshci_softc *sc, struct ufshci_ccb *ccb,
653 struct scsi_xfer *xs)
654 {
655 int slot, off, len, i;
656 uint64_t dva;
657 struct ufshci_utrd *utrd;
658 struct ufshci_ucd *ucd;
659 bus_dmamap_t dmap = ccb->ccb_dmamap;
660
661 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 1) */
662 slot = ccb->ccb_slot;
663 utrd = UFSHCI_DMA_KVA(sc->sc_dmamem_utrd);
664 utrd += slot;
665 memset(utrd, 0, sizeof(*utrd));
666
667 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2a) */
668 utrd->dw0 = UFSHCI_UTRD_DW0_CT_UFS;
669
670 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2b) */
671 utrd->dw0 |= UFSHCI_UTRD_DW0_DD_T2I;
672
673 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2c) */
674 if (sc->sc_flags & UFSHCI_FLAGS_AGGR_INTR)
675 utrd->dw0 |= UFSHCI_UTRD_DW0_I_REG;
676 else
677 utrd->dw0 |= UFSHCI_UTRD_DW0_I_INT;
678
679 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2d) */
680 utrd->dw2 = UFSHCI_UTRD_DW2_OCS_IOV;
681
682 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2e) */
683 ucd = UFSHCI_DMA_KVA(sc->sc_dmamem_ucd);
684 ucd += slot;
685 memset(ucd, 0, sizeof(*ucd));
686
687 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2f) */
688 ucd->cmd.hdr.tc = UPIU_TC_I2T_COMMAND;
689 ucd->cmd.hdr.flags = (1 << 6); /* Bit-5 = Write, Bit-6 = Read */
690 ucd->cmd.hdr.lun = 0;
691 ucd->cmd.hdr.task_tag = slot;
692 ucd->cmd.hdr.cmd_set_type = 0; /* SCSI command */
693 ucd->cmd.hdr.query = 0;
694 ucd->cmd.hdr.response = 0;
695 ucd->cmd.hdr.status = 0;
696 ucd->cmd.hdr.ehs_len = 0;
697 ucd->cmd.hdr.device_info = 0;
698 ucd->cmd.hdr.ds_len = 0;
699
700 ucd->cmd.expected_xfer_len = htobe32(xs->datalen);
701
702 ucd->cmd.cdb[0] = REPORT_LUNS;
703 ucd->cmd.cdb[6] = 0;
704 ucd->cmd.cdb[7] = 0;
705 ucd->cmd.cdb[8] = 0;
706 ucd->cmd.cdb[9] = xs->datalen;
707
708 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2g) */
709 /* Already done with above memset */
710
711 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 3) */
712 dva = UFSHCI_DMA_DVA(sc->sc_dmamem_ucd);
713 utrd->dw4 = (uint32_t)dva;
714 utrd->dw5 = (uint32_t)(dva >> 32);
715
716 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 4) */
717 off = sizeof(struct upiu_command) / 4; /* DWORD offset */
718 utrd->dw6 = UFSHCI_UTRD_DW6_RUO(off);
719
720 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 5) */
721 len = sizeof(struct upiu_response) / 4; /* DWORD length */
722 utrd->dw6 |= UFSHCI_UTRD_DW6_RUL(len);
723
724 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 6) */
725 off = (sizeof(struct upiu_command) + sizeof(struct upiu_response)) / 4;
726 utrd->dw7 = UFSHCI_UTRD_DW7_PRDTO(off);
727
728 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 7) */
729 utrd->dw7 |= UFSHCI_UTRD_DW7_PRDTL(dmap->dm_nsegs);
730
731 /* Build PRDT data segment. */
732 for (i = 0; i < dmap->dm_nsegs; i++) {
733 dva = dmap->dm_segs[i].ds_addr;
734 ucd->prdt[i].dw0 = (uint32_t)dva;
735 ucd->prdt[i].dw1 = (uint32_t)(dva >> 32);
736 ucd->prdt[i].dw2 = 0;
737 ucd->prdt[i].dw3 = dmap->dm_segs[i].ds_len - 1;
738 }
739
740 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 9) */
741 if (UFSHCI_READ_4(sc, UFSHCI_REG_UTRLRSR) != 1) {
742 printf("%s: %s: UTRLRSR not set\n",
743 sc->sc_dev.dv_xname, __func__);
744 return 1;
745 }
746
747 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_utrd),
748 sizeof(*utrd) * slot, sizeof(*utrd), BUS_DMASYNC_PREWRITE);
749 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_ucd),
750 sizeof(*ucd) * slot, sizeof(*ucd), BUS_DMASYNC_PREWRITE);
751
752 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 14) */
753 ccb->ccb_status = CCB_STATUS_INPROGRESS;
754 ufshci_doorbell_write(sc, slot);
755
756 return 0;
757 }
758
759 int
ufshci_utr_cmd_inquiry(struct ufshci_softc * sc,struct ufshci_ccb * ccb,struct scsi_xfer * xs)760 ufshci_utr_cmd_inquiry(struct ufshci_softc *sc, struct ufshci_ccb *ccb,
761 struct scsi_xfer *xs)
762 {
763 int slot, off, len, i;
764 uint64_t dva;
765 struct ufshci_utrd *utrd;
766 struct ufshci_ucd *ucd;
767 bus_dmamap_t dmap = ccb->ccb_dmamap;
768
769 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 1) */
770 slot = ccb->ccb_slot;
771 utrd = UFSHCI_DMA_KVA(sc->sc_dmamem_utrd);
772 utrd += slot;
773 memset(utrd, 0, sizeof(*utrd));
774
775 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2a) */
776 utrd->dw0 = UFSHCI_UTRD_DW0_CT_UFS;
777
778 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2b) */
779 utrd->dw0 |= UFSHCI_UTRD_DW0_DD_T2I;
780
781 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2c) */
782 if (sc->sc_flags & UFSHCI_FLAGS_AGGR_INTR)
783 utrd->dw0 |= UFSHCI_UTRD_DW0_I_REG;
784 else
785 utrd->dw0 |= UFSHCI_UTRD_DW0_I_INT;
786
787 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2d) */
788 utrd->dw2 = UFSHCI_UTRD_DW2_OCS_IOV;
789
790 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2e) */
791 ucd = UFSHCI_DMA_KVA(sc->sc_dmamem_ucd);
792 ucd += slot;
793 memset(ucd, 0, sizeof(*ucd));
794
795 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2f) */
796 ucd->cmd.hdr.tc = UPIU_TC_I2T_COMMAND;
797 ucd->cmd.hdr.flags = (1 << 6); /* Bit-5 = Write, Bit-6 = Read */
798 ucd->cmd.hdr.lun = 0;
799 ucd->cmd.hdr.task_tag = slot;
800 ucd->cmd.hdr.cmd_set_type = 0; /* SCSI command */
801 ucd->cmd.hdr.query = 0;
802 ucd->cmd.hdr.response = 0;
803 ucd->cmd.hdr.status = 0;
804 ucd->cmd.hdr.ehs_len = 0;
805 ucd->cmd.hdr.device_info = 0;
806 ucd->cmd.hdr.ds_len = 0;
807
808 ucd->cmd.expected_xfer_len = htobe32(xs->datalen);
809
810 ucd->cmd.cdb[0] = INQUIRY; /* 0x12 */
811 ucd->cmd.cdb[3] = 0;
812 ucd->cmd.cdb[4] = xs->datalen;
813
814 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2g) */
815 /* Already done with above memset */
816
817 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 3) */
818 dva = UFSHCI_DMA_DVA(sc->sc_dmamem_ucd) + (sizeof(*ucd) * slot);
819 utrd->dw4 = (uint32_t)dva;
820 utrd->dw5 = (uint32_t)(dva >> 32);
821
822 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 4) */
823 off = sizeof(struct upiu_command) / 4; /* DWORD offset */
824 utrd->dw6 = UFSHCI_UTRD_DW6_RUO(off);
825
826 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 5) */
827 len = sizeof(struct upiu_response) / 4; /* DWORD length */
828 utrd->dw6 |= UFSHCI_UTRD_DW6_RUL(len);
829
830 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 6) */
831 off = (sizeof(struct upiu_command) + sizeof(struct upiu_response)) / 4;
832 utrd->dw7 = UFSHCI_UTRD_DW7_PRDTO(off);
833
834 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 7) */
835 utrd->dw7 |= UFSHCI_UTRD_DW7_PRDTL(dmap->dm_nsegs);
836
837 /* Build PRDT data segment. */
838 for (i = 0; i < dmap->dm_nsegs; i++) {
839 dva = dmap->dm_segs[i].ds_addr;
840 ucd->prdt[i].dw0 = (uint32_t)dva;
841 ucd->prdt[i].dw1 = (uint32_t)(dva >> 32);
842 ucd->prdt[i].dw2 = 0;
843 ucd->prdt[i].dw3 = dmap->dm_segs[i].ds_len - 1;
844 }
845
846 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 9) */
847 if (UFSHCI_READ_4(sc, UFSHCI_REG_UTRLRSR) != 1) {
848 printf("%s: %s: UTRLRSR not set\n",
849 sc->sc_dev.dv_xname, __func__);
850 return 1;
851 }
852
853 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_utrd),
854 sizeof(*utrd) * slot, sizeof(*utrd), BUS_DMASYNC_PREWRITE);
855 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_ucd),
856 sizeof(*ucd) * slot, sizeof(*ucd), BUS_DMASYNC_PREWRITE);
857
858 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 14) */
859 ccb->ccb_status = CCB_STATUS_INPROGRESS;
860 ufshci_doorbell_write(sc, slot);
861
862 return 0;
863 }
864
865 int
ufshci_utr_cmd_capacity16(struct ufshci_softc * sc,struct ufshci_ccb * ccb,struct scsi_xfer * xs)866 ufshci_utr_cmd_capacity16(struct ufshci_softc *sc, struct ufshci_ccb *ccb,
867 struct scsi_xfer *xs)
868 {
869 int slot, off, len, i;
870 uint64_t dva;
871 struct ufshci_utrd *utrd;
872 struct ufshci_ucd *ucd;
873 bus_dmamap_t dmap = ccb->ccb_dmamap;
874
875 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 1) */
876 slot = ccb->ccb_slot;
877 utrd = UFSHCI_DMA_KVA(sc->sc_dmamem_utrd);
878 utrd += slot;
879 memset(utrd, 0, sizeof(*utrd));
880
881 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2a) */
882 utrd->dw0 = UFSHCI_UTRD_DW0_CT_UFS;
883
884 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2b) */
885 utrd->dw0 |= UFSHCI_UTRD_DW0_DD_T2I;
886
887 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2c) */
888 if (sc->sc_flags & UFSHCI_FLAGS_AGGR_INTR)
889 utrd->dw0 |= UFSHCI_UTRD_DW0_I_REG;
890 else
891 utrd->dw0 |= UFSHCI_UTRD_DW0_I_INT;
892
893 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2d) */
894 utrd->dw2 = UFSHCI_UTRD_DW2_OCS_IOV;
895
896 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2e) */
897 ucd = UFSHCI_DMA_KVA(sc->sc_dmamem_ucd);
898 ucd += slot;
899 memset(ucd, 0, sizeof(*ucd));
900
901 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2f) */
902 ucd->cmd.hdr.tc = UPIU_TC_I2T_COMMAND;
903 ucd->cmd.hdr.flags = (1 << 6); /* Bit-5 = Write, Bit-6 = Read */
904 ucd->cmd.hdr.lun = 0;
905 ucd->cmd.hdr.task_tag = slot;
906 ucd->cmd.hdr.cmd_set_type = 0; /* SCSI command */
907 ucd->cmd.hdr.query = 0;
908 ucd->cmd.hdr.response = 0;
909 ucd->cmd.hdr.status = 0;
910 ucd->cmd.hdr.ehs_len = 0;
911 ucd->cmd.hdr.device_info = 0;
912 ucd->cmd.hdr.ds_len = 0;
913
914 ucd->cmd.expected_xfer_len = htobe32(xs->datalen);
915
916 ucd->cmd.cdb[0] = READ_CAPACITY_16; /* 0x9e */
917 ucd->cmd.cdb[1] = 0x10; /* Service Action */
918 /* Logical Block Address = 0 for UFS */
919 ucd->cmd.cdb[10] = 0;
920 ucd->cmd.cdb[11] = 0;
921 ucd->cmd.cdb[12] = 0;
922 ucd->cmd.cdb[13] = xs->datalen;
923
924 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2g) */
925 /* Already done with above memset */
926
927 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 3) */
928 dva = UFSHCI_DMA_DVA(sc->sc_dmamem_ucd) + (sizeof(*ucd) * slot);
929 utrd->dw4 = (uint32_t)dva;
930 utrd->dw5 = (uint32_t)(dva >> 32);
931
932 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 4) */
933 off = sizeof(struct upiu_command) / 4; /* DWORD offset */
934 utrd->dw6 = UFSHCI_UTRD_DW6_RUO(off);
935
936 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 5) */
937 len = sizeof(struct upiu_response) / 4; /* DWORD length */
938 utrd->dw6 |= UFSHCI_UTRD_DW6_RUL(len);
939
940 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 6) */
941 off = (sizeof(struct upiu_command) + sizeof(struct upiu_response)) / 4;
942 utrd->dw7 = UFSHCI_UTRD_DW7_PRDTO(off);
943
944 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 7) */
945 utrd->dw7 |= UFSHCI_UTRD_DW7_PRDTL(dmap->dm_nsegs);
946
947 /* Build PRDT data segment. */
948 for (i = 0; i < dmap->dm_nsegs; i++) {
949 dva = dmap->dm_segs[i].ds_addr;
950 ucd->prdt[i].dw0 = (uint32_t)dva;
951 ucd->prdt[i].dw1 = (uint32_t)(dva >> 32);
952 ucd->prdt[i].dw2 = 0;
953 ucd->prdt[i].dw3 = dmap->dm_segs[i].ds_len - 1;
954 }
955
956 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 9) */
957 if (UFSHCI_READ_4(sc, UFSHCI_REG_UTRLRSR) != 1) {
958 printf("%s: %s: UTRLRSR not set\n",
959 sc->sc_dev.dv_xname, __func__);
960 return 1;
961 }
962
963 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_utrd),
964 sizeof(*utrd) * slot, sizeof(*utrd), BUS_DMASYNC_PREWRITE);
965 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_ucd),
966 sizeof(*ucd) * slot, sizeof(*ucd), BUS_DMASYNC_PREWRITE);
967
968 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 14) */
969 ccb->ccb_status = CCB_STATUS_INPROGRESS;
970 ufshci_doorbell_write(sc, slot);
971
972 return 0;
973 }
974
975 int
ufshci_utr_cmd_capacity(struct ufshci_softc * sc,struct ufshci_ccb * ccb,struct scsi_xfer * xs)976 ufshci_utr_cmd_capacity(struct ufshci_softc *sc, struct ufshci_ccb *ccb,
977 struct scsi_xfer *xs)
978 {
979 int slot, off, len, i;
980 uint64_t dva;
981 struct ufshci_utrd *utrd;
982 struct ufshci_ucd *ucd;
983 bus_dmamap_t dmap = ccb->ccb_dmamap;
984
985 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 1) */
986 slot = ccb->ccb_slot;
987 utrd = UFSHCI_DMA_KVA(sc->sc_dmamem_utrd);
988 utrd += slot;
989 memset(utrd, 0, sizeof(*utrd));
990
991 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2a) */
992 utrd->dw0 = UFSHCI_UTRD_DW0_CT_UFS;
993
994 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2b) */
995 utrd->dw0 |= UFSHCI_UTRD_DW0_DD_T2I;
996
997 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2c) */
998 if (sc->sc_flags & UFSHCI_FLAGS_AGGR_INTR)
999 utrd->dw0 |= UFSHCI_UTRD_DW0_I_REG;
1000 else
1001 utrd->dw0 |= UFSHCI_UTRD_DW0_I_INT;
1002
1003 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2d) */
1004 utrd->dw2 = UFSHCI_UTRD_DW2_OCS_IOV;
1005
1006 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2e) */
1007 ucd = UFSHCI_DMA_KVA(sc->sc_dmamem_ucd);
1008 ucd += slot;
1009 memset(ucd, 0, sizeof(*ucd));
1010
1011 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2f) */
1012 ucd->cmd.hdr.tc = UPIU_TC_I2T_COMMAND;
1013 ucd->cmd.hdr.flags = (1 << 6); /* Bit-5 = Write, Bit-6 = Read */
1014 ucd->cmd.hdr.lun = 0;
1015 ucd->cmd.hdr.task_tag = slot;
1016 ucd->cmd.hdr.cmd_set_type = 0; /* SCSI command */
1017 ucd->cmd.hdr.query = 0;
1018 ucd->cmd.hdr.response = 0;
1019 ucd->cmd.hdr.status = 0;
1020 ucd->cmd.hdr.ehs_len = 0;
1021 ucd->cmd.hdr.device_info = 0;
1022 ucd->cmd.hdr.ds_len = 0;
1023
1024 ucd->cmd.expected_xfer_len = htobe32(xs->datalen);
1025
1026 ucd->cmd.cdb[0] = READ_CAPACITY; /* 0x25 */
1027 /* Logical Block Address = 0 for UFS */
1028 ucd->cmd.cdb[2] = 0;
1029 ucd->cmd.cdb[3] = 0;
1030 ucd->cmd.cdb[4] = 0;
1031 ucd->cmd.cdb[5] = 0;
1032
1033 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2g) */
1034 /* Already done with above memset */
1035
1036 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 3) */
1037 dva = UFSHCI_DMA_DVA(sc->sc_dmamem_ucd) + (sizeof(*ucd) * slot);
1038 utrd->dw4 = (uint32_t)dva;
1039 utrd->dw5 = (uint32_t)(dva >> 32);
1040
1041 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 4) */
1042 off = sizeof(struct upiu_command) / 4; /* DWORD offset */
1043 utrd->dw6 = UFSHCI_UTRD_DW6_RUO(off);
1044
1045 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 5) */
1046 len = sizeof(struct upiu_response) / 4; /* DWORD length */
1047 utrd->dw6 |= UFSHCI_UTRD_DW6_RUL(len);
1048
1049 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 6) */
1050 off = (sizeof(struct upiu_command) + sizeof(struct upiu_response)) / 4;
1051 utrd->dw7 = UFSHCI_UTRD_DW7_PRDTO(off);
1052
1053 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 7) */
1054 utrd->dw7 |= UFSHCI_UTRD_DW7_PRDTL(dmap->dm_nsegs);
1055
1056 /* Build PRDT data segment. */
1057 for (i = 0; i < dmap->dm_nsegs; i++) {
1058 dva = dmap->dm_segs[i].ds_addr;
1059 ucd->prdt[i].dw0 = (uint32_t)dva;
1060 ucd->prdt[i].dw1 = (uint32_t)(dva >> 32);
1061 ucd->prdt[i].dw2 = 0;
1062 ucd->prdt[i].dw3 = dmap->dm_segs[i].ds_len - 1;
1063 }
1064
1065 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 9) */
1066 if (UFSHCI_READ_4(sc, UFSHCI_REG_UTRLRSR) != 1) {
1067 printf("%s: %s: UTRLRSR not set\n",
1068 sc->sc_dev.dv_xname, __func__);
1069 return 1;
1070 }
1071
1072 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_utrd),
1073 sizeof(*utrd) * slot, sizeof(*utrd), BUS_DMASYNC_PREWRITE);
1074 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_ucd),
1075 sizeof(*ucd) * slot, sizeof(*ucd), BUS_DMASYNC_PREWRITE);
1076
1077 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 14) */
1078 ccb->ccb_status = CCB_STATUS_INPROGRESS;
1079 ufshci_doorbell_write(sc, slot);
1080
1081 return 0;
1082 }
1083
1084 int
ufshci_utr_cmd_io(struct ufshci_softc * sc,struct ufshci_ccb * ccb,struct scsi_xfer * xs,int dir)1085 ufshci_utr_cmd_io(struct ufshci_softc *sc, struct ufshci_ccb *ccb,
1086 struct scsi_xfer *xs, int dir)
1087 {
1088 int slot, off, len, i;
1089 uint64_t dva;
1090 struct ufshci_utrd *utrd;
1091 struct ufshci_ucd *ucd;
1092 bus_dmamap_t dmap = ccb->ccb_dmamap;
1093 uint32_t blocks;
1094 uint64_t lba;
1095
1096 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 1) */
1097 slot = ccb->ccb_slot;
1098 utrd = UFSHCI_DMA_KVA(sc->sc_dmamem_utrd);
1099 utrd += slot;
1100 memset(utrd, 0, sizeof(*utrd));
1101
1102 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2a) */
1103 utrd->dw0 = UFSHCI_UTRD_DW0_CT_UFS;
1104
1105 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2b) */
1106 if (dir == SCSI_DATA_IN)
1107 utrd->dw0 |= UFSHCI_UTRD_DW0_DD_T2I;
1108 else
1109 utrd->dw0 |= UFSHCI_UTRD_DW0_DD_I2T;
1110
1111 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2c) */
1112 if (sc->sc_flags & UFSHCI_FLAGS_AGGR_INTR)
1113 utrd->dw0 |= UFSHCI_UTRD_DW0_I_REG;
1114 else
1115 utrd->dw0 |= UFSHCI_UTRD_DW0_I_INT;
1116
1117 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2d) */
1118 utrd->dw2 = UFSHCI_UTRD_DW2_OCS_IOV;
1119
1120 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2e) */
1121 ucd = UFSHCI_DMA_KVA(sc->sc_dmamem_ucd);
1122 ucd += slot;
1123 memset(ucd, 0, sizeof(*ucd));
1124
1125 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2f) */
1126 ucd->cmd.hdr.tc = UPIU_TC_I2T_COMMAND;
1127 if (dir == SCSI_DATA_IN)
1128 ucd->cmd.hdr.flags = (1 << 6); /* Bit-6 = Read */
1129 else
1130 ucd->cmd.hdr.flags = (1 << 5); /* Bit-5 = Write */
1131 ucd->cmd.hdr.lun = 0;
1132 ucd->cmd.hdr.task_tag = slot;
1133 ucd->cmd.hdr.cmd_set_type = 0; /* SCSI command */
1134 ucd->cmd.hdr.query = 0;
1135 ucd->cmd.hdr.response = 0;
1136 ucd->cmd.hdr.status = 0;
1137 ucd->cmd.hdr.ehs_len = 0;
1138 ucd->cmd.hdr.device_info = 0;
1139 ucd->cmd.hdr.ds_len = 0;
1140
1141 /*
1142 * JESD220C-2_1.pdf, page 88, d) Expected Data Transfer Length:
1143 * "When the COMMAND UPIU encodes a SCSI WRITE or SCSI READ command
1144 * (specifically WRITE (6), READ (6), WRITE (10), READ (10),
1145 * WRITE (16), or READ (16)), the value of this field shall be the
1146 * product of the Logical Block Size (bLogicalBlockSize) and the
1147 * TRANSFER LENGTH field of the CDB."
1148 */
1149 scsi_cmd_rw_decode(&xs->cmd, &lba, &blocks);
1150 ucd->cmd.expected_xfer_len = htobe32(UFSHCI_LBS * blocks);
1151
1152 memcpy(ucd->cmd.cdb, &xs->cmd, sizeof(ucd->cmd.cdb));
1153
1154 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2g) */
1155 /* Already done with above memset */
1156
1157 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 3) */
1158 dva = UFSHCI_DMA_DVA(sc->sc_dmamem_ucd) + (sizeof(*ucd) * slot);
1159 utrd->dw4 = (uint32_t)dva;
1160 utrd->dw5 = (uint32_t)(dva >> 32);
1161
1162 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 4) */
1163 off = sizeof(struct upiu_command) / 4; /* DWORD offset */
1164 utrd->dw6 = UFSHCI_UTRD_DW6_RUO(off);
1165
1166 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 5) */
1167 len = sizeof(struct upiu_response) / 4; /* DWORD length */
1168 utrd->dw6 |= UFSHCI_UTRD_DW6_RUL(len);
1169
1170 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 6) */
1171 off = (sizeof(struct upiu_command) + sizeof(struct upiu_response)) / 4;
1172 utrd->dw7 = UFSHCI_UTRD_DW7_PRDTO(off);
1173
1174 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 7) */
1175 utrd->dw7 |= UFSHCI_UTRD_DW7_PRDTL(dmap->dm_nsegs);
1176
1177 /* Build PRDT data segment. */
1178 for (i = 0; i < dmap->dm_nsegs; i++) {
1179 dva = dmap->dm_segs[i].ds_addr;
1180 ucd->prdt[i].dw0 = (uint32_t)dva;
1181 ucd->prdt[i].dw1 = (uint32_t)(dva >> 32);
1182 ucd->prdt[i].dw2 = 0;
1183 ucd->prdt[i].dw3 = dmap->dm_segs[i].ds_len - 1;
1184 }
1185
1186 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 9) */
1187 if (UFSHCI_READ_4(sc, UFSHCI_REG_UTRLRSR) != 1) {
1188 printf("%s: %s: UTRLRSR not set\n",
1189 sc->sc_dev.dv_xname, __func__);
1190 return 1;
1191 }
1192
1193 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_utrd),
1194 sizeof(*utrd) * slot, sizeof(*utrd), BUS_DMASYNC_PREWRITE);
1195 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_ucd),
1196 sizeof(*ucd) * slot, sizeof(*ucd), BUS_DMASYNC_PREWRITE);
1197 #if NKSTAT > 0
1198 if (sc->sc_stats_slots)
1199 sc->sc_stats_slots[slot]++;
1200 #endif
1201 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 14) */
1202 ccb->ccb_status = CCB_STATUS_INPROGRESS;
1203 ufshci_doorbell_write(sc, slot);
1204
1205 return 0;
1206 }
1207
1208 int
ufshci_utr_cmd_sync(struct ufshci_softc * sc,struct ufshci_ccb * ccb,struct scsi_xfer * xs,uint32_t lba,uint16_t blocks)1209 ufshci_utr_cmd_sync(struct ufshci_softc *sc, struct ufshci_ccb *ccb,
1210 struct scsi_xfer *xs, uint32_t lba, uint16_t blocks)
1211 {
1212 int slot, off, len;
1213 uint64_t dva;
1214 struct ufshci_utrd *utrd;
1215 struct ufshci_ucd *ucd;
1216
1217 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 1) */
1218 slot = ccb->ccb_slot;
1219 utrd = UFSHCI_DMA_KVA(sc->sc_dmamem_utrd);
1220 utrd += slot;
1221 memset(utrd, 0, sizeof(*utrd));
1222
1223 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2a) */
1224 utrd->dw0 = UFSHCI_UTRD_DW0_CT_UFS;
1225
1226 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2b) */
1227 utrd->dw0 |= UFSHCI_UTRD_DW0_DD_I2T;
1228
1229 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2c) */
1230 if (sc->sc_flags & UFSHCI_FLAGS_AGGR_INTR)
1231 utrd->dw0 |= UFSHCI_UTRD_DW0_I_REG;
1232 else
1233 utrd->dw0 |= UFSHCI_UTRD_DW0_I_INT;
1234
1235 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2d) */
1236 utrd->dw2 = UFSHCI_UTRD_DW2_OCS_IOV;
1237
1238 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2e) */
1239 ucd = UFSHCI_DMA_KVA(sc->sc_dmamem_ucd);
1240 ucd += slot;
1241 memset(ucd, 0, sizeof(*ucd));
1242
1243 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2f) */
1244 ucd->cmd.hdr.tc = UPIU_TC_I2T_COMMAND;
1245 ucd->cmd.hdr.flags = 0; /* No data transfer */
1246 ucd->cmd.hdr.lun = 0;
1247 ucd->cmd.hdr.task_tag = slot;
1248 ucd->cmd.hdr.cmd_set_type = 0; /* SCSI command */
1249 ucd->cmd.hdr.query = 0;
1250 ucd->cmd.hdr.response = 0;
1251 ucd->cmd.hdr.status = 0;
1252 ucd->cmd.hdr.ehs_len = 0;
1253 ucd->cmd.hdr.device_info = 0;
1254 ucd->cmd.hdr.ds_len = 0;
1255
1256 ucd->cmd.expected_xfer_len = htobe32(0); /* No data transfer */
1257
1258 ucd->cmd.cdb[0] = SYNCHRONIZE_CACHE; /* 0x35 */
1259 ucd->cmd.cdb[2] = (lba >> 24) & 0xff;
1260 ucd->cmd.cdb[3] = (lba >> 16) & 0xff;
1261 ucd->cmd.cdb[4] = (lba >> 8) & 0xff;
1262 ucd->cmd.cdb[5] = (lba >> 0) & 0xff;
1263 ucd->cmd.cdb[7] = (blocks >> 8) & 0xff;
1264 ucd->cmd.cdb[8] = (blocks >> 0) & 0xff;
1265
1266 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 2g) */
1267 /* Already done with above memset */
1268
1269 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 3) */
1270 dva = UFSHCI_DMA_DVA(sc->sc_dmamem_ucd) + (sizeof(*ucd) * slot);
1271 utrd->dw4 = (uint32_t)dva;
1272 utrd->dw5 = (uint32_t)(dva >> 32);
1273
1274 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 4) */
1275 off = sizeof(struct upiu_command) / 4; /* DWORD offset */
1276 utrd->dw6 = UFSHCI_UTRD_DW6_RUO(off);
1277
1278 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 5) */
1279 len = sizeof(struct upiu_response) / 4; /* DWORD length */
1280 utrd->dw6 |= UFSHCI_UTRD_DW6_RUL(len);
1281
1282 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 6) */
1283 off = (sizeof(struct upiu_command) + sizeof(struct upiu_response)) / 4;
1284 utrd->dw7 = UFSHCI_UTRD_DW7_PRDTO(off);
1285
1286 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 7) */
1287 utrd->dw7 |= UFSHCI_UTRD_DW7_PRDTL(0); /* No data xfer */
1288
1289 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 9) */
1290 if (UFSHCI_READ_4(sc, UFSHCI_REG_UTRLRSR) != 1) {
1291 printf("%s: %s: UTRLRSR not set\n",
1292 sc->sc_dev.dv_xname, __func__);
1293 return 1;
1294 }
1295
1296 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_utrd),
1297 sizeof(*utrd) * slot, sizeof(*utrd), BUS_DMASYNC_PREWRITE);
1298 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_ucd),
1299 sizeof(*ucd) * slot, sizeof(*ucd), BUS_DMASYNC_PREWRITE);
1300
1301 /* 7.2.1 Basic Steps when Building a UTP Transfer Request: 14) */
1302 ccb->ccb_status = CCB_STATUS_INPROGRESS;
1303 ufshci_doorbell_write(sc, slot);
1304
1305 return 0;
1306 }
1307
1308 void
ufshci_xfer_complete(struct ufshci_softc * sc)1309 ufshci_xfer_complete(struct ufshci_softc *sc)
1310 {
1311 struct ufshci_ccb *ccb;
1312 uint32_t reg;
1313 int i, timeout;
1314
1315 mtx_enter(&sc->sc_cmd_mtx);
1316
1317 /* Wait for all commands to complete. */
1318 for (timeout = 5000; timeout != 0; timeout--) {
1319 reg = ufshci_doorbell_read(sc);
1320 if (reg == 0)
1321 break;
1322 delay(10);
1323 }
1324 if (timeout == 0)
1325 printf("%s: timeout (reg=0x%x)\n", __func__, reg);
1326
1327 for (i = 0; i < sc->sc_nutrs; i++) {
1328 ccb = &sc->sc_ccbs[i];
1329
1330 /* Skip unused CCBs. */
1331 if (ccb->ccb_status != CCB_STATUS_INPROGRESS)
1332 continue;
1333
1334 if (ccb->ccb_done == NULL)
1335 panic("ccb done wasn't defined");
1336
1337 /* 7.2.3: Clear completion notification 3b) */
1338 UFSHCI_WRITE_4(sc, UFSHCI_REG_UTRLCNR, (1U << i));
1339
1340 /* 7.2.3: Mark software slot for reuse 3c) */
1341 ccb->ccb_status = CCB_STATUS_READY2FREE;
1342 }
1343
1344 /* 7.2.3: Reset Interrupt Aggregation Counter and Timer 4) */
1345 if (sc->sc_flags & UFSHCI_FLAGS_AGGR_INTR) {
1346 UFSHCI_WRITE_4(sc, UFSHCI_REG_UTRIACR,
1347 UFSHCI_REG_UTRIACR_IAEN | UFSHCI_REG_UTRIACR_CTR);
1348 }
1349
1350 mtx_leave(&sc->sc_cmd_mtx);
1351
1352 /*
1353 * Complete the CCB, which will re-schedule new transfers if any are
1354 * pending.
1355 */
1356 for (i = 0; i < sc->sc_nutrs; i++) {
1357 ccb = &sc->sc_ccbs[i];
1358
1359 /* 7.2.3: Process the transfer by higher OS layer 3a) */
1360 if (ccb->ccb_status == CCB_STATUS_READY2FREE)
1361 ccb->ccb_done(sc, ccb);
1362 }
1363 }
1364
1365 int
ufshci_activate(struct device * self,int act)1366 ufshci_activate(struct device *self, int act)
1367 {
1368 struct ufshci_softc *sc = (struct ufshci_softc *)self;
1369 int rv = 0;
1370
1371 switch (act) {
1372 case DVACT_POWERDOWN:
1373 DPRINTF(1, "%s: POWERDOWN\n", __func__);
1374 rv = config_activate_children(&sc->sc_dev, act);
1375 ufshci_disable(sc);
1376 break;
1377 case DVACT_RESUME:
1378 DPRINTF(1, "%s: RESUME\n", __func__);
1379 rv = ufshci_init(sc);
1380 if (rv == 0)
1381 rv = config_activate_children(&sc->sc_dev, act);
1382 break;
1383 default:
1384 rv = config_activate_children(&sc->sc_dev, act);
1385 break;
1386 }
1387
1388 return rv;
1389 }
1390
1391 /* SCSI */
1392
1393 int
ufshci_ccb_alloc(struct ufshci_softc * sc,int nccbs)1394 ufshci_ccb_alloc(struct ufshci_softc *sc, int nccbs)
1395 {
1396 struct ufshci_ccb *ccb;
1397 int i;
1398
1399 DPRINTF(2, "%s: nccbs=%d, dma_size=%d, dma_nsegs=%d, "
1400 "dma_segmaxsize=%d\n",
1401 __func__, nccbs, UFSHCI_UCD_PRDT_MAX_XFER, UFSHCI_UCD_PRDT_MAX_SEGS,
1402 UFSHCI_UCD_PRDT_MAX_XFER);
1403
1404 sc->sc_ccbs = mallocarray(nccbs, sizeof(*ccb), M_DEVBUF,
1405 M_WAITOK | M_CANFAIL);
1406 if (sc->sc_ccbs == NULL)
1407 return 1;
1408
1409 for (i = 0; i < nccbs; i++) {
1410 ccb = &sc->sc_ccbs[i];
1411
1412 if (bus_dmamap_create(sc->sc_dmat, UFSHCI_UCD_PRDT_MAX_XFER,
1413 UFSHCI_UCD_PRDT_MAX_SEGS, UFSHCI_UCD_PRDT_MAX_XFER, 0,
1414 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW |
1415 ((sc->sc_cap & UFSHCI_REG_CAP_64AS) ? BUS_DMA_64BIT : 0),
1416 &ccb->ccb_dmamap) != 0)
1417 goto free_maps;
1418
1419 ccb->ccb_cookie = NULL;
1420 ccb->ccb_status = CCB_STATUS_FREE;
1421 ccb->ccb_slot = i;
1422
1423 SIMPLEQ_INSERT_TAIL(&sc->sc_ccb_list, ccb, ccb_entry);
1424 }
1425
1426 return 0;
1427
1428 free_maps:
1429 ufshci_ccb_free(sc, nccbs);
1430 return 1;
1431 }
1432
1433 void *
ufshci_ccb_get(void * cookie)1434 ufshci_ccb_get(void *cookie)
1435 {
1436 struct ufshci_softc *sc = cookie;
1437 struct ufshci_ccb *ccb;
1438
1439 mtx_enter(&sc->sc_ccb_mtx);
1440 ccb = SIMPLEQ_FIRST(&sc->sc_ccb_list);
1441 if (ccb != NULL)
1442 SIMPLEQ_REMOVE_HEAD(&sc->sc_ccb_list, ccb_entry);
1443 mtx_leave(&sc->sc_ccb_mtx);
1444
1445 return ccb;
1446 }
1447
1448 void
ufshci_ccb_put(void * cookie,void * io)1449 ufshci_ccb_put(void *cookie, void *io)
1450 {
1451 struct ufshci_softc *sc = cookie;
1452 struct ufshci_ccb *ccb = io;
1453
1454 mtx_enter(&sc->sc_ccb_mtx);
1455 SIMPLEQ_INSERT_HEAD(&sc->sc_ccb_list, ccb, ccb_entry);
1456 mtx_leave(&sc->sc_ccb_mtx);
1457 }
1458
1459 void
ufshci_ccb_free(struct ufshci_softc * sc,int nccbs)1460 ufshci_ccb_free(struct ufshci_softc *sc, int nccbs)
1461 {
1462 struct ufshci_ccb *ccb;
1463
1464 while ((ccb = SIMPLEQ_FIRST(&sc->sc_ccb_list)) != NULL) {
1465 SIMPLEQ_REMOVE_HEAD(&sc->sc_ccb_list, ccb_entry);
1466 bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap);
1467 }
1468
1469 ufshci_dmamem_free(sc, sc->sc_dmamem_utrd);
1470 free(sc->sc_ccbs, M_DEVBUF, nccbs * sizeof(*ccb));
1471 }
1472
1473 void
ufshci_scsi_cmd(struct scsi_xfer * xs)1474 ufshci_scsi_cmd(struct scsi_xfer *xs)
1475 {
1476 struct scsi_link *link = xs->sc_link;
1477 struct ufshci_softc *sc = link->bus->sb_adapter_softc;
1478
1479 mtx_enter(&sc->sc_cmd_mtx);
1480
1481 switch (xs->cmd.opcode) {
1482
1483 case READ_COMMAND:
1484 case READ_10:
1485 case READ_12:
1486 case READ_16:
1487 ufshci_scsi_io(xs, SCSI_DATA_IN);
1488 break;
1489 case WRITE_COMMAND:
1490 case WRITE_10:
1491 case WRITE_12:
1492 case WRITE_16:
1493 ufshci_scsi_io(xs, SCSI_DATA_OUT);
1494 break;
1495 case SYNCHRONIZE_CACHE:
1496 ufshci_scsi_sync(xs);
1497 break;
1498 case INQUIRY:
1499 ufshci_scsi_inquiry(xs);
1500 break;
1501 case READ_CAPACITY_16:
1502 ufshci_scsi_capacity16(xs);
1503 break;
1504 case READ_CAPACITY:
1505 ufshci_scsi_capacity(xs);
1506 break;
1507 case TEST_UNIT_READY:
1508 case PREVENT_ALLOW:
1509 case START_STOP:
1510 xs->error = XS_NOERROR;
1511 scsi_done(xs);
1512 break;
1513 default:
1514 DPRINTF(3, "%s: unhandled scsi command 0x%02x\n",
1515 __func__, xs->cmd.opcode);
1516 xs->error = XS_DRIVER_STUFFUP;
1517 scsi_done(xs);
1518 break;
1519 }
1520
1521 mtx_leave(&sc->sc_cmd_mtx);
1522 }
1523
1524 void
ufshci_scsi_inquiry(struct scsi_xfer * xs)1525 ufshci_scsi_inquiry(struct scsi_xfer *xs)
1526 {
1527 struct scsi_link *link = xs->sc_link;
1528 struct ufshci_softc *sc = link->bus->sb_adapter_softc;
1529 struct ufshci_ccb *ccb = xs->io;
1530 bus_dmamap_t dmap = ccb->ccb_dmamap;
1531 int error;
1532
1533 DPRINTF(3, "%s: INQUIRY (%s)\n",
1534 __func__, ISSET(xs->flags, SCSI_POLL) ? "poll" : "no poll");
1535
1536 if (xs->datalen > UPIU_SCSI_RSP_INQUIRY_SIZE) {
1537 DPRINTF(2, "%s: request len too large\n", __func__);
1538 goto error1;
1539 }
1540
1541 error = bus_dmamap_load(sc->sc_dmat, dmap, xs->data, xs->datalen, NULL,
1542 ISSET(xs->flags, SCSI_NOSLEEP) ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK);
1543 if (error) {
1544 printf("%s: bus_dmamap_load error=%d\n", __func__, error);
1545 goto error1;
1546 }
1547
1548 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
1549 BUS_DMASYNC_PREREAD);
1550
1551 ccb->ccb_cookie = xs;
1552 ccb->ccb_done = ufshci_scsi_io_done;
1553
1554 /* Response length should be UPIU_SCSI_RSP_INQUIRY_SIZE. */
1555 error = ufshci_utr_cmd_inquiry(sc, ccb, xs);
1556 if (error)
1557 goto error2;
1558
1559 if (ISSET(xs->flags, SCSI_POLL)) {
1560 if (ufshci_doorbell_poll(sc, ccb->ccb_slot, xs->timeout) == 0) {
1561 ccb->ccb_done(sc, ccb);
1562 return;
1563 }
1564 goto error2;
1565 }
1566
1567 return;
1568
1569 error2:
1570 bus_dmamap_unload(sc->sc_dmat, dmap);
1571 ccb->ccb_cookie = NULL;
1572 ccb->ccb_status = CCB_STATUS_FREE;
1573 ccb->ccb_done = NULL;
1574 error1:
1575 xs->error = XS_DRIVER_STUFFUP;
1576 scsi_done(xs);
1577 }
1578
1579 void
ufshci_scsi_capacity16(struct scsi_xfer * xs)1580 ufshci_scsi_capacity16(struct scsi_xfer *xs)
1581 {
1582 struct scsi_link *link = xs->sc_link;
1583 struct ufshci_softc *sc = link->bus->sb_adapter_softc;
1584 struct ufshci_ccb *ccb = xs->io;
1585 bus_dmamap_t dmap = ccb->ccb_dmamap;
1586 int error;
1587
1588 DPRINTF(3, "%s: CAPACITY16 (%s)\n",
1589 __func__, ISSET(xs->flags, SCSI_POLL) ? "poll" : "no poll");
1590
1591 if (xs->datalen > UPIU_SCSI_RSP_CAPACITY16_SIZE) {
1592 DPRINTF(2, "%s: request len too large\n", __func__);
1593 goto error1;
1594 }
1595
1596 error = bus_dmamap_load(sc->sc_dmat, dmap, xs->data, xs->datalen, NULL,
1597 ISSET(xs->flags, SCSI_NOSLEEP) ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK);
1598 if (error) {
1599 printf("%s: bus_dmamap_load error=%d\n", __func__, error);
1600 goto error1;
1601 }
1602
1603 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
1604 BUS_DMASYNC_PREREAD);
1605
1606 ccb->ccb_cookie = xs;
1607 ccb->ccb_done = ufshci_scsi_io_done;
1608
1609 /* Response length should be UPIU_SCSI_RSP_CAPACITY16_SIZE. */
1610 error = ufshci_utr_cmd_capacity16(sc, ccb, xs);
1611 if (error)
1612 goto error2;
1613
1614 if (ISSET(xs->flags, SCSI_POLL)) {
1615 if (ufshci_doorbell_poll(sc, ccb->ccb_slot, xs->timeout) == 0) {
1616 ccb->ccb_done(sc, ccb);
1617 return;
1618 }
1619 goto error2;
1620 }
1621
1622 return;
1623
1624 error2:
1625 bus_dmamap_unload(sc->sc_dmat, dmap);
1626 ccb->ccb_cookie = NULL;
1627 ccb->ccb_status = CCB_STATUS_FREE;
1628 ccb->ccb_done = NULL;
1629 error1:
1630 xs->error = XS_DRIVER_STUFFUP;
1631 scsi_done(xs);
1632 }
1633
1634 void
ufshci_scsi_capacity(struct scsi_xfer * xs)1635 ufshci_scsi_capacity(struct scsi_xfer *xs)
1636 {
1637 struct scsi_link *link = xs->sc_link;
1638 struct ufshci_softc *sc = link->bus->sb_adapter_softc;
1639 struct ufshci_ccb *ccb = xs->io;
1640 bus_dmamap_t dmap = ccb->ccb_dmamap;
1641 int error;
1642
1643 DPRINTF(3, "%s: CAPACITY (%s)\n",
1644 __func__, ISSET(xs->flags, SCSI_POLL) ? "poll" : "no poll");
1645
1646 if (xs->datalen > UPIU_SCSI_RSP_CAPACITY_SIZE) {
1647 DPRINTF(2, "%s: request len too large\n", __func__);
1648 goto error1;
1649 }
1650
1651 error = bus_dmamap_load(sc->sc_dmat, dmap, xs->data, xs->datalen, NULL,
1652 ISSET(xs->flags, SCSI_NOSLEEP) ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK);
1653 if (error) {
1654 printf("%s: bus_dmamap_load error=%d\n", __func__, error);
1655 goto error1;
1656 }
1657
1658 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
1659 BUS_DMASYNC_PREREAD);
1660
1661 ccb->ccb_cookie = xs;
1662 ccb->ccb_done = ufshci_scsi_io_done;
1663
1664 /* Response length should be UPIU_SCSI_RSP_CAPACITY_SIZE */
1665 error = ufshci_utr_cmd_capacity(sc, ccb, xs);
1666 if (error)
1667 goto error2;
1668
1669 if (ISSET(xs->flags, SCSI_POLL)) {
1670 if (ufshci_doorbell_poll(sc, ccb->ccb_slot, xs->timeout) == 0) {
1671 ccb->ccb_done(sc, ccb);
1672 return;
1673 }
1674 goto error2;
1675 }
1676
1677 return;
1678
1679 error2:
1680 bus_dmamap_unload(sc->sc_dmat, dmap);
1681 ccb->ccb_cookie = NULL;
1682 ccb->ccb_status = CCB_STATUS_FREE;
1683 ccb->ccb_done = NULL;
1684 error1:
1685 xs->error = XS_DRIVER_STUFFUP;
1686 scsi_done(xs);
1687 }
1688
1689 void
ufshci_scsi_sync(struct scsi_xfer * xs)1690 ufshci_scsi_sync(struct scsi_xfer *xs)
1691 {
1692 struct scsi_link *link = xs->sc_link;
1693 struct ufshci_softc *sc = link->bus->sb_adapter_softc;
1694 struct ufshci_ccb *ccb = xs->io;
1695 uint64_t lba;
1696 uint32_t blocks;
1697 int error;
1698
1699 /* lba = 0, blocks = 0: Synchronize all logical blocks. */
1700 lba = 0; blocks = 0;
1701
1702 DPRINTF(3, "%s: SYNC, lba=%llu, blocks=%u (%s)\n",
1703 __func__, lba, blocks,
1704 ISSET(xs->flags, SCSI_POLL) ? "poll" : "no poll");
1705
1706 ccb->ccb_cookie = xs;
1707 ccb->ccb_done = ufshci_scsi_done;
1708
1709 error = ufshci_utr_cmd_sync(sc, ccb, xs, (uint32_t)lba,
1710 (uint16_t)blocks);
1711 if (error)
1712 goto error;
1713
1714 if (ISSET(xs->flags, SCSI_POLL)) {
1715 if (ufshci_doorbell_poll(sc, ccb->ccb_slot, xs->timeout) == 0) {
1716 ccb->ccb_done(sc, ccb);
1717 return;
1718 }
1719 goto error;
1720 }
1721
1722 return;
1723
1724 error:
1725 ccb->ccb_cookie = NULL;
1726 ccb->ccb_status = CCB_STATUS_FREE;
1727 ccb->ccb_done = NULL;
1728
1729 xs->error = XS_DRIVER_STUFFUP;
1730 scsi_done(xs);
1731 }
1732
1733 void
ufshci_scsi_io(struct scsi_xfer * xs,int dir)1734 ufshci_scsi_io(struct scsi_xfer *xs, int dir)
1735 {
1736 struct scsi_link *link = xs->sc_link;
1737 struct ufshci_softc *sc = link->bus->sb_adapter_softc;
1738 struct ufshci_ccb *ccb = xs->io;
1739 bus_dmamap_t dmap = ccb->ccb_dmamap;
1740 int error;
1741
1742 if ((xs->flags & (SCSI_DATA_IN | SCSI_DATA_OUT)) != dir)
1743 goto error1;
1744
1745 error = bus_dmamap_load(sc->sc_dmat, dmap, xs->data, xs->datalen, NULL,
1746 ISSET(xs->flags, SCSI_NOSLEEP) ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK);
1747 if (error) {
1748 printf("%s: bus_dmamap_load error=%d\n", __func__, error);
1749 goto error1;
1750 }
1751
1752 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
1753 ISSET(xs->flags, SCSI_DATA_IN) ? BUS_DMASYNC_PREREAD :
1754 BUS_DMASYNC_PREWRITE);
1755
1756 ccb->ccb_cookie = xs;
1757 ccb->ccb_done = ufshci_scsi_io_done;
1758
1759 if (dir == SCSI_DATA_IN)
1760 error = ufshci_utr_cmd_io(sc, ccb, xs, SCSI_DATA_IN);
1761 else
1762 error = ufshci_utr_cmd_io(sc, ccb, xs, SCSI_DATA_OUT);
1763 if (error)
1764 goto error2;
1765
1766 if (ISSET(xs->flags, SCSI_POLL)) {
1767 if (ufshci_doorbell_poll(sc, ccb->ccb_slot, xs->timeout) == 0) {
1768 ccb->ccb_done(sc, ccb);
1769 return;
1770 }
1771 goto error2;
1772 }
1773
1774 return;
1775
1776 error2:
1777 bus_dmamap_unload(sc->sc_dmat, dmap);
1778 ccb->ccb_cookie = NULL;
1779 ccb->ccb_status = CCB_STATUS_FREE;
1780 ccb->ccb_done = NULL;
1781 error1:
1782 xs->error = XS_DRIVER_STUFFUP;
1783 scsi_done(xs);
1784 }
1785
1786 void
ufshci_scsi_io_done(struct ufshci_softc * sc,struct ufshci_ccb * ccb)1787 ufshci_scsi_io_done(struct ufshci_softc *sc, struct ufshci_ccb *ccb)
1788 {
1789 struct scsi_xfer *xs = ccb->ccb_cookie;
1790 bus_dmamap_t dmap = ccb->ccb_dmamap;
1791 struct ufshci_ucd *ucd;
1792 struct ufshci_utrd *utrd;
1793
1794 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
1795 ISSET(xs->flags, SCSI_DATA_IN) ? BUS_DMASYNC_POSTREAD :
1796 BUS_DMASYNC_POSTWRITE);
1797
1798 bus_dmamap_unload(sc->sc_dmat, dmap);
1799
1800 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_ucd),
1801 sizeof(*ucd) * ccb->ccb_slot, sizeof(*ucd),
1802 BUS_DMASYNC_POSTWRITE);
1803 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_utrd),
1804 sizeof(*utrd) * ccb->ccb_slot, sizeof(*utrd),
1805 BUS_DMASYNC_POSTWRITE);
1806
1807 /* TODO: Do more checks on the Response UPIU in case of errors? */
1808 utrd = UFSHCI_DMA_KVA(sc->sc_dmamem_utrd);
1809 utrd += ccb->ccb_slot;
1810 ucd = UFSHCI_DMA_KVA(sc->sc_dmamem_ucd);
1811 ucd += ccb->ccb_slot;
1812 if (utrd->dw2 != UFSHCI_UTRD_DW2_OCS_SUCCESS) {
1813 printf("%s: error: slot=%d, ocs=0x%x, rsp-tc=0x%x\n",
1814 __func__, ccb->ccb_slot, utrd->dw2, ucd->rsp.hdr.tc);
1815 }
1816
1817 ccb->ccb_cookie = NULL;
1818 ccb->ccb_status = CCB_STATUS_FREE;
1819 ccb->ccb_done = NULL;
1820
1821 xs->error = (utrd->dw2 == UFSHCI_UTRD_DW2_OCS_SUCCESS) ?
1822 XS_NOERROR : XS_DRIVER_STUFFUP;
1823 xs->status = SCSI_OK;
1824 xs->resid = 0;
1825 scsi_done(xs);
1826 }
1827
1828 void
ufshci_scsi_done(struct ufshci_softc * sc,struct ufshci_ccb * ccb)1829 ufshci_scsi_done(struct ufshci_softc *sc, struct ufshci_ccb *ccb)
1830 {
1831 struct scsi_xfer *xs = ccb->ccb_cookie;
1832 struct ufshci_ucd *ucd;
1833 struct ufshci_utrd *utrd;
1834
1835 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_ucd),
1836 sizeof(*ucd) * ccb->ccb_slot, sizeof(*ucd),
1837 BUS_DMASYNC_POSTWRITE);
1838 bus_dmamap_sync(sc->sc_dmat, UFSHCI_DMA_MAP(sc->sc_dmamem_utrd),
1839 sizeof(*utrd) * ccb->ccb_slot, sizeof(*utrd),
1840 BUS_DMASYNC_POSTWRITE);
1841
1842 /* TODO: Do more checks on the Response UPIU in case of errors? */
1843 utrd = UFSHCI_DMA_KVA(sc->sc_dmamem_utrd);
1844 utrd += ccb->ccb_slot;
1845 ucd = UFSHCI_DMA_KVA(sc->sc_dmamem_ucd);
1846 ucd += ccb->ccb_slot;
1847 if (utrd->dw2 != UFSHCI_UTRD_DW2_OCS_SUCCESS) {
1848 printf("%s: error: slot=%d, ocs=0x%x, rsp-tc=0x%x\n",
1849 __func__, ccb->ccb_slot, utrd->dw2, ucd->rsp.hdr.tc);
1850 }
1851
1852 ccb->ccb_cookie = NULL;
1853 ccb->ccb_status = CCB_STATUS_FREE;
1854 ccb->ccb_done = NULL;
1855
1856 xs->error = (utrd->dw2 == UFSHCI_UTRD_DW2_OCS_SUCCESS) ?
1857 XS_NOERROR : XS_DRIVER_STUFFUP;
1858 xs->status = SCSI_OK;
1859 xs->resid = 0;
1860 scsi_done(xs);
1861 }
1862
1863 #ifdef HIBERNATE
1864 int
ufshci_hibernate_io(dev_t dev,daddr_t blkno,vaddr_t addr,size_t size,int op,void * page)1865 ufshci_hibernate_io(dev_t dev, daddr_t blkno, vaddr_t addr, size_t size,
1866 int op, void *page)
1867 {
1868 struct ufshci_hibernate_page {
1869 struct ufshci_utrd utrd;
1870 struct ufshci_ucd ucd;
1871
1872 struct ufshci_softc *sc; /* Copy of softc */
1873
1874 daddr_t poffset; /* Start of SWAP partition */
1875 size_t psize; /* Size of SWAP partition */
1876 uint32_t secsize; /* Our sector size */
1877 } *my = page;
1878 paddr_t data_phys, page_phys;
1879 uint64_t data_bus_phys, page_bus_phys;
1880 uint64_t timeout_us;
1881 int off, len, slot;
1882 uint32_t blocks, reg;
1883 uint64_t lba;
1884
1885 if (op == HIB_INIT) {
1886 struct device *disk;
1887 struct device *scsibus;
1888 extern struct cfdriver sd_cd;
1889
1890 /* Find ufshci softc. */
1891 disk = disk_lookup(&sd_cd, DISKUNIT(dev));
1892 if (disk == NULL)
1893 return ENOTTY;
1894 scsibus = disk->dv_parent;
1895 my->sc = (struct ufshci_softc *)disk->dv_parent->dv_parent;
1896
1897 /* Stop run queues and disable interrupts. */
1898 ufshci_disable(my->sc);
1899
1900 /* Tell the controller the new hibernate UTRD address. */
1901 pmap_extract(pmap_kernel(), (vaddr_t)page, &page_phys);
1902 page_bus_phys = page_phys + ((void *)&my->utrd - page);
1903 UFSHCI_WRITE_4(my->sc, UFSHCI_REG_UTRLBA,
1904 (uint32_t)page_bus_phys);
1905 UFSHCI_WRITE_4(my->sc, UFSHCI_REG_UTRLBAU,
1906 (uint32_t)(page_bus_phys >> 32));
1907
1908 /* Start run queues. */
1909 UFSHCI_WRITE_4(my->sc, UFSHCI_REG_UTMRLRSR,
1910 UFSHCI_REG_UTMRLRSR_START);
1911 UFSHCI_WRITE_4(my->sc, UFSHCI_REG_UTRLRSR,
1912 UFSHCI_REG_UTRLRSR_START);
1913
1914 my->poffset = blkno;
1915 my->psize = size;
1916 my->secsize = UFSHCI_LBS;
1917
1918 return 0;
1919 }
1920
1921 if (op != HIB_W)
1922 return 0;
1923
1924 if (blkno + (size / DEV_BSIZE) > my->psize)
1925 return E2BIG;
1926 blocks = size / my->secsize;
1927 lba = (blkno + my->poffset) / (my->secsize / DEV_BSIZE);
1928
1929 /*
1930 * The following code is a ripped down version of ufshci_utr_cmd_io()
1931 * adapted for hibernate.
1932 */
1933 slot = 0; /* We only use the first slot for hibernate */
1934
1935 memset(&my->utrd, 0, sizeof(struct ufshci_utrd));
1936
1937 my->utrd.dw0 = UFSHCI_UTRD_DW0_CT_UFS;
1938 my->utrd.dw0 |= UFSHCI_UTRD_DW0_DD_I2T;
1939 my->utrd.dw0 |= UFSHCI_UTRD_DW0_I_REG;
1940 my->utrd.dw2 = UFSHCI_UTRD_DW2_OCS_IOV;
1941
1942 memset(&my->ucd, 0, sizeof(struct ufshci_ucd));
1943
1944 my->ucd.cmd.hdr.tc = UPIU_TC_I2T_COMMAND;
1945 my->ucd.cmd.hdr.flags = (1 << 5); /* Bit-5 = Write */
1946
1947 my->ucd.cmd.hdr.lun = 0;
1948 my->ucd.cmd.hdr.task_tag = slot;
1949 my->ucd.cmd.hdr.cmd_set_type = 0; /* SCSI command */
1950 my->ucd.cmd.hdr.query = 0;
1951 my->ucd.cmd.hdr.response = 0;
1952 my->ucd.cmd.hdr.status = 0;
1953 my->ucd.cmd.hdr.ehs_len = 0;
1954 my->ucd.cmd.hdr.device_info = 0;
1955 my->ucd.cmd.hdr.ds_len = 0;
1956
1957 my->ucd.cmd.expected_xfer_len = htobe32(UFSHCI_LBS * blocks);
1958 my->ucd.cmd.cdb[0] = WRITE_10; /* 0x2a */
1959 my->ucd.cmd.cdb[1] = (1 << 3); /* FUA: Force Unit Access */
1960 my->ucd.cmd.cdb[2] = (lba >> 24) & 0xff;
1961 my->ucd.cmd.cdb[3] = (lba >> 16) & 0xff;
1962 my->ucd.cmd.cdb[4] = (lba >> 8) & 0xff;
1963 my->ucd.cmd.cdb[5] = (lba >> 0) & 0xff;
1964 my->ucd.cmd.cdb[7] = (blocks >> 8) & 0xff;
1965 my->ucd.cmd.cdb[8] = (blocks >> 0) & 0xff;
1966
1967 pmap_extract(pmap_kernel(), (vaddr_t)page, &page_phys);
1968 page_bus_phys = page_phys + ((void *)&my->ucd - page);
1969 my->utrd.dw4 = (uint32_t)page_bus_phys;
1970 my->utrd.dw5 = (uint32_t)(page_bus_phys >> 32);
1971
1972 off = sizeof(struct upiu_command) / 4; /* DWORD offset */
1973 my->utrd.dw6 = UFSHCI_UTRD_DW6_RUO(off);
1974
1975 len = sizeof(struct upiu_response) / 4; /* DWORD length */
1976 my->utrd.dw6 |= UFSHCI_UTRD_DW6_RUL(len);
1977
1978 off = (sizeof(struct upiu_command) + sizeof(struct upiu_response)) / 4;
1979 my->utrd.dw7 = UFSHCI_UTRD_DW7_PRDTO(off);
1980
1981 my->utrd.dw7 |= UFSHCI_UTRD_DW7_PRDTL(1); /* dm_nsegs */
1982
1983 pmap_extract(pmap_kernel(), (vaddr_t)addr, &data_phys);
1984 data_bus_phys = data_phys;
1985 my->ucd.prdt[0].dw0 = (uint32_t)data_bus_phys;
1986 my->ucd.prdt[0].dw1 = (uint32_t)(data_bus_phys >> 32);
1987 my->ucd.prdt[0].dw2 = 0;
1988 my->ucd.prdt[0].dw3 = size - 1; /* ds_len */
1989
1990 if (UFSHCI_READ_4(my->sc, UFSHCI_REG_UTRLRSR) != 1)
1991 return EIO;
1992
1993 ufshci_doorbell_write(my->sc, slot);
1994
1995 /* ufshci_doorbell_poll() adaption for hibernate. */
1996 for (timeout_us = 1000000 * 1000; timeout_us != 0;
1997 timeout_us -= 1000) {
1998 reg = UFSHCI_READ_4(my->sc, UFSHCI_REG_UTRLDBR);
1999 if ((reg & (1U << slot)) == 0)
2000 break;
2001 delay(1000);
2002 }
2003 if (timeout_us == 0)
2004 return EIO;
2005 UFSHCI_WRITE_4(my->sc, UFSHCI_REG_UTRLCNR, (1U << slot));
2006
2007 /* Check if the command was successfully executed. */
2008 if (my->utrd.dw2 != UFSHCI_UTRD_DW2_OCS_SUCCESS)
2009 return EIO;
2010
2011 return 0;
2012 }
2013 #endif /* HIBERNATE */
2014
2015 #if NKSTAT > 0
2016 struct kstat_kv ufshci_counters_slot[CCB_STATUS_COUNT] = {
2017 KSTAT_KV_UNIT_INITIALIZER("slots free", KSTAT_KV_T_COUNTER16,
2018 KSTAT_KV_U_NONE),
2019 KSTAT_KV_UNIT_INITIALIZER("slots inpr", KSTAT_KV_T_COUNTER16,
2020 KSTAT_KV_U_NONE),
2021 KSTAT_KV_UNIT_INITIALIZER("slots r2fr", KSTAT_KV_T_COUNTER16,
2022 KSTAT_KV_U_NONE),
2023 };
2024
2025 void
ufshci_kstat_attach(struct ufshci_softc * sc)2026 ufshci_kstat_attach(struct ufshci_softc *sc)
2027 {
2028 struct kstat *ks;
2029 struct kstat_kv *kvs;
2030 char name[KSTAT_KV_NAMELEN];
2031 int i;
2032
2033 /*
2034 * Allocate array to count ccb slot utilization.
2035 */
2036 sc->sc_stats_slots = mallocarray(sc->sc_nutrs, sizeof(uint64_t),
2037 M_DEVBUF, M_WAITOK | M_ZERO);
2038 if (sc->sc_stats_slots == NULL) {
2039 printf("%s: can't allocate stats_slots array\n",
2040 sc->sc_dev.dv_xname);
2041 return;
2042 }
2043
2044 /*
2045 * Setup 'ccbs' kstat.
2046 */
2047 kvs = mallocarray(sc->sc_nutrs, sizeof(*kvs), M_DEVBUF,
2048 M_WAITOK | M_ZERO);
2049 if (kvs == NULL) {
2050 printf("%s: can't allocate kvs ccbs array\n",
2051 sc->sc_dev.dv_xname);
2052 return;
2053 }
2054 for (i = 0; i < sc->sc_nutrs; i++) {
2055 snprintf(name, sizeof(name), "slot %d ccbs", i);
2056 kstat_kv_unit_init(&kvs[i], name, KSTAT_KV_T_COUNTER64,
2057 KSTAT_KV_U_NONE);
2058 }
2059
2060 mtx_init(&sc->sc_kstat_mtx_ccb, IPL_SOFTCLOCK);
2061
2062 ks = kstat_create(sc->sc_dev.dv_xname, 0, "ccbs", 0, KSTAT_T_KV, 0);
2063 if (ks == NULL) {
2064 printf("%s: can't create ccbs kstats\n", sc->sc_dev.dv_xname);
2065 free(kvs, M_DEVBUF, sc->sc_nutrs * sizeof(*kvs));
2066 return;
2067 }
2068
2069 kstat_set_mutex(ks, &sc->sc_kstat_mtx_ccb);
2070 ks->ks_softc = sc;
2071 ks->ks_data = kvs;
2072 ks->ks_datalen = sc->sc_nutrs * sizeof(*kvs);
2073 ks->ks_read = ufshci_kstat_read_ccb;
2074
2075 sc->sc_kstat_ccb = ks;
2076 kstat_install(ks);
2077
2078 /*
2079 * Setup 'slots' kstat.
2080 */
2081 mtx_init(&sc->sc_kstat_mtx_slot, IPL_SOFTCLOCK);
2082
2083 ks = kstat_create(sc->sc_dev.dv_xname, 0, "slots", 0, KSTAT_T_KV, 0);
2084 if (ks == NULL) {
2085 printf("%s: can't create slots kstats\n", sc->sc_dev.dv_xname);
2086 return;
2087 }
2088
2089 kstat_set_mutex(ks, &sc->sc_kstat_mtx_slot);
2090 ks->ks_softc = sc;
2091 ks->ks_data = ufshci_counters_slot;
2092 ks->ks_datalen = CCB_STATUS_COUNT * sizeof(*kvs);
2093 ks->ks_read = ufshci_kstat_read_slot;
2094
2095 sc->sc_kstat_slot = ks;
2096 kstat_install(ks);
2097 }
2098
2099 int
ufshci_kstat_read_ccb(struct kstat * ks)2100 ufshci_kstat_read_ccb(struct kstat *ks)
2101 {
2102 struct ufshci_softc *sc = ks->ks_softc;
2103 struct kstat_kv *kvs = ks->ks_data;
2104 int i;
2105
2106 for (i = 0; i < sc->sc_nutrs; i++)
2107 kstat_kv_u64(&kvs[i]) = sc->sc_stats_slots[i];
2108
2109 return 0;
2110 }
2111
2112 int
ufshci_kstat_read_slot(struct kstat * ks)2113 ufshci_kstat_read_slot(struct kstat *ks)
2114 {
2115 struct ufshci_softc *sc = ks->ks_softc;
2116 struct kstat_kv *kvs = ks->ks_data;
2117 struct ufshci_ccb *ccb;
2118 uint16_t free, inprogress, ready2free;
2119 int i;
2120
2121 free = inprogress = ready2free = 0;
2122
2123 for (i = 0; i < sc->sc_nutrs; i++) {
2124 ccb = &sc->sc_ccbs[i];
2125
2126 switch (ccb->ccb_status) {
2127 case CCB_STATUS_FREE:
2128 free++;
2129 break;
2130 case CCB_STATUS_INPROGRESS:
2131 inprogress++;
2132 break;
2133 case CCB_STATUS_READY2FREE:
2134 ready2free++;
2135 break;
2136 }
2137 }
2138
2139 kstat_kv_u16(&kvs[0]) = free;
2140 kstat_kv_u16(&kvs[1]) = inprogress;
2141 kstat_kv_u16(&kvs[2]) = ready2free;
2142
2143 return 0;
2144 }
2145 #endif /* NKSTAT > 0 */
2146