1 /* $OpenBSD: ahci.c,v 1.43 2024/11/22 09:29:41 jan Exp $ */
2
3 /*
4 * Copyright (c) 2006 David Gwynne <dlg@openbsd.org>
5 * Copyright (c) 2010 Conformal Systems LLC <info@conformal.com>
6 * Copyright (c) 2010 Jonathan Matthew <jonathan@d14n.org>
7 *
8 * Permission to use, copy, modify, and distribute this software for any
9 * purpose with or without fee is hereby granted, provided that the above
10 * copyright notice and this permission notice appear in all copies.
11 *
12 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19 */
20
21 #include <sys/param.h>
22 #include <sys/systm.h>
23 #include <sys/buf.h>
24 #include <sys/kernel.h>
25 #include <sys/malloc.h>
26 #include <sys/device.h>
27 #include <sys/queue.h>
28 #include <sys/mutex.h>
29 #include <sys/pool.h>
30
31 #include <machine/bus.h>
32
33 #include <dev/ic/ahcireg.h>
34 #include <dev/ic/ahcivar.h>
35
36 #ifdef AHCI_DEBUG
37 #define DPRINTF(m, f...) do { if ((ahcidebug & (m)) == (m)) printf(f); } \
38 while (0)
39 #define AHCI_D_TIMEOUT 0x00
40 #define AHCI_D_VERBOSE 0x01
41 #define AHCI_D_INTR 0x02
42 #define AHCI_D_XFER 0x08
43 int ahcidebug = AHCI_D_VERBOSE;
44 #else
45 #define DPRINTF(m, f...)
46 #endif
47
48 #ifdef HIBERNATE
49 #include <uvm/uvm_extern.h>
50 #include <sys/hibernate.h>
51 #include <sys/disk.h>
52 #include <sys/disklabel.h>
53
54 #include <scsi/scsi_all.h>
55 #include <scsi/scsiconf.h>
56
57 void ahci_hibernate_io_start(struct ahci_port *,
58 struct ahci_ccb *);
59 int ahci_hibernate_io_poll(struct ahci_port *,
60 struct ahci_ccb *);
61 void ahci_hibernate_load_prdt(struct ahci_ccb *);
62
63 int ahci_hibernate_io(dev_t dev, daddr_t blkno,
64 vaddr_t addr, size_t size, int wr, void *page);
65 #endif
66
67 struct cfdriver ahci_cd = {
68 NULL, "ahci", DV_DULL
69 };
70
71 void ahci_enable_interrupts(struct ahci_port *);
72
73 int ahci_init(struct ahci_softc *);
74 int ahci_port_alloc(struct ahci_softc *, u_int);
75 void ahci_port_detect(struct ahci_softc *, u_int);
76 void ahci_port_free(struct ahci_softc *, u_int);
77 int ahci_port_init(struct ahci_softc *, u_int);
78
79 int ahci_default_port_start(struct ahci_port *, int);
80 int ahci_port_stop(struct ahci_port *, int);
81 int ahci_port_clo(struct ahci_port *);
82 int ahci_port_softreset(struct ahci_port *);
83 void ahci_port_comreset(struct ahci_port *);
84 int ahci_port_portreset(struct ahci_port *, int);
85 void ahci_port_portreset_start(struct ahci_port *);
86 int ahci_port_portreset_poll(struct ahci_port *);
87 void ahci_port_portreset_wait(struct ahci_port *);
88 int ahci_port_portreset_finish(struct ahci_port *, int);
89 int ahci_port_signature(struct ahci_port *);
90 int ahci_pmp_port_softreset(struct ahci_port *, int);
91 int ahci_pmp_port_portreset(struct ahci_port *, int);
92 int ahci_pmp_port_probe(struct ahci_port *ap, int pmp_port);
93
94 int ahci_load_prdt(struct ahci_ccb *);
95 void ahci_load_prdt_seg(struct ahci_prdt *, u_int64_t,
96 u_int32_t, u_int32_t);
97 void ahci_unload_prdt(struct ahci_ccb *);
98
99 int ahci_poll(struct ahci_ccb *, int, void (*)(void *));
100 void ahci_start(struct ahci_ccb *);
101
102 void ahci_issue_pending_ncq_commands(struct ahci_port *);
103 void ahci_issue_pending_commands(struct ahci_port *, int);
104
105 int ahci_intr(void *);
106 u_int32_t ahci_port_intr(struct ahci_port *, u_int32_t);
107
108 struct ahci_ccb *ahci_get_ccb(struct ahci_port *);
109 void ahci_put_ccb(struct ahci_ccb *);
110
111 struct ahci_ccb *ahci_get_err_ccb(struct ahci_port *);
112 void ahci_put_err_ccb(struct ahci_ccb *);
113
114 struct ahci_ccb *ahci_get_pmp_ccb(struct ahci_port *);
115 void ahci_put_pmp_ccb(struct ahci_ccb *);
116
117 int ahci_port_read_ncq_error(struct ahci_port *, int *, int);
118
119 struct ahci_dmamem *ahci_dmamem_alloc(struct ahci_softc *, size_t);
120 void ahci_dmamem_free(struct ahci_softc *,
121 struct ahci_dmamem *);
122
123 u_int32_t ahci_read(struct ahci_softc *, bus_size_t);
124 void ahci_write(struct ahci_softc *, bus_size_t, u_int32_t);
125 int ahci_wait_ne(struct ahci_softc *, bus_size_t,
126 u_int32_t, u_int32_t);
127
128 u_int32_t ahci_pread(struct ahci_port *, bus_size_t);
129 void ahci_pwrite(struct ahci_port *, bus_size_t, u_int32_t);
130 int ahci_pwait_eq(struct ahci_port *, bus_size_t,
131 u_int32_t, u_int32_t, int);
132 void ahci_flush_tfd(struct ahci_port *ap);
133 u_int32_t ahci_active_mask(struct ahci_port *);
134 int ahci_port_detect_pmp(struct ahci_port *);
135 void ahci_pmp_probe_timeout(void *);
136
137 /* pmp operations */
138 int ahci_pmp_read(struct ahci_port *, int, int,
139 u_int32_t *);
140 int ahci_pmp_write(struct ahci_port *, int, int, u_int32_t);
141 int ahci_pmp_phy_status(struct ahci_port *, int,
142 u_int32_t *);
143 int ahci_pmp_identify(struct ahci_port *, int *);
144
145
146 /* Wait for all bits in _b to be cleared */
147 #define ahci_pwait_clr(_ap, _r, _b, _n) \
148 ahci_pwait_eq((_ap), (_r), (_b), 0, (_n))
149
150 /* Wait for all bits in _b to be set */
151 #define ahci_pwait_set(_ap, _r, _b, _n) \
152 ahci_pwait_eq((_ap), (_r), (_b), (_b), (_n))
153
154
155
156 /* provide methods for atascsi to call */
157 int ahci_ata_probe(void *, int, int);
158 void ahci_ata_free(void *, int, int);
159 struct ata_xfer * ahci_ata_get_xfer(void *, int);
160 void ahci_ata_put_xfer(struct ata_xfer *);
161 void ahci_ata_cmd(struct ata_xfer *);
162
163 const struct atascsi_methods ahci_atascsi_methods = {
164 ahci_ata_probe,
165 ahci_ata_free,
166 ahci_ata_get_xfer,
167 ahci_ata_put_xfer,
168 ahci_ata_cmd
169 };
170
171 /* ccb completions */
172 void ahci_ata_cmd_done(struct ahci_ccb *);
173 void ahci_pmp_cmd_done(struct ahci_ccb *);
174 void ahci_ata_cmd_timeout(void *);
175 void ahci_empty_done(struct ahci_ccb *);
176
177 int
ahci_attach(struct ahci_softc * sc)178 ahci_attach(struct ahci_softc *sc)
179 {
180 struct atascsi_attach_args aaa;
181 u_int32_t pi;
182 int i, j, done;
183
184 if (sc->sc_port_start == NULL)
185 sc->sc_port_start = ahci_default_port_start;
186
187 if (ahci_init(sc) != 0) {
188 /* error already printed by ahci_init */
189 goto unmap;
190 }
191
192 printf("\n");
193
194 sc->sc_cap = ahci_read(sc, AHCI_REG_CAP);
195 sc->sc_ncmds = AHCI_REG_CAP_NCS(sc->sc_cap);
196 #ifdef AHCI_DEBUG
197 if (ahcidebug & AHCI_D_VERBOSE) {
198 const char *gen;
199
200 switch (sc->sc_cap & AHCI_REG_CAP_ISS) {
201 case AHCI_REG_CAP_ISS_G1:
202 gen = "1 (1.5Gbps)";
203 break;
204 case AHCI_REG_CAP_ISS_G2:
205 gen = "2 (3.0Gb/s)";
206 break;
207 case AHCI_REG_CAP_ISS_G3:
208 gen = "3 (6.0Gb/s)";
209 break;
210 default:
211 gen = "unknown";
212 break;
213 }
214
215 printf("%s: capabilities 0x%b, %d ports, %d cmds, gen %s\n",
216 DEVNAME(sc), sc->sc_cap, AHCI_FMT_CAP,
217 AHCI_REG_CAP_NP(sc->sc_cap), sc->sc_ncmds, gen);
218 printf("%s: extended capabilities 0x%b\n", DEVNAME(sc),
219 ahci_read(sc, AHCI_REG_CAP2), AHCI_FMT_CAP2);
220 }
221 #endif
222
223 pi = ahci_read(sc, AHCI_REG_PI);
224 DPRINTF(AHCI_D_VERBOSE, "%s: ports implemented: 0x%08x\n",
225 DEVNAME(sc), pi);
226
227 #ifdef AHCI_COALESCE
228 /* Naive coalescing support - enable for all ports. */
229 if (sc->sc_cap & AHCI_REG_CAP_CCCS) {
230 u_int16_t ccc_timeout = 20;
231 u_int8_t ccc_numcomplete = 12;
232 u_int32_t ccc_ctl;
233
234 /* disable coalescing during reconfiguration. */
235 ccc_ctl = ahci_read(sc, AHCI_REG_CCC_CTL);
236 ccc_ctl &= ~0x00000001;
237 ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl);
238
239 sc->sc_ccc_mask = 1 << AHCI_REG_CCC_CTL_INT(ccc_ctl);
240 if (pi & sc->sc_ccc_mask) {
241 /* A conflict with the implemented port list? */
242 printf("%s: coalescing interrupt/implemented port list "
243 "conflict, PI: %08x, ccc_mask: %08x\n",
244 DEVNAME(sc), pi, sc->sc_ccc_mask);
245 sc->sc_ccc_mask = 0;
246 goto noccc;
247 }
248
249 /* ahci_port_start will enable each port when it starts. */
250 sc->sc_ccc_ports = pi;
251 sc->sc_ccc_ports_cur = 0;
252
253 /* program thresholds and enable overall coalescing. */
254 ccc_ctl &= ~0xffffff00;
255 ccc_ctl |= (ccc_timeout << 16) | (ccc_numcomplete << 8);
256 ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl);
257 ahci_write(sc, AHCI_REG_CCC_PORTS, 0);
258 ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl | 1);
259 }
260 noccc:
261 #endif
262 /*
263 * Given that ahci_port_alloc() will grab one CCB for error recovery
264 * in the NCQ case from the pool of CCBs sized based on sc->sc_ncmds
265 * pretend at least 2 command slots for devices without NCQ support.
266 * That way, also at least 1 slot is made available for atascsi(4).
267 */
268 sc->sc_ncmds = max(2, sc->sc_ncmds);
269 for (i = 0; i < AHCI_MAX_PORTS; i++) {
270 if (!ISSET(pi, 1U << i)) {
271 /* don't allocate stuff if the port isn't implemented */
272 continue;
273 }
274
275 if (ahci_port_alloc(sc, i) == ENOMEM)
276 goto freeports;
277
278 if (sc->sc_ports[i] != NULL)
279 ahci_port_portreset_start(sc->sc_ports[i]);
280 }
281
282 /*
283 * Poll for device detection until all ports report a device, or one
284 * second has elapsed.
285 */
286 for (i = 0; i < 1000; i++) {
287 done = 1;
288 for (j = 0; j < AHCI_MAX_PORTS; j++) {
289 if (sc->sc_ports[j] == NULL)
290 continue;
291
292 if (ahci_port_portreset_poll(sc->sc_ports[j]))
293 done = 0;
294 }
295
296 if (done)
297 break;
298
299 delay(1000);
300 }
301
302 /*
303 * Finish device detection on all ports that initialized.
304 */
305 for (i = 0; i < AHCI_MAX_PORTS; i++) {
306 if (sc->sc_ports[i] != NULL)
307 ahci_port_detect(sc, i);
308 }
309
310 memset(&aaa, 0, sizeof(aaa));
311 aaa.aaa_cookie = sc;
312 aaa.aaa_methods = &ahci_atascsi_methods;
313 aaa.aaa_minphys = NULL;
314 aaa.aaa_nports = AHCI_MAX_PORTS;
315 aaa.aaa_ncmds = sc->sc_ncmds - 1;
316 if (!(sc->sc_flags & AHCI_F_NO_NCQ) &&
317 sc->sc_ncmds > 2 &&
318 (sc->sc_cap & AHCI_REG_CAP_SNCQ)) {
319 aaa.aaa_capability |= ASAA_CAP_NCQ | ASAA_CAP_PMP_NCQ;
320 }
321
322 sc->sc_atascsi = atascsi_attach(&sc->sc_dev, &aaa);
323
324 /* Flush all residual bits of the interrupt status register */
325 ahci_write(sc, AHCI_REG_IS, ahci_read(sc, AHCI_REG_IS));
326
327 /* Enable interrupts */
328 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE | AHCI_REG_GHC_IE);
329
330 return 0;
331
332 freeports:
333 for (i = 0; i < AHCI_MAX_PORTS; i++) {
334 if (sc->sc_ports[i] != NULL)
335 ahci_port_free(sc, i);
336 }
337 unmap:
338 /* Disable controller */
339 ahci_write(sc, AHCI_REG_GHC, 0);
340 return 1;
341 }
342
343 int
ahci_detach(struct ahci_softc * sc,int flags)344 ahci_detach(struct ahci_softc *sc, int flags)
345 {
346 int rv, i;
347
348 if (sc->sc_atascsi != NULL) {
349 rv = atascsi_detach(sc->sc_atascsi, flags);
350 if (rv != 0)
351 return (rv);
352 }
353
354 for (i = 0; i < AHCI_MAX_PORTS; i++) {
355 if (sc->sc_ports[i] != NULL)
356 ahci_port_free(sc, i);
357 }
358
359 return (0);
360 }
361
362 int
ahci_activate(struct device * self,int act)363 ahci_activate(struct device *self, int act)
364 {
365 struct ahci_softc *sc = (struct ahci_softc *)self;
366 int i, rv = 0;
367
368 switch (act) {
369 case DVACT_RESUME:
370 /* enable ahci (global interrupts disabled) */
371 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE);
372
373 /* restore BIOS initialised parameters */
374 ahci_write(sc, AHCI_REG_CAP, sc->sc_cap);
375
376 for (i = 0; i < AHCI_MAX_PORTS; i++) {
377 if (sc->sc_ports[i] != NULL)
378 ahci_port_init(sc, i);
379 }
380
381 /* Enable interrupts */
382 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE | AHCI_REG_GHC_IE);
383
384 rv = config_activate_children(self, act);
385 break;
386 case DVACT_POWERDOWN:
387 rv = config_activate_children(self, act);
388 for (i = 0; i < AHCI_MAX_PORTS; i++) {
389 if (sc->sc_ports[i] != NULL)
390 ahci_port_stop(sc->sc_ports[i], 1);
391 }
392 break;
393 default:
394 rv = config_activate_children(self, act);
395 break;
396 }
397 return (rv);
398 }
399
400 int
ahci_init(struct ahci_softc * sc)401 ahci_init(struct ahci_softc *sc)
402 {
403 u_int32_t reg, cap, pi;
404 const char *revision;
405
406 DPRINTF(AHCI_D_VERBOSE, " GHC 0x%b", ahci_read(sc, AHCI_REG_GHC),
407 AHCI_FMT_GHC);
408
409 /* save BIOS initialised parameters, enable staggered spin up */
410 cap = ahci_read(sc, AHCI_REG_CAP);
411 cap &= AHCI_REG_CAP_SMPS;
412 cap |= AHCI_REG_CAP_SSS;
413 pi = ahci_read(sc, AHCI_REG_PI);
414
415 if (ISSET(AHCI_REG_GHC_AE, ahci_read(sc, AHCI_REG_GHC))) {
416 /* reset the controller */
417 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_HR);
418 if (ahci_wait_ne(sc, AHCI_REG_GHC, AHCI_REG_GHC_HR,
419 AHCI_REG_GHC_HR) != 0) {
420 printf(" unable to reset controller\n");
421 return (1);
422 }
423 }
424
425 /* enable ahci (global interrupts disabled) */
426 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE);
427
428 /* restore parameters */
429 ahci_write(sc, AHCI_REG_CAP, cap);
430 ahci_write(sc, AHCI_REG_PI, pi);
431
432 /* check the revision */
433 reg = ahci_read(sc, AHCI_REG_VS);
434 switch (reg) {
435 case AHCI_REG_VS_0_95:
436 revision = "0.95";
437 break;
438 case AHCI_REG_VS_1_0:
439 revision = "1.0";
440 break;
441 case AHCI_REG_VS_1_1:
442 revision = "1.1";
443 break;
444 case AHCI_REG_VS_1_2:
445 revision = "1.2";
446 break;
447 case AHCI_REG_VS_1_3:
448 revision = "1.3";
449 break;
450 case AHCI_REG_VS_1_3_1:
451 revision = "1.3.1";
452 break;
453
454 default:
455 printf(" unsupported AHCI revision 0x%08x\n", reg);
456 return (1);
457 }
458
459 printf(" AHCI %s", revision);
460
461 return (0);
462 }
463
464 void
ahci_enable_interrupts(struct ahci_port * ap)465 ahci_enable_interrupts(struct ahci_port *ap)
466 {
467 ahci_pwrite(ap, AHCI_PREG_IE, AHCI_PREG_IE_TFEE | AHCI_PREG_IE_HBFE |
468 AHCI_PREG_IE_IFE | AHCI_PREG_IE_OFE | AHCI_PREG_IE_DPE |
469 AHCI_PREG_IE_UFE |
470 ((ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF) ? AHCI_PREG_IE_IPME : 0) |
471 #ifdef AHCI_COALESCE
472 ((ap->ap_sc->sc_ccc_ports & (1 << ap->ap_port)) ? 0 :
473 (AHCI_PREG_IE_SDBE | AHCI_PREG_IE_DHRE))
474 #else
475 AHCI_PREG_IE_SDBE | AHCI_PREG_IE_DHRE
476 #endif
477 );
478 }
479
480 int
ahci_port_alloc(struct ahci_softc * sc,u_int port)481 ahci_port_alloc(struct ahci_softc *sc, u_int port)
482 {
483 struct ahci_port *ap;
484 struct ahci_ccb *ccb;
485 u_int64_t dva;
486 u_int32_t cmd;
487 struct ahci_cmd_hdr *hdr;
488 struct ahci_cmd_table *table;
489 int i, rc = ENOMEM;
490
491 ap = malloc(sizeof(*ap), M_DEVBUF, M_NOWAIT | M_ZERO);
492 if (ap == NULL) {
493 printf("%s: unable to allocate memory for port %d\n",
494 DEVNAME(sc), port);
495 goto reterr;
496 }
497 ap->ap_err_scratch = dma_alloc(DEV_BSIZE, PR_NOWAIT | PR_ZERO);
498 if (ap->ap_err_scratch == NULL) {
499 printf("%s: unable to allocate DMA scratch buf for port %d\n",
500 DEVNAME(sc), port);
501 free(ap, M_DEVBUF, sizeof(*ap));
502 goto reterr;
503 }
504
505 #ifdef AHCI_DEBUG
506 snprintf(ap->ap_name, sizeof(ap->ap_name), "%s.%d",
507 DEVNAME(sc), port);
508 #endif
509 ap->ap_port = port;
510 sc->sc_ports[port] = ap;
511
512 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh,
513 AHCI_PORT_REGION(port), AHCI_PORT_SIZE, &ap->ap_ioh) != 0) {
514 printf("%s: unable to create register window for port %d\n",
515 DEVNAME(sc), port);
516 goto freeport;
517 }
518
519 ap->ap_sc = sc;
520 #ifdef AHCI_COALESCE
521 ap->ap_num = port;
522 #endif
523 TAILQ_INIT(&ap->ap_ccb_free);
524 TAILQ_INIT(&ap->ap_ccb_pending);
525 mtx_init(&ap->ap_ccb_mtx, IPL_BIO);
526
527 /* Disable port interrupts */
528 ahci_pwrite(ap, AHCI_PREG_IE, 0);
529
530 /* Sec 10.1.2 - deinitialise port if it is already running */
531 cmd = ahci_pread(ap, AHCI_PREG_CMD);
532 if (ISSET(cmd, (AHCI_PREG_CMD_ST | AHCI_PREG_CMD_CR |
533 AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_FR)) ||
534 ISSET(ahci_pread(ap, AHCI_PREG_SCTL), AHCI_PREG_SCTL_DET)) {
535 int r;
536
537 r = ahci_port_stop(ap, 1);
538 if (r) {
539 printf("%s: unable to disable %s, ignoring port %d\n",
540 DEVNAME(sc), r == 2 ? "CR" : "FR", port);
541 rc = ENXIO;
542 goto freeport;
543 }
544
545 /* Write DET to zero */
546 ahci_pwrite(ap, AHCI_PREG_SCTL, 0);
547 }
548
549 /* Allocate RFIS */
550 ap->ap_dmamem_rfis = ahci_dmamem_alloc(sc, sizeof(struct ahci_rfis));
551 if (ap->ap_dmamem_rfis == NULL)
552 goto nomem;
553
554 /* Setup RFIS base address */
555 ap->ap_rfis = (struct ahci_rfis *) AHCI_DMA_KVA(ap->ap_dmamem_rfis);
556 dva = AHCI_DMA_DVA(ap->ap_dmamem_rfis);
557 ahci_pwrite(ap, AHCI_PREG_FBU, (u_int32_t)(dva >> 32));
558 ahci_pwrite(ap, AHCI_PREG_FB, (u_int32_t)dva);
559
560 /* Enable FIS reception and activate port. */
561 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
562 cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD | AHCI_PREG_CMD_SUD;
563 ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_ICC_ACTIVE);
564
565 /* Check whether port activated. Skip it if not. */
566 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
567 if (!ISSET(cmd, AHCI_PREG_CMD_FRE)) {
568 rc = ENXIO;
569 goto freeport;
570 }
571
572 /* Allocate a CCB for each command slot */
573 ap->ap_ccbs = mallocarray(sc->sc_ncmds, sizeof(struct ahci_ccb),
574 M_DEVBUF, M_NOWAIT | M_ZERO);
575 if (ap->ap_ccbs == NULL) {
576 printf("%s: unable to allocate command list for port %d\n",
577 DEVNAME(sc), port);
578 goto freeport;
579 }
580
581 /* Command List Structures and Command Tables */
582 ap->ap_dmamem_cmd_list = ahci_dmamem_alloc(sc,
583 sc->sc_ncmds * sizeof(struct ahci_cmd_hdr));
584 ap->ap_dmamem_cmd_table = ahci_dmamem_alloc(sc,
585 sc->sc_ncmds * sizeof(struct ahci_cmd_table));
586 if (ap->ap_dmamem_cmd_table == NULL || ap->ap_dmamem_cmd_list == NULL) {
587 nomem:
588 printf("%s: unable to allocate DMA memory for port %d\n",
589 DEVNAME(sc), port);
590 goto freeport;
591 }
592
593 /* Setup command list base address */
594 dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_list);
595 ahci_pwrite(ap, AHCI_PREG_CLBU, (u_int32_t)(dva >> 32));
596 ahci_pwrite(ap, AHCI_PREG_CLB, (u_int32_t)dva);
597
598 /* Split CCB allocation into CCBs and assign to command header/table */
599 hdr = AHCI_DMA_KVA(ap->ap_dmamem_cmd_list);
600 table = AHCI_DMA_KVA(ap->ap_dmamem_cmd_table);
601 for (i = 0; i < sc->sc_ncmds; i++) {
602 ccb = &ap->ap_ccbs[i];
603
604 if (bus_dmamap_create(sc->sc_dmat, MAXPHYS, AHCI_MAX_PRDT,
605 (4 * 1024 * 1024), 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW,
606 &ccb->ccb_dmamap) != 0) {
607 printf("%s: unable to create dmamap for port %d "
608 "ccb %d\n", DEVNAME(sc), port, i);
609 goto freeport;
610 }
611
612 ccb->ccb_slot = i;
613 ccb->ccb_port = ap;
614 ccb->ccb_cmd_hdr = &hdr[i];
615 ccb->ccb_cmd_table = &table[i];
616 htolem64(&ccb->ccb_cmd_hdr->ctba,
617 AHCI_DMA_DVA(ap->ap_dmamem_cmd_table) +
618 ccb->ccb_slot * sizeof(struct ahci_cmd_table));
619
620 ccb->ccb_xa.fis =
621 (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
622 ccb->ccb_xa.packetcmd = ccb->ccb_cmd_table->acmd;
623 ccb->ccb_xa.tag = i;
624
625 ccb->ccb_xa.state = ATA_S_COMPLETE;
626 ahci_put_ccb(ccb);
627 }
628
629 /* grab a ccb for use during error recovery */
630 ap->ap_ccb_err = &ap->ap_ccbs[sc->sc_ncmds - 1];
631 TAILQ_REMOVE(&ap->ap_ccb_free, ap->ap_ccb_err, ccb_entry);
632 ap->ap_ccb_err->ccb_xa.state = ATA_S_COMPLETE;
633
634 /* Wait for ICC change to complete */
635 ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1);
636 rc = 0;
637
638 freeport:
639 if (rc != 0)
640 ahci_port_free(sc, port);
641 reterr:
642 return (rc);
643 }
644
645 void
ahci_port_detect(struct ahci_softc * sc,u_int port)646 ahci_port_detect(struct ahci_softc *sc, u_int port)
647 {
648 struct ahci_port *ap;
649 const char *speed;
650 int rc;
651
652 ap = sc->sc_ports[port];
653
654 rc = ahci_port_portreset_finish(ap, 1);
655 switch (rc) {
656 case ENODEV:
657 switch (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) {
658 case AHCI_PREG_SSTS_DET_DEV_NE:
659 printf("%s: device not communicating on port %d\n",
660 DEVNAME(sc), port);
661 break;
662 case AHCI_PREG_SSTS_DET_PHYOFFLINE:
663 printf("%s: PHY offline on port %d\n", DEVNAME(sc),
664 port);
665 break;
666 default:
667 DPRINTF(AHCI_D_VERBOSE, "%s: no device detected "
668 "on port %d\n", DEVNAME(sc), port);
669 break;
670 }
671 goto freeport;
672
673 case EBUSY:
674 printf("%s: device on port %d didn't come ready, "
675 "TFD: 0x%b\n", DEVNAME(sc), port,
676 ahci_pread(ap, AHCI_PREG_TFD), AHCI_PFMT_TFD_STS);
677
678 /* Try a soft reset to clear busy */
679 rc = ahci_port_softreset(ap);
680 if (rc) {
681 printf("%s: unable to communicate "
682 "with device on port %d\n", DEVNAME(sc), port);
683 goto freeport;
684 }
685 break;
686
687 default:
688 break;
689 }
690
691 DPRINTF(AHCI_D_VERBOSE, "%s: detected device on port %d; %d\n",
692 DEVNAME(sc), port, rc);
693
694 /* Read current link speed */
695 switch(ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_SPD) {
696 case AHCI_PREG_SSTS_SPD_GEN1:
697 speed = "1.5Gb/s";
698 break;
699 case AHCI_PREG_SSTS_SPD_GEN2:
700 speed = "3.0Gb/s";
701 break;
702 case AHCI_PREG_SSTS_SPD_GEN3:
703 speed = "6.0Gb/s";
704 break;
705 default:
706 speed = NULL;
707 break;
708 }
709 if (speed != NULL)
710 printf("%s: port %d: %s\n", PORTNAME(ap), port, speed);
711
712 /* Enable command transfers on port */
713 if (ahci_port_start(ap, 0)) {
714 printf("%s: failed to start command DMA on port %d, "
715 "disabling\n", DEVNAME(sc), port);
716 rc = ENXIO; /* couldn't start port */
717 }
718
719 /* Flush interrupts for port */
720 ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS));
721 ahci_write(sc, AHCI_REG_IS, 1 << port);
722
723 ahci_enable_interrupts(ap);
724 freeport:
725 if (rc != 0)
726 ahci_port_free(sc, port);
727 }
728
729 void
ahci_port_free(struct ahci_softc * sc,u_int port)730 ahci_port_free(struct ahci_softc *sc, u_int port)
731 {
732 struct ahci_port *ap = sc->sc_ports[port];
733 struct ahci_ccb *ccb;
734
735 /* Ensure port is disabled and its interrupts are flushed */
736 if (ap->ap_sc) {
737 ahci_pwrite(ap, AHCI_PREG_CMD, 0);
738 ahci_pwrite(ap, AHCI_PREG_IE, 0);
739 ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS));
740 ahci_write(sc, AHCI_REG_IS, 1 << port);
741 }
742
743 if (ap->ap_ccb_err)
744 ahci_put_ccb(ap->ap_ccb_err);
745
746 if (ap->ap_ccbs) {
747 while ((ccb = ahci_get_ccb(ap)) != NULL)
748 bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap);
749 free(ap->ap_ccbs, M_DEVBUF, sc->sc_ncmds * sizeof(*ccb));
750 }
751
752 if (ap->ap_dmamem_cmd_list)
753 ahci_dmamem_free(sc, ap->ap_dmamem_cmd_list);
754 if (ap->ap_dmamem_rfis)
755 ahci_dmamem_free(sc, ap->ap_dmamem_rfis);
756 if (ap->ap_dmamem_cmd_table)
757 ahci_dmamem_free(sc, ap->ap_dmamem_cmd_table);
758 if (ap->ap_err_scratch)
759 dma_free(ap->ap_err_scratch, DEV_BSIZE);
760
761 /* bus_space(9) says we dont free the subregions handle */
762
763 free(ap, M_DEVBUF, sizeof(*ap));
764 sc->sc_ports[port] = NULL;
765 }
766
767 int
ahci_port_init(struct ahci_softc * sc,u_int port)768 ahci_port_init(struct ahci_softc *sc, u_int port)
769 {
770 struct ahci_port *ap;
771 u_int64_t dva;
772 u_int32_t cmd;
773 int rc = ENOMEM;
774
775 ap = sc->sc_ports[port];
776 #ifdef AHCI_DEBUG
777 snprintf(ap->ap_name, sizeof(ap->ap_name), "%s.%d",
778 DEVNAME(sc), port);
779 #endif
780
781 /* Disable port interrupts */
782 ahci_pwrite(ap, AHCI_PREG_IE, 0);
783
784 /* Sec 10.1.2 - deinitialise port if it is already running */
785 cmd = ahci_pread(ap, AHCI_PREG_CMD);
786 if (ISSET(cmd, (AHCI_PREG_CMD_ST | AHCI_PREG_CMD_CR |
787 AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_FR)) ||
788 ISSET(ahci_pread(ap, AHCI_PREG_SCTL), AHCI_PREG_SCTL_DET)) {
789 int r;
790
791 r = ahci_port_stop(ap, 1);
792 if (r) {
793 printf("%s: unable to disable %s, ignoring port %d\n",
794 DEVNAME(sc), r == 2 ? "CR" : "FR", port);
795 rc = ENXIO;
796 goto reterr;
797 }
798
799 /* Write DET to zero */
800 ahci_pwrite(ap, AHCI_PREG_SCTL, 0);
801 }
802
803 /* Setup RFIS base address */
804 ap->ap_rfis = (struct ahci_rfis *) AHCI_DMA_KVA(ap->ap_dmamem_rfis);
805 dva = AHCI_DMA_DVA(ap->ap_dmamem_rfis);
806 ahci_pwrite(ap, AHCI_PREG_FBU, (u_int32_t)(dva >> 32));
807 ahci_pwrite(ap, AHCI_PREG_FB, (u_int32_t)dva);
808
809 /* Enable FIS reception and activate port. */
810 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
811 cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD | AHCI_PREG_CMD_SUD;
812 ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_ICC_ACTIVE);
813
814 /* Check whether port activated. Skip it if not. */
815 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
816 if (!ISSET(cmd, AHCI_PREG_CMD_FRE)) {
817 rc = ENXIO;
818 goto reterr;
819 }
820
821 /* Setup command list base address */
822 dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_list);
823 ahci_pwrite(ap, AHCI_PREG_CLBU, (u_int32_t)(dva >> 32));
824 ahci_pwrite(ap, AHCI_PREG_CLB, (u_int32_t)dva);
825
826 /* Wait for ICC change to complete */
827 ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1);
828
829 /* Reset port */
830 rc = ahci_port_portreset(ap, 1);
831 switch (rc) {
832 case ENODEV:
833 switch (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) {
834 case AHCI_PREG_SSTS_DET_DEV_NE:
835 printf("%s: device not communicating on port %d\n",
836 DEVNAME(sc), port);
837 break;
838 case AHCI_PREG_SSTS_DET_PHYOFFLINE:
839 printf("%s: PHY offline on port %d\n", DEVNAME(sc),
840 port);
841 break;
842 default:
843 DPRINTF(AHCI_D_VERBOSE, "%s: no device detected "
844 "on port %d\n", DEVNAME(sc), port);
845 break;
846 }
847 goto reterr;
848
849 case EBUSY:
850 printf("%s: device on port %d didn't come ready, "
851 "TFD: 0x%b\n", DEVNAME(sc), port,
852 ahci_pread(ap, AHCI_PREG_TFD), AHCI_PFMT_TFD_STS);
853
854 /* Try a soft reset to clear busy */
855 rc = ahci_port_softreset(ap);
856 if (rc) {
857 printf("%s: unable to communicate "
858 "with device on port %d\n", DEVNAME(sc), port);
859 goto reterr;
860 }
861 break;
862
863 default:
864 break;
865 }
866 DPRINTF(AHCI_D_VERBOSE, "%s: detected device on port %d\n",
867 DEVNAME(sc), port);
868
869 if (ap->ap_pmp_ports > 0) {
870 int p;
871
872 for (p = 0; p < ap->ap_pmp_ports; p++) {
873 int sig;
874
875 /* might need to do a portreset first here? */
876
877 /* softreset the port */
878 if (ahci_pmp_port_softreset(ap, p)) {
879 printf("%s.%d: unable to probe PMP port due to"
880 " softreset failure\n", PORTNAME(ap), p);
881 continue;
882 }
883
884 sig = ahci_port_signature(ap);
885 printf("%s.%d: port signature returned %d\n",
886 PORTNAME(ap), p, sig);
887 }
888 }
889
890 /* Enable command transfers on port */
891 if (ahci_port_start(ap, 0)) {
892 printf("%s: failed to start command DMA on port %d, "
893 "disabling\n", DEVNAME(sc), port);
894 rc = ENXIO; /* couldn't start port */
895 }
896
897 /* Flush interrupts for port */
898 ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS));
899 ahci_write(sc, AHCI_REG_IS, 1 << port);
900
901 ahci_enable_interrupts(ap);
902
903 reterr:
904 return (rc);
905 }
906
907 int
ahci_default_port_start(struct ahci_port * ap,int fre_only)908 ahci_default_port_start(struct ahci_port *ap, int fre_only)
909 {
910 u_int32_t r;
911
912 /* Turn on FRE (and ST) */
913 r = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
914 r |= AHCI_PREG_CMD_FRE;
915 if (!fre_only)
916 r |= AHCI_PREG_CMD_ST;
917 ahci_pwrite(ap, AHCI_PREG_CMD, r);
918
919 #ifdef AHCI_COALESCE
920 /* (Re-)enable coalescing on the port. */
921 if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) {
922 ap->ap_sc->sc_ccc_ports_cur |= (1 << ap->ap_num);
923 ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS,
924 ap->ap_sc->sc_ccc_ports_cur);
925 }
926 #endif
927
928 return (0);
929 }
930
931 int
ahci_port_stop(struct ahci_port * ap,int stop_fis_rx)932 ahci_port_stop(struct ahci_port *ap, int stop_fis_rx)
933 {
934 u_int32_t r;
935
936 #ifdef AHCI_COALESCE
937 /* Disable coalescing on the port while it is stopped. */
938 if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) {
939 ap->ap_sc->sc_ccc_ports_cur &= ~(1 << ap->ap_num);
940 ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS,
941 ap->ap_sc->sc_ccc_ports_cur);
942 }
943 #endif
944
945 /* Turn off ST (and FRE) */
946 r = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
947 r &= ~AHCI_PREG_CMD_ST;
948 if (stop_fis_rx)
949 r &= ~AHCI_PREG_CMD_FRE;
950 ahci_pwrite(ap, AHCI_PREG_CMD, r);
951
952 /* Wait for CR to go off */
953 if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CR, 1))
954 return (1);
955
956 /* Wait for FR to go off */
957 if (stop_fis_rx &&
958 ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_FR, 1))
959 return (2);
960
961 return (0);
962 }
963
964 /* AHCI command list override -> forcibly clear TFD.STS.{BSY,DRQ} */
965 int
ahci_port_clo(struct ahci_port * ap)966 ahci_port_clo(struct ahci_port *ap)
967 {
968 struct ahci_softc *sc = ap->ap_sc;
969 u_int32_t cmd;
970
971 /* Only attempt CLO if supported by controller */
972 if (!ISSET(ahci_read(sc, AHCI_REG_CAP), AHCI_REG_CAP_SCLO))
973 return (1);
974
975 /* Issue CLO */
976 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
977 #ifdef DIAGNOSTIC
978 if (ISSET(cmd, AHCI_PREG_CMD_ST))
979 printf("%s: CLO requested while port running\n", PORTNAME(ap));
980 #endif
981 ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_CLO);
982
983 /* Wait for completion */
984 if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CLO, 1)) {
985 printf("%s: CLO did not complete\n", PORTNAME(ap));
986 return (1);
987 }
988
989 return (0);
990 }
991
992 /* AHCI soft reset, Section 10.4.1 */
993 int
ahci_port_softreset(struct ahci_port * ap)994 ahci_port_softreset(struct ahci_port *ap)
995 {
996 struct ahci_ccb *ccb = NULL;
997 struct ahci_cmd_hdr *cmd_slot;
998 u_int8_t *fis;
999 int s, rc = EIO, oldstate;
1000 u_int32_t cmd;
1001
1002 DPRINTF(AHCI_D_VERBOSE, "%s: soft reset\n", PORTNAME(ap));
1003
1004 s = splbio();
1005 oldstate = ap->ap_state;
1006 ap->ap_state = AP_S_ERROR_RECOVERY;
1007
1008 /* Save previous command register state */
1009 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
1010
1011 /* Idle port */
1012 if (ahci_port_stop(ap, 0)) {
1013 printf("%s: failed to stop port, cannot softreset\n",
1014 PORTNAME(ap));
1015 goto err;
1016 }
1017
1018 /* Request CLO if device appears hung */
1019 if (ISSET(ahci_pread(ap, AHCI_PREG_TFD), AHCI_PREG_TFD_STS_BSY |
1020 AHCI_PREG_TFD_STS_DRQ))
1021 ahci_port_clo(ap);
1022
1023 /* Clear port errors to permit TFD transfer */
1024 ahci_pwrite(ap, AHCI_PREG_SERR, ahci_pread(ap, AHCI_PREG_SERR));
1025
1026 /* Restart port */
1027 if (ahci_port_start(ap, 0)) {
1028 printf("%s: failed to start port, cannot softreset\n",
1029 PORTNAME(ap));
1030 goto err;
1031 }
1032
1033 /* Check whether CLO worked */
1034 if (ahci_pwait_clr(ap, AHCI_PREG_TFD,
1035 AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)) {
1036 printf("%s: CLO %s, need port reset\n", PORTNAME(ap),
1037 ISSET(ahci_read(ap->ap_sc, AHCI_REG_CAP), AHCI_REG_CAP_SCLO)
1038 ? "failed" : "unsupported");
1039 rc = EBUSY;
1040 goto err;
1041 }
1042
1043 /* Prep first D2H command with SRST feature & clear busy/reset flags */
1044 ccb = ahci_get_err_ccb(ap);
1045 cmd_slot = ccb->ccb_cmd_hdr;
1046 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table));
1047
1048 fis = ccb->ccb_cmd_table->cfis;
1049 fis[0] = ATA_FIS_TYPE_H2D;
1050 fis[15] = ATA_FIS_CONTROL_SRST;
1051
1052 cmd_slot->prdtl = 0;
1053 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ |
1054 AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R |
1055 AHCI_CMD_LIST_FLAG_W);
1056
1057 ccb->ccb_xa.state = ATA_S_PENDING;
1058 if (ahci_poll(ccb, 1000, NULL) != 0)
1059 goto err;
1060
1061 /* Prep second D2H command to read status and complete reset sequence */
1062 fis[0] = ATA_FIS_TYPE_H2D;
1063 fis[15] = 0;
1064
1065 cmd_slot->prdtl = 0;
1066 htolem16(&cmd_slot->flags, 5 | AHCI_CMD_LIST_FLAG_W);
1067
1068 ccb->ccb_xa.state = ATA_S_PENDING;
1069 if (ahci_poll(ccb, 1000, NULL) != 0)
1070 goto err;
1071
1072 if (ahci_pwait_clr(ap, AHCI_PREG_TFD, AHCI_PREG_TFD_STS_BSY |
1073 AHCI_PREG_TFD_STS_DRQ | AHCI_PREG_TFD_STS_ERR, 1)) {
1074 printf("%s: device didn't come ready after reset, TFD: 0x%b\n",
1075 PORTNAME(ap), ahci_pread(ap, AHCI_PREG_TFD),
1076 AHCI_PFMT_TFD_STS);
1077 rc = EBUSY;
1078 goto err;
1079 }
1080
1081 rc = 0;
1082 err:
1083 if (ccb != NULL) {
1084 /* Abort our command, if it failed, by stopping command DMA. */
1085 if (rc != 0 && ISSET(ap->ap_active, 1 << ccb->ccb_slot)) {
1086 printf("%s: stopping the port, softreset slot %d was "
1087 "still active.\n", PORTNAME(ap), ccb->ccb_slot);
1088 ahci_port_stop(ap, 0);
1089 }
1090 ccb->ccb_xa.state = ATA_S_ERROR;
1091 ahci_put_err_ccb(ccb);
1092 }
1093
1094 /* Restore saved CMD register state */
1095 ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
1096 ap->ap_state = oldstate;
1097
1098 splx(s);
1099
1100 return (rc);
1101 }
1102
1103 int
ahci_pmp_port_softreset(struct ahci_port * ap,int pmp_port)1104 ahci_pmp_port_softreset(struct ahci_port *ap, int pmp_port)
1105 {
1106 struct ahci_ccb *ccb = NULL;
1107 u_int32_t data;
1108 int count;
1109 int rc;
1110 int s;
1111 struct ahci_cmd_hdr *cmd_slot;
1112 u_int8_t *fis;
1113
1114 s = splbio();
1115 /* ignore spurious IFS errors while resetting */
1116 DPRINTF(AHCI_D_VERBOSE, "%s: now ignoring IFS\n", PORTNAME(ap));
1117 ap->ap_pmp_ignore_ifs = 1;
1118
1119 count = 2;
1120 rc = 0;
1121 do {
1122 if (ccb != NULL) {
1123 ahci_put_pmp_ccb(ccb);
1124 ccb = NULL;
1125 }
1126
1127 if (ahci_pmp_phy_status(ap, pmp_port, &data)) {
1128 printf("%s.%d: unable to clear PHY status\n",
1129 PORTNAME(ap), pmp_port);
1130 }
1131 ahci_pwrite(ap, AHCI_PREG_SERR, -1);
1132 /* maybe don't do this on the first loop: */
1133 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS);
1134 ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1);
1135
1136 /* send first softreset FIS */
1137 ccb = ahci_get_pmp_ccb(ap); /* Always returns non-NULL. */
1138 cmd_slot = ccb->ccb_cmd_hdr;
1139 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table));
1140
1141 fis = ccb->ccb_cmd_table->cfis;
1142 fis[0] = ATA_FIS_TYPE_H2D;
1143 fis[1] = pmp_port;
1144 fis[15] = ATA_FIS_CONTROL_SRST | ATA_FIS_CONTROL_4BIT;
1145
1146 cmd_slot->prdtl = 0;
1147 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ |
1148 AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R |
1149 (pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT));
1150
1151 ccb->ccb_xa.state = ATA_S_PENDING;
1152
1153 DPRINTF(AHCI_D_VERBOSE, "%s.%d: sending PMP softreset cmd\n",
1154 PORTNAME(ap), pmp_port);
1155 if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) {
1156 printf("%s.%d: PMP port softreset cmd failed\n",
1157 PORTNAME(ap), pmp_port);
1158 rc = EBUSY;
1159 if (count > 0) {
1160 /* probably delay a while to allow
1161 * it to settle down?
1162 */
1163 }
1164 continue;
1165 }
1166
1167 /* send signature FIS */
1168 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table));
1169 fis[0] = ATA_FIS_TYPE_H2D;
1170 fis[1] = pmp_port;
1171 fis[15] = ATA_FIS_CONTROL_4BIT;
1172
1173 cmd_slot->prdtl = 0;
1174 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ |
1175 (pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT));
1176
1177 DPRINTF(AHCI_D_VERBOSE, "%s.%d: sending PMP probe status cmd\n",
1178 PORTNAME(ap), pmp_port);
1179 ccb->ccb_xa.state = ATA_S_PENDING;
1180 if (ahci_poll(ccb, 5000, ahci_pmp_probe_timeout) != 0) {
1181 DPRINTF(AHCI_D_VERBOSE, "%s.%d: PMP probe status cmd "
1182 "failed\n", PORTNAME(ap), pmp_port);
1183 rc = EBUSY;
1184 if (count > 0) {
1185 /* sleep a while? */
1186 }
1187 continue;
1188 }
1189
1190 fis[15] = 0;
1191 break;
1192 } while (count--);
1193
1194 if (ccb != NULL) {
1195 ahci_put_pmp_ccb(ccb);
1196 ccb = NULL;
1197 }
1198
1199 /* clean up a bit */
1200 ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1);
1201 ahci_pwrite(ap, AHCI_PREG_SERR, -1);
1202 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS);
1203 ap->ap_pmp_ignore_ifs = 0;
1204 DPRINTF(AHCI_D_VERBOSE, "%s: no longer ignoring IFS\n", PORTNAME(ap));
1205 splx(s);
1206
1207 return (rc);
1208 }
1209
1210 int
ahci_pmp_port_probe(struct ahci_port * ap,int pmp_port)1211 ahci_pmp_port_probe(struct ahci_port *ap, int pmp_port)
1212 {
1213 int sig;
1214
1215 ap->ap_state = AP_S_PMP_PORT_PROBE;
1216
1217 DPRINTF(AHCI_D_VERBOSE, "%s.%d: probing pmp port\n", PORTNAME(ap),
1218 pmp_port);
1219 if (ahci_pmp_port_portreset(ap, pmp_port)) {
1220 printf("%s.%d: unable to probe PMP port; portreset failed\n",
1221 PORTNAME(ap), pmp_port);
1222 ap->ap_state = AP_S_NORMAL;
1223 return (ATA_PORT_T_NONE);
1224 }
1225
1226 if (ahci_pmp_port_softreset(ap, pmp_port)) {
1227 printf("%s.%d: unable to probe PMP port due to softreset "
1228 "failure\n", PORTNAME(ap), pmp_port);
1229 ap->ap_state = AP_S_NORMAL;
1230 return (ATA_PORT_T_NONE);
1231 }
1232
1233 sig = ahci_port_signature(ap);
1234 DPRINTF(AHCI_D_VERBOSE, "%s.%d: port signature returned %d\n",
1235 PORTNAME(ap), pmp_port, sig);
1236 ap->ap_state = AP_S_NORMAL;
1237 return (sig);
1238 }
1239
1240
1241 void
ahci_flush_tfd(struct ahci_port * ap)1242 ahci_flush_tfd(struct ahci_port *ap)
1243 {
1244 u_int32_t r;
1245
1246 r = ahci_pread(ap, AHCI_PREG_SERR);
1247 if (r & AHCI_PREG_SERR_DIAG_X)
1248 ahci_pwrite(ap, AHCI_PREG_SERR, AHCI_PREG_SERR_DIAG_X);
1249 }
1250
1251 u_int32_t
ahci_active_mask(struct ahci_port * ap)1252 ahci_active_mask(struct ahci_port *ap)
1253 {
1254 u_int32_t mask;
1255
1256 mask = ahci_pread(ap, AHCI_PREG_CI);
1257 if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SNCQ)
1258 mask |= ahci_pread(ap, AHCI_PREG_SACT);
1259 return mask;
1260 }
1261
1262 void
ahci_pmp_probe_timeout(void * cookie)1263 ahci_pmp_probe_timeout(void *cookie)
1264 {
1265 struct ahci_ccb *ccb = cookie;
1266 struct ahci_port *ap = ccb->ccb_port;
1267 u_int32_t mask;
1268
1269 DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe cmd timed out\n", PORTNAME(ap));
1270 switch (ccb->ccb_xa.state) {
1271 case ATA_S_PENDING:
1272 TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry);
1273 ccb->ccb_xa.state = ATA_S_TIMEOUT;
1274 break;
1275
1276 case ATA_S_ONCHIP:
1277 case ATA_S_ERROR: /* currently mostly here for the ATI SBx00 quirk */
1278 /* clear the command on-chip */
1279 KASSERT(ap->ap_active == (1 << ccb->ccb_slot) &&
1280 ap->ap_sactive == 0);
1281 ahci_port_stop(ap, 0);
1282 ahci_port_start(ap, 0);
1283
1284 if (ahci_active_mask(ap) != 0) {
1285 ahci_port_stop(ap, 0);
1286 ahci_port_start(ap, 0);
1287 mask = ahci_active_mask(ap);
1288 if (mask != 0) {
1289 printf("%s: ahci_pmp_probe_timeout: failed to "
1290 "clear active cmds: %08x\n", PORTNAME(ap),
1291 mask);
1292 }
1293 }
1294
1295 ccb->ccb_xa.state = ATA_S_TIMEOUT;
1296 ap->ap_active &= ~(1 << ccb->ccb_slot);
1297 KASSERT(ap->ap_active_cnt > 0);
1298 --ap->ap_active_cnt;
1299 DPRINTF(AHCI_D_VERBOSE, "%s: timed out %d, active %x, "
1300 "active_cnt %d\n", PORTNAME(ap), ccb->ccb_slot,
1301 ap->ap_active, ap->ap_active_cnt);
1302 break;
1303
1304 default:
1305 panic("%s: ahci_pmp_probe_timeout: ccb in bad state %d",
1306 PORTNAME(ap), ccb->ccb_xa.state);
1307 }
1308 }
1309
1310 int
ahci_port_signature(struct ahci_port * ap)1311 ahci_port_signature(struct ahci_port *ap)
1312 {
1313 u_int32_t sig;
1314
1315 sig = ahci_pread(ap, AHCI_PREG_SIG);
1316 if ((sig & 0xffff0000) == (SATA_SIGNATURE_ATAPI & 0xffff0000))
1317 return (ATA_PORT_T_ATAPI);
1318 else if ((sig & 0xffff0000) == (SATA_SIGNATURE_PORT_MULTIPLIER &
1319 0xffff0000))
1320 return (ATA_PORT_T_PM);
1321 else
1322 return (ATA_PORT_T_DISK);
1323 }
1324
1325 int
ahci_pmp_port_portreset(struct ahci_port * ap,int pmp_port)1326 ahci_pmp_port_portreset(struct ahci_port *ap, int pmp_port)
1327 {
1328 u_int32_t cmd, data;
1329 int loop;
1330 int rc = 1;
1331 int s;
1332
1333 s = splbio();
1334 DPRINTF(AHCI_D_VERBOSE, "%s.%d: PMP port reset\n", PORTNAME(ap),
1335 pmp_port);
1336
1337 /* Save previous command register state */
1338 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
1339
1340 /* turn off power management and disable the PHY */
1341 data = AHCI_PREG_SCTL_IPM_DISABLED;
1342 /* maybe add AHCI_PREG_SCTL_DET_DISABLE */
1343 if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1))
1344 goto err;
1345 if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL, data))
1346 goto err;
1347 delay(10000);
1348
1349 /* start COMRESET */
1350 data = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_INIT;
1351 if ((ap->ap_sc->sc_dev.dv_cfdata->cf_flags & 0x01) != 0) {
1352 DPRINTF(AHCI_D_VERBOSE, "%s.%d: forcing GEN1\n", PORTNAME(ap),
1353 pmp_port);
1354 data |= AHCI_PREG_SCTL_SPD_GEN1;
1355 } else
1356 data |= AHCI_PREG_SCTL_SPD_ANY;
1357
1358 if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL, data))
1359 goto err;
1360
1361 /* give it a while to settle down */
1362 delay(100000);
1363
1364 if (ahci_pmp_phy_status(ap, pmp_port, &data)) {
1365 printf("%s.%d: cannot clear PHY status\n", PORTNAME(ap),
1366 pmp_port);
1367 }
1368
1369 /* start trying to negotiate */
1370 ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1);
1371 data = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_NONE;
1372 if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL, data))
1373 goto err;
1374
1375 /* give it a while to detect */
1376 for (loop = 3; loop; --loop) {
1377 if (ahci_pmp_read(ap, pmp_port, SATA_PMREG_SSTS, &data))
1378 goto err;
1379 if (data & AHCI_PREG_SSTS_DET)
1380 break;
1381 delay(100000);
1382 }
1383 if (loop == 0) {
1384 printf("%s.%d: port is unplugged\n", PORTNAME(ap), pmp_port);
1385 goto err;
1386 }
1387
1388 /* give it even longer to fully negotiate */
1389 for (loop = 30; loop; --loop) {
1390 if (ahci_pmp_read(ap, pmp_port, SATA_PMREG_SSTS, &data))
1391 goto err;
1392 if ((data & AHCI_PREG_SSTS_DET) == AHCI_PREG_SSTS_DET_DEV)
1393 break;
1394 delay(100000);
1395 }
1396
1397 if (loop == 0) {
1398 printf("%s.%d: device is not negotiating\n", PORTNAME(ap),
1399 pmp_port);
1400 goto err;
1401 }
1402
1403 /* device detected */
1404 DPRINTF(AHCI_D_VERBOSE, "%s.%d: device detected\n", PORTNAME(ap),
1405 pmp_port);
1406
1407 /* clean up a bit */
1408 delay(100000);
1409 ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1);
1410 ahci_pwrite(ap, AHCI_PREG_SERR, -1);
1411 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS);
1412
1413 rc = 0;
1414 err:
1415 /* Restore preserved port state */
1416 ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
1417 splx(s);
1418 return (rc);
1419 }
1420
1421 /* AHCI port reset, Section 10.4.2 */
1422
1423 void
ahci_port_comreset(struct ahci_port * ap)1424 ahci_port_comreset(struct ahci_port *ap)
1425 {
1426 u_int32_t r;
1427
1428 r = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_INIT;
1429 if ((ap->ap_sc->sc_dev.dv_cfdata->cf_flags & 0x01) != 0) {
1430 DPRINTF(AHCI_D_VERBOSE, "%s: forcing GEN1\n", PORTNAME(ap));
1431 r |= AHCI_PREG_SCTL_SPD_GEN1;
1432 } else
1433 r |= AHCI_PREG_SCTL_SPD_ANY;
1434 ahci_pwrite(ap, AHCI_PREG_SCTL, r);
1435 delay(10000); /* wait at least 1ms for COMRESET to be sent */
1436 r &= ~AHCI_PREG_SCTL_DET_INIT;
1437 r |= AHCI_PREG_SCTL_DET_NONE;
1438 ahci_pwrite(ap, AHCI_PREG_SCTL, r);
1439 delay(10000);
1440 }
1441
1442 void
ahci_port_portreset_start(struct ahci_port * ap)1443 ahci_port_portreset_start(struct ahci_port *ap)
1444 {
1445 int s;
1446
1447 s = splbio();
1448 DPRINTF(AHCI_D_VERBOSE, "%s: port reset\n", PORTNAME(ap));
1449
1450 /* Save previous command register state */
1451 ap->ap_saved_cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
1452
1453 /* Clear ST, ignoring failure */
1454 ahci_port_stop(ap, 0);
1455
1456 /* Perform device detection */
1457 ahci_pwrite(ap, AHCI_PREG_SCTL, 0);
1458 delay(10000);
1459 ahci_port_comreset(ap);
1460 splx(s);
1461 }
1462
1463 int
ahci_port_portreset_poll(struct ahci_port * ap)1464 ahci_port_portreset_poll(struct ahci_port *ap)
1465 {
1466 if ((ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) !=
1467 AHCI_PREG_SSTS_DET_DEV)
1468 return (EAGAIN);
1469 return (0);
1470 }
1471
1472 void
ahci_port_portreset_wait(struct ahci_port * ap)1473 ahci_port_portreset_wait(struct ahci_port *ap)
1474 {
1475 int i;
1476
1477 for (i = 0; i < 1000; i++) {
1478 if (ahci_port_portreset_poll(ap) == 0)
1479 break;
1480 delay(1000);
1481 }
1482 }
1483
1484 int
ahci_port_portreset_finish(struct ahci_port * ap,int pmp)1485 ahci_port_portreset_finish(struct ahci_port *ap, int pmp)
1486 {
1487 int rc, s, retries = 0;
1488
1489 s = splbio();
1490 retry:
1491 if (ahci_port_portreset_poll(ap)) {
1492 rc = ENODEV;
1493 if (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) {
1494 /* this may be a port multiplier with no device
1495 * on port 0, so still do the pmp check if requested.
1496 */
1497 } else {
1498 goto err;
1499 }
1500 } else {
1501 /* Clear SERR (incl X bit), so TFD can update */
1502 ahci_pwrite(ap, AHCI_PREG_SERR, ahci_pread(ap, AHCI_PREG_SERR));
1503
1504 /* Wait for device to become ready */
1505 if (ahci_pwait_clr(ap, AHCI_PREG_TFD, AHCI_PREG_TFD_STS_BSY |
1506 AHCI_PREG_TFD_STS_DRQ | AHCI_PREG_TFD_STS_ERR, 3)) {
1507 /* even if the device doesn't wake up, check if there's
1508 * a port multiplier there
1509 */
1510 if (retries == 0) {
1511 retries = 1;
1512 ahci_port_comreset(ap);
1513 ahci_port_portreset_wait(ap);
1514 goto retry;
1515 }
1516 rc = EBUSY;
1517 } else {
1518 rc = 0;
1519 }
1520 }
1521
1522 if (pmp != 0) {
1523 if (ahci_port_detect_pmp(ap)) {
1524 /* reset again without pmp support */
1525 pmp = 0;
1526 retries = 0;
1527 ahci_port_comreset(ap);
1528 ahci_port_portreset_wait(ap);
1529 goto retry;
1530 }
1531 }
1532
1533 err:
1534 /* Restore preserved port state */
1535 ahci_pwrite(ap, AHCI_PREG_CMD, ap->ap_saved_cmd);
1536 ap->ap_saved_cmd = 0;
1537 splx(s);
1538
1539 return (rc);
1540 }
1541
1542 int
ahci_port_portreset(struct ahci_port * ap,int pmp)1543 ahci_port_portreset(struct ahci_port *ap, int pmp)
1544 {
1545 ahci_port_portreset_start(ap);
1546 ahci_port_portreset_wait(ap);
1547 return (ahci_port_portreset_finish(ap, pmp));
1548 }
1549
1550 int
ahci_port_detect_pmp(struct ahci_port * ap)1551 ahci_port_detect_pmp(struct ahci_port *ap)
1552 {
1553 int count, pmp_rc, rc;
1554 u_int32_t r, cmd;
1555 struct ahci_cmd_hdr *cmd_slot;
1556 struct ahci_ccb *ccb = NULL;
1557 u_int8_t *fis = NULL;
1558
1559 if ((ap->ap_sc->sc_flags & AHCI_F_NO_PMP) ||
1560 !ISSET(ahci_read(ap->ap_sc, AHCI_REG_CAP), AHCI_REG_CAP_SPM)) {
1561 return 0;
1562 }
1563
1564 rc = 0;
1565 pmp_rc = 0;
1566 count = 2;
1567 do {
1568 DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe %d\n", PORTNAME(ap),
1569 count);
1570 if (ccb != NULL) {
1571 ahci_put_pmp_ccb(ccb);
1572 ccb = NULL;
1573 }
1574 ahci_port_stop(ap, 0);
1575 ap->ap_state = AP_S_PMP_PROBE;
1576
1577 /* set PMA in cmd reg */
1578 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
1579 if ((cmd & AHCI_PREG_CMD_PMA) == 0) {
1580 cmd |= AHCI_PREG_CMD_PMA;
1581 ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
1582 }
1583
1584 /* Flush errors and request CLO unconditionally,
1585 * then start the port
1586 */
1587 r = ahci_pread(ap, AHCI_PREG_SERR);
1588 if (r & AHCI_PREG_SERR_DIAG_X)
1589 ahci_pwrite(ap, AHCI_PREG_SERR,
1590 AHCI_PREG_SERR_DIAG_X);
1591
1592 /* Request CLO */
1593 ahci_port_clo(ap);
1594
1595 /* Clear port errors to permit TFD transfer */
1596 r = ahci_pread(ap, AHCI_PREG_SERR);
1597 ahci_pwrite(ap, AHCI_PREG_SERR, r);
1598
1599 /* Restart port */
1600 if (ahci_port_start(ap, 0)) {
1601 rc = EBUSY;
1602 printf("%s: failed to start port, cannot probe PMP\n",
1603 PORTNAME(ap));
1604 break;
1605 }
1606
1607 /* Check whether CLO worked */
1608 if (ahci_pwait_clr(ap, AHCI_PREG_TFD,
1609 AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)) {
1610 u_int32_t cap;
1611
1612 cap = ahci_read(ap->ap_sc, AHCI_REG_CAP);
1613 printf("%s: CLO %s, need port reset\n",
1614 PORTNAME(ap),
1615 ISSET(cap, AHCI_REG_CAP_SCLO)
1616 ? "failed" : "unsupported");
1617 pmp_rc = EBUSY;
1618 break;
1619 }
1620
1621 /* Prep first command with SRST feature &
1622 * clear busy/reset flags
1623 */
1624 ccb = ahci_get_pmp_ccb(ap); /* Always returns non-NULL. */
1625 cmd_slot = ccb->ccb_cmd_hdr;
1626 memset(ccb->ccb_cmd_table, 0,
1627 sizeof(struct ahci_cmd_table));
1628
1629 fis = ccb->ccb_cmd_table->cfis;
1630 fis[0] = ATA_FIS_TYPE_H2D;
1631 fis[1] = SATA_PMP_CONTROL_PORT;
1632 fis[15] = ATA_FIS_CONTROL_SRST | ATA_FIS_CONTROL_4BIT;
1633
1634 cmd_slot->prdtl = 0;
1635 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ |
1636 AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R |
1637 AHCI_CMD_LIST_FLAG_PMP);
1638
1639 DPRINTF(AHCI_D_VERBOSE, "%s: sending PMP reset cmd\n",
1640 PORTNAME(ap));
1641 ccb->ccb_xa.state = ATA_S_PENDING;
1642 if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) {
1643 DPRINTF(AHCI_D_VERBOSE, "%s: PMP reset cmd failed\n",
1644 PORTNAME(ap));
1645 pmp_rc = EBUSY;
1646 continue;
1647 }
1648
1649 if (ahci_pwait_clr(ap, AHCI_PREG_TFD,
1650 AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)) {
1651 printf("%s: port busy after first PMP probe FIS\n",
1652 PORTNAME(ap));
1653 }
1654
1655 /* clear errors in case the device
1656 * didn't reset cleanly
1657 */
1658 ahci_flush_tfd(ap);
1659 r = ahci_pread(ap, AHCI_PREG_SERR);
1660 ahci_pwrite(ap, AHCI_PREG_SERR, r);
1661
1662 /* Prep second command to read status and
1663 * complete reset sequence
1664 */
1665 memset(ccb->ccb_cmd_table, 0,
1666 sizeof(struct ahci_cmd_table));
1667 fis[0] = ATA_FIS_TYPE_H2D;
1668 fis[1] = SATA_PMP_CONTROL_PORT;
1669 fis[15] = ATA_FIS_CONTROL_4BIT;
1670
1671 cmd_slot->prdtl = 0;
1672 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ |
1673 AHCI_CMD_LIST_FLAG_PMP);
1674
1675 DPRINTF(AHCI_D_VERBOSE, "%s: sending PMP probe status cmd\n",
1676 PORTNAME(ap));
1677 ccb->ccb_xa.state = ATA_S_PENDING;
1678 if (ahci_poll(ccb, 5000, ahci_pmp_probe_timeout) != 0) {
1679 DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe status "
1680 "cmd failed\n", PORTNAME(ap));
1681 pmp_rc = EBUSY;
1682 continue;
1683 }
1684
1685 /* apparently we need to retry at least once
1686 * to get the right signature
1687 */
1688 fis[15] = 0;
1689 pmp_rc = 0;
1690 } while (--count);
1691
1692 if (ccb != NULL) {
1693 ahci_put_pmp_ccb(ccb);
1694 ccb = NULL;
1695 }
1696
1697 if (ap->ap_state == AP_S_PMP_PROBE) {
1698 ap->ap_state = AP_S_NORMAL;
1699 }
1700
1701 if (pmp_rc == 0) {
1702 if (ahci_port_signature(ap) != ATA_PORT_T_PM) {
1703 DPRINTF(AHCI_D_VERBOSE, "%s: device is not a PMP\n",
1704 PORTNAME(ap));
1705 pmp_rc = EBUSY;
1706 } else {
1707 DPRINTF(AHCI_D_VERBOSE, "%s: PMP found\n",
1708 PORTNAME(ap));
1709 }
1710 }
1711
1712 if (pmp_rc == 0) {
1713 if (ahci_pmp_identify(ap, &ap->ap_pmp_ports)) {
1714 pmp_rc = EBUSY;
1715 } else {
1716 rc = 0;
1717 }
1718 }
1719
1720 /* if PMP detection failed, so turn off the PMA bit and
1721 * reset the port again
1722 */
1723 if (pmp_rc != 0) {
1724 DPRINTF(AHCI_D_VERBOSE, "%s: no PMP found, resetting "
1725 "the port\n", PORTNAME(ap));
1726 ahci_port_stop(ap, 0);
1727 ahci_port_clo(ap);
1728 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
1729 cmd &= ~AHCI_PREG_CMD_PMA;
1730 ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
1731
1732 ahci_pwrite(ap, AHCI_PREG_IE, 0);
1733 ahci_port_stop(ap, 0);
1734 if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF)
1735 ahci_pwrite(ap, AHCI_PREG_SNTF, -1);
1736 ahci_flush_tfd(ap);
1737 ahci_pwrite(ap, AHCI_PREG_SERR, -1);
1738
1739 ahci_pwrite(ap, AHCI_PREG_IS, -1);
1740
1741 ahci_enable_interrupts(ap);
1742
1743 rc = pmp_rc;
1744 }
1745
1746 return (rc);
1747 }
1748
1749 void
ahci_load_prdt_seg(struct ahci_prdt * prd,u_int64_t addr,u_int32_t len,u_int32_t flags)1750 ahci_load_prdt_seg(struct ahci_prdt *prd, u_int64_t addr, u_int32_t len,
1751 u_int32_t flags)
1752 {
1753 flags |= len - 1;
1754
1755 htolem64(&prd->dba, addr);
1756 htolem32(&prd->flags, flags);
1757 }
1758
1759 int
ahci_load_prdt(struct ahci_ccb * ccb)1760 ahci_load_prdt(struct ahci_ccb *ccb)
1761 {
1762 struct ahci_port *ap = ccb->ccb_port;
1763 struct ahci_softc *sc = ap->ap_sc;
1764 struct ata_xfer *xa = &ccb->ccb_xa;
1765 struct ahci_prdt *prdt = ccb->ccb_cmd_table->prdt;
1766 bus_dmamap_t dmap = ccb->ccb_dmamap;
1767 struct ahci_cmd_hdr *cmd_slot = ccb->ccb_cmd_hdr;
1768 int i, error;
1769
1770 if (xa->datalen == 0) {
1771 ccb->ccb_cmd_hdr->prdtl = 0;
1772 return (0);
1773 }
1774
1775 error = bus_dmamap_load(sc->sc_dmat, dmap, xa->data, xa->datalen, NULL,
1776 (xa->flags & ATA_F_NOWAIT) ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK);
1777 if (error != 0) {
1778 printf("%s: error %d loading dmamap\n", PORTNAME(ap), error);
1779 return (1);
1780 }
1781
1782 for (i = 0; i < dmap->dm_nsegs - 1; i++) {
1783 ahci_load_prdt_seg(&prdt[i], dmap->dm_segs[i].ds_addr,
1784 dmap->dm_segs[i].ds_len, 0);
1785 }
1786
1787 ahci_load_prdt_seg(&prdt[i],
1788 dmap->dm_segs[i].ds_addr, dmap->dm_segs[i].ds_len,
1789 ISSET(xa->flags, ATA_F_PIO) ? AHCI_PRDT_FLAG_INTR : 0);
1790
1791 htolem16(&cmd_slot->prdtl, dmap->dm_nsegs);
1792
1793 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
1794 (xa->flags & ATA_F_READ) ? BUS_DMASYNC_PREREAD :
1795 BUS_DMASYNC_PREWRITE);
1796
1797 return (0);
1798 }
1799
1800 void
ahci_unload_prdt(struct ahci_ccb * ccb)1801 ahci_unload_prdt(struct ahci_ccb *ccb)
1802 {
1803 struct ahci_port *ap = ccb->ccb_port;
1804 struct ahci_softc *sc = ap->ap_sc;
1805 struct ata_xfer *xa = &ccb->ccb_xa;
1806 bus_dmamap_t dmap = ccb->ccb_dmamap;
1807
1808 if (xa->datalen != 0) {
1809 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
1810 (xa->flags & ATA_F_READ) ? BUS_DMASYNC_POSTREAD :
1811 BUS_DMASYNC_POSTWRITE);
1812
1813 bus_dmamap_unload(sc->sc_dmat, dmap);
1814
1815 if (ccb->ccb_xa.flags & ATA_F_NCQ)
1816 xa->resid = 0;
1817 else
1818 xa->resid = xa->datalen -
1819 lemtoh32(&ccb->ccb_cmd_hdr->prdbc);
1820 }
1821 }
1822
1823 int
ahci_poll(struct ahci_ccb * ccb,int timeout,void (* timeout_fn)(void *))1824 ahci_poll(struct ahci_ccb *ccb, int timeout, void (*timeout_fn)(void *))
1825 {
1826 struct ahci_port *ap = ccb->ccb_port;
1827 int s;
1828
1829 s = splbio();
1830 ahci_start(ccb);
1831 do {
1832 if (ISSET(ahci_port_intr(ap, AHCI_PREG_CI_ALL_SLOTS),
1833 1 << ccb->ccb_slot)) {
1834 splx(s);
1835 return (0);
1836 }
1837 if (ccb->ccb_xa.state == ATA_S_ERROR) {
1838 DPRINTF(AHCI_D_VERBOSE, "%s: ccb in slot %d errored\n",
1839 PORTNAME(ap), ccb->ccb_slot);
1840 /* pretend it timed out? */
1841 if (timeout_fn != NULL) {
1842 timeout_fn(ccb);
1843 }
1844 splx(s);
1845 return (1);
1846 }
1847
1848 delay(1000);
1849 } while (--timeout > 0);
1850
1851 /* Run timeout while at splbio, otherwise ahci_intr could interfere. */
1852 if (timeout_fn != NULL)
1853 timeout_fn(ccb);
1854
1855 splx(s);
1856
1857 return (1);
1858 }
1859
1860 void
ahci_start(struct ahci_ccb * ccb)1861 ahci_start(struct ahci_ccb *ccb)
1862 {
1863 struct ahci_port *ap = ccb->ccb_port;
1864 struct ahci_softc *sc = ap->ap_sc;
1865
1866 /* Zero transferred byte count before transfer */
1867 ccb->ccb_cmd_hdr->prdbc = 0;
1868
1869 /* Sync command list entry and corresponding command table entry */
1870 bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_cmd_list),
1871 ccb->ccb_slot * sizeof(struct ahci_cmd_hdr),
1872 sizeof(struct ahci_cmd_hdr), BUS_DMASYNC_PREWRITE);
1873 bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_cmd_table),
1874 ccb->ccb_slot * sizeof(struct ahci_cmd_table),
1875 sizeof(struct ahci_cmd_table), BUS_DMASYNC_PREWRITE);
1876
1877 /* Prepare RFIS area for write by controller */
1878 bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_rfis), 0,
1879 sizeof(struct ahci_rfis), BUS_DMASYNC_PREREAD);
1880
1881 if (ccb->ccb_xa.flags & ATA_F_NCQ) {
1882 /* Issue NCQ commands only when there are no outstanding
1883 * standard commands. */
1884 if (ap->ap_active != 0 || !TAILQ_EMPTY(&ap->ap_ccb_pending) ||
1885 (ap->ap_sactive != 0 &&
1886 ap->ap_pmp_ncq_port != ccb->ccb_xa.pmp_port)) {
1887 TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry);
1888 } else {
1889 KASSERT(ap->ap_active_cnt == 0);
1890 ap->ap_sactive |= (1 << ccb->ccb_slot);
1891 ccb->ccb_xa.state = ATA_S_ONCHIP;
1892 ahci_pwrite(ap, AHCI_PREG_SACT, 1 << ccb->ccb_slot);
1893 ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot);
1894 ap->ap_pmp_ncq_port = ccb->ccb_xa.pmp_port;
1895 }
1896 } else {
1897 /* Wait for all NCQ commands to finish before issuing standard
1898 * command. */
1899 if (ap->ap_sactive != 0 || ap->ap_active_cnt == 2)
1900 TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry);
1901 else if (ap->ap_active_cnt < 2) {
1902 ap->ap_active |= 1 << ccb->ccb_slot;
1903 ccb->ccb_xa.state = ATA_S_ONCHIP;
1904 ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot);
1905 ap->ap_active_cnt++;
1906 }
1907 }
1908 }
1909
1910 void
ahci_issue_pending_ncq_commands(struct ahci_port * ap)1911 ahci_issue_pending_ncq_commands(struct ahci_port *ap)
1912 {
1913 struct ahci_ccb *nextccb;
1914 u_int32_t sact_change = 0;
1915
1916 KASSERT(ap->ap_active_cnt == 0);
1917
1918 nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
1919 if (nextccb == NULL || !(nextccb->ccb_xa.flags & ATA_F_NCQ))
1920 return;
1921
1922 /* Start all the NCQ commands at the head of the pending list.
1923 * If a port multiplier is attached to the port, we can only
1924 * issue commands for one of its ports at a time.
1925 */
1926 if (ap->ap_sactive != 0 &&
1927 ap->ap_pmp_ncq_port != nextccb->ccb_xa.pmp_port) {
1928 return;
1929 }
1930
1931 ap->ap_pmp_ncq_port = nextccb->ccb_xa.pmp_port;
1932 do {
1933 TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry);
1934 sact_change |= 1 << nextccb->ccb_slot;
1935 nextccb->ccb_xa.state = ATA_S_ONCHIP;
1936 nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
1937 } while (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ) &&
1938 (nextccb->ccb_xa.pmp_port == ap->ap_pmp_ncq_port));
1939
1940 ap->ap_sactive |= sact_change;
1941 ahci_pwrite(ap, AHCI_PREG_SACT, sact_change);
1942 ahci_pwrite(ap, AHCI_PREG_CI, sact_change);
1943 }
1944
1945 void
ahci_issue_pending_commands(struct ahci_port * ap,int last_was_ncq)1946 ahci_issue_pending_commands(struct ahci_port *ap, int last_was_ncq)
1947 {
1948 struct ahci_ccb *nextccb;
1949
1950 nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
1951 if (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ)) {
1952 if (last_was_ncq) {
1953 KASSERT(nextccb->ccb_xa.pmp_port !=
1954 ap->ap_pmp_ncq_port);
1955 /* otherwise it should have been started already */
1956 } else {
1957 ap->ap_active_cnt--;
1958 }
1959
1960 /* Issue NCQ commands only when there are no outstanding
1961 * standard commands, and previous NCQ commands for other
1962 * PMP ports have finished.
1963 */
1964 if (ap->ap_active == 0)
1965 ahci_issue_pending_ncq_commands(ap);
1966 else
1967 KASSERT(ap->ap_active_cnt == 1);
1968 } else if (nextccb) {
1969 if (ap->ap_sactive != 0 || last_was_ncq)
1970 KASSERT(ap->ap_active_cnt == 0);
1971
1972 /* Wait for all NCQ commands to finish before issuing standard
1973 * command. */
1974 if (ap->ap_sactive != 0)
1975 return;
1976
1977 /* Keep up to 2 standard commands on-chip at a time. */
1978 do {
1979 TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry);
1980 ap->ap_active |= 1 << nextccb->ccb_slot;
1981 nextccb->ccb_xa.state = ATA_S_ONCHIP;
1982 ahci_pwrite(ap, AHCI_PREG_CI, 1 << nextccb->ccb_slot);
1983 if (last_was_ncq)
1984 ap->ap_active_cnt++;
1985 if (ap->ap_active_cnt == 2)
1986 break;
1987 KASSERT(ap->ap_active_cnt == 1);
1988 nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
1989 } while (nextccb && !(nextccb->ccb_xa.flags & ATA_F_NCQ));
1990 } else if (!last_was_ncq) {
1991 KASSERT(ap->ap_active_cnt == 1 || ap->ap_active_cnt == 2);
1992
1993 /* Standard command finished, none waiting to start. */
1994 ap->ap_active_cnt--;
1995 } else {
1996 KASSERT(ap->ap_active_cnt == 0);
1997
1998 /* NCQ command finished. */
1999 }
2000 }
2001
2002 int
ahci_intr(void * arg)2003 ahci_intr(void *arg)
2004 {
2005 struct ahci_softc *sc = arg;
2006 u_int32_t is, ack = 0;
2007 int port;
2008
2009 /* Read global interrupt status */
2010 is = ahci_read(sc, AHCI_REG_IS);
2011 if (is == 0 || is == 0xffffffff)
2012 return (0);
2013 ack = is;
2014
2015 #ifdef AHCI_COALESCE
2016 /* Check coalescing interrupt first */
2017 if (is & sc->sc_ccc_mask) {
2018 DPRINTF(AHCI_D_INTR, "%s: command coalescing interrupt\n",
2019 DEVNAME(sc));
2020 is &= ~sc->sc_ccc_mask;
2021 is |= sc->sc_ccc_ports_cur;
2022 }
2023 #endif
2024
2025 /* Process interrupts for each port */
2026 while (is) {
2027 port = ffs(is) - 1;
2028 if (sc->sc_ports[port])
2029 ahci_port_intr(sc->sc_ports[port],
2030 AHCI_PREG_CI_ALL_SLOTS);
2031 is &= ~(1 << port);
2032 }
2033
2034 /* Finally, acknowledge global interrupt */
2035 ahci_write(sc, AHCI_REG_IS, ack);
2036
2037 return (1);
2038 }
2039
2040 u_int32_t
ahci_port_intr(struct ahci_port * ap,u_int32_t ci_mask)2041 ahci_port_intr(struct ahci_port *ap, u_int32_t ci_mask)
2042 {
2043 struct ahci_softc *sc = ap->ap_sc;
2044 u_int32_t is, ci_saved, ci_masked, processed = 0;
2045 int slot, need_restart = 0;
2046 int process_error = 0;
2047 struct ahci_ccb *ccb;
2048 volatile u_int32_t *active;
2049 #ifdef DIAGNOSTIC
2050 u_int32_t tmp;
2051 #endif
2052
2053 is = ahci_pread(ap, AHCI_PREG_IS);
2054
2055 /* Ack port interrupt only if checking all command slots. */
2056 if (ci_mask == AHCI_PREG_CI_ALL_SLOTS)
2057 ahci_pwrite(ap, AHCI_PREG_IS, is);
2058
2059 if (is)
2060 DPRINTF(AHCI_D_INTR, "%s: interrupt: %b\n", PORTNAME(ap),
2061 is, AHCI_PFMT_IS);
2062
2063 if (ap->ap_sactive) {
2064 /* Active NCQ commands - use SActive instead of CI */
2065 KASSERT(ap->ap_active == 0);
2066 KASSERT(ap->ap_active_cnt == 0);
2067 ci_saved = ahci_pread(ap, AHCI_PREG_SACT);
2068 active = &ap->ap_sactive;
2069 } else {
2070 /* Save CI */
2071 ci_saved = ahci_pread(ap, AHCI_PREG_CI);
2072 active = &ap->ap_active;
2073 }
2074
2075 if (is & AHCI_PREG_IS_TFES) {
2076 process_error = 1;
2077 } else if (is & AHCI_PREG_IS_DHRS) {
2078 u_int32_t tfd;
2079 u_int32_t cmd;
2080 u_int32_t serr;
2081
2082 tfd = ahci_pread(ap, AHCI_PREG_TFD);
2083 cmd = ahci_pread(ap, AHCI_PREG_CMD);
2084 serr = ahci_pread(ap, AHCI_PREG_SERR);
2085 if ((tfd & AHCI_PREG_TFD_STS_ERR) &&
2086 (cmd & AHCI_PREG_CMD_CR) == 0) {
2087 DPRINTF(AHCI_D_VERBOSE, "%s: DHRS error, TFD: %b, SERR:"
2088 " %b, DIAG: %b\n", PORTNAME(ap), tfd,
2089 AHCI_PFMT_TFD_STS, AHCI_PREG_SERR_ERR(serr),
2090 AHCI_PFMT_SERR_ERR, AHCI_PREG_SERR_DIAG(serr),
2091 AHCI_PFMT_SERR_DIAG);
2092 process_error = 1;
2093 } else {
2094 /* rfis copy back is in the normal execution path */
2095 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_DHRS);
2096 }
2097 }
2098
2099 /* Command failed. See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2. */
2100 if (process_error) {
2101 u_int32_t tfd, serr;
2102 int err_slot;
2103
2104 tfd = ahci_pread(ap, AHCI_PREG_TFD);
2105 serr = ahci_pread(ap, AHCI_PREG_SERR);
2106
2107 if (ap->ap_sactive == 0) {
2108 /* Errored slot is easy to determine from CMD. */
2109 err_slot = AHCI_PREG_CMD_CCS(ahci_pread(ap,
2110 AHCI_PREG_CMD));
2111
2112 if ((ci_saved & (1 << err_slot)) == 0) {
2113 /*
2114 * Hardware doesn't seem to report correct
2115 * slot number. If there's only one
2116 * outstanding command we can cope,
2117 * otherwise fail all active commands.
2118 */
2119 if (ap->ap_active_cnt == 1)
2120 err_slot = ffs(ap->ap_active) - 1;
2121 else
2122 goto failall;
2123 }
2124
2125 ccb = &ap->ap_ccbs[err_slot];
2126
2127 /* Preserve received taskfile data from the RFIS. */
2128 memcpy(&ccb->ccb_xa.rfis, ap->ap_rfis->rfis,
2129 sizeof(struct ata_fis_d2h));
2130 } else
2131 err_slot = -1; /* Must extract error from log page */
2132
2133 DPRINTF(AHCI_D_VERBOSE, "%s: errored slot %d, TFD: %b, SERR:"
2134 " %b, DIAG: %b\n", PORTNAME(ap), err_slot, tfd,
2135 AHCI_PFMT_TFD_STS, AHCI_PREG_SERR_ERR(serr),
2136 AHCI_PFMT_SERR_ERR, AHCI_PREG_SERR_DIAG(serr),
2137 AHCI_PFMT_SERR_DIAG);
2138
2139 /* Turn off ST to clear CI and SACT. */
2140 ahci_port_stop(ap, 0);
2141 need_restart = 1;
2142
2143 /* Clear SERR to enable capturing new errors. */
2144 ahci_pwrite(ap, AHCI_PREG_SERR, serr);
2145
2146 /* Acknowledge the interrupts we can recover from. */
2147 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_TFES |
2148 AHCI_PREG_IS_IFS);
2149 is = ahci_pread(ap, AHCI_PREG_IS);
2150
2151 /* If device hasn't cleared its busy status, try to idle it. */
2152 if (ISSET(tfd, AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) {
2153
2154 if ((ap->ap_state == AP_S_PMP_PORT_PROBE) ||
2155 (ap->ap_state == AP_S_ERROR_RECOVERY)) {
2156 /* can't reset the port here, just make sure
2157 * the operation fails and the port still works.
2158 */
2159 } else if (ap->ap_pmp_ports != 0 && err_slot != -1) {
2160 printf("%s: error on PMP port %d, idling "
2161 "device\n", PORTNAME(ap),
2162 ccb->ccb_xa.pmp_port);
2163 if (ahci_pmp_port_softreset(ap,
2164 ccb->ccb_xa.pmp_port) == 0) {
2165 printf("%s: unable to softreset port "
2166 "%d\n", PORTNAME(ap),
2167 ccb->ccb_xa.pmp_port);
2168 if (ahci_pmp_port_portreset(ap,
2169 ccb->ccb_xa.pmp_port)) {
2170 printf("%s: failed to port "
2171 " reset %d, giving up on "
2172 "it\n", PORTNAME(ap),
2173 ccb->ccb_xa.pmp_port);
2174 goto fatal;
2175 }
2176 }
2177 } else {
2178 printf("%s: attempting to idle device\n",
2179 PORTNAME(ap));
2180 if (ahci_port_softreset(ap)) {
2181 printf("%s: failed to soft reset "
2182 "device\n", PORTNAME(ap));
2183 if (ahci_port_portreset(ap, 0)) {
2184 printf("%s: failed to port "
2185 "reset device, give up on "
2186 "it\n", PORTNAME(ap));
2187 goto fatal;
2188 }
2189 }
2190 }
2191
2192 /* Had to reset device, can't gather extended info. */
2193 } else if (ap->ap_sactive) {
2194 /* Recover the NCQ error from log page 10h.
2195 * We can only have queued commands active for one port
2196 * at a time, so we know which device errored.
2197 */
2198 ahci_port_read_ncq_error(ap, &err_slot,
2199 ap->ap_pmp_ncq_port);
2200 if (err_slot < 0)
2201 goto failall;
2202
2203 DPRINTF(AHCI_D_VERBOSE, "%s: NCQ errored slot %d\n",
2204 PORTNAME(ap), err_slot);
2205
2206 ccb = &ap->ap_ccbs[err_slot];
2207 if (ccb->ccb_xa.state != ATA_S_ONCHIP) {
2208 printf("%s: NCQ errored slot %d is idle"
2209 " (%08x active)\n", PORTNAME(ap), err_slot,
2210 ci_saved);
2211 goto failall;
2212 }
2213 } else {
2214 /* Didn't reset, could gather extended info from log. */
2215 }
2216
2217 /*
2218 * If we couldn't determine the errored slot, reset the port
2219 * and fail all the active slots.
2220 */
2221 if (err_slot == -1) {
2222 if (ahci_port_softreset(ap) != 0 &&
2223 ahci_port_portreset(ap, 0) != 0) {
2224 printf("%s: couldn't reset after NCQ error, "
2225 "disabling device.\n", PORTNAME(ap));
2226 goto fatal;
2227 }
2228 printf("%s: couldn't recover NCQ error, failing "
2229 "all outstanding commands.\n", PORTNAME(ap));
2230 goto failall;
2231 }
2232
2233 /* Clear the failed command in saved CI so completion runs. */
2234 ci_saved &= ~(1 << err_slot);
2235
2236 /* Note the error in the ata_xfer. */
2237 KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP);
2238 ccb->ccb_xa.state = ATA_S_ERROR;
2239
2240 #ifdef DIAGNOSTIC
2241 /* There may only be one outstanding standard command now. */
2242 if (ap->ap_sactive == 0) {
2243 tmp = ci_saved;
2244 if (tmp) {
2245 slot = ffs(tmp) - 1;
2246 tmp &= ~(1 << slot);
2247 KASSERT(tmp == 0);
2248 }
2249 }
2250 #endif
2251 }
2252
2253 /* ATI SBx00 AHCI controllers respond to PMP probes with IPMS interrupts
2254 * when there's a normal SATA device attached.
2255 */
2256 if ((ap->ap_state == AP_S_PMP_PROBE) &&
2257 (ap->ap_sc->sc_flags & AHCI_F_IPMS_PROBE) &&
2258 (is & AHCI_PREG_IS_IPMS)) {
2259 slot = AHCI_PREG_CMD_CCS(ahci_pread(ap, AHCI_PREG_CMD));
2260 DPRINTF(AHCI_D_INTR, "%s: slot %d received IPMS\n",
2261 PORTNAME(ap), slot);
2262
2263 ccb = &ap->ap_ccbs[slot];
2264 ccb->ccb_xa.state = ATA_S_ERROR;
2265
2266 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IPMS);
2267 is &= ~AHCI_PREG_IS_IPMS;
2268 }
2269
2270 /* ignore IFS errors while resetting a PMP port */
2271 if ((is & AHCI_PREG_IS_IFS) /*&& ap->ap_pmp_ignore_ifs*/) {
2272 DPRINTF(AHCI_D_INTR, "%s: ignoring IFS while resetting PMP "
2273 "port\n", PORTNAME(ap));
2274
2275 need_restart = 1;
2276 ahci_pwrite(ap, AHCI_PREG_SERR, -1);
2277 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS);
2278 is &= ~AHCI_PREG_IS_IFS;
2279 goto failall;
2280 }
2281
2282 /* Check for remaining errors - they are fatal. */
2283 if (is & (AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS | AHCI_PREG_IS_IFS |
2284 AHCI_PREG_IS_OFS | AHCI_PREG_IS_UFS)) {
2285 printf("%s: unrecoverable errors (IS: %b), disabling port.\n",
2286 PORTNAME(ap), is, AHCI_PFMT_IS);
2287
2288 /* XXX try recovery first */
2289 goto fatal;
2290 }
2291
2292 /* Fail all outstanding commands if we know the port won't recover. */
2293 if (ap->ap_state == AP_S_FATAL_ERROR) {
2294 fatal:
2295 ap->ap_state = AP_S_FATAL_ERROR;
2296 failall:
2297
2298 /* Ensure port is shut down. */
2299 ahci_port_stop(ap, 1);
2300
2301 /* Error all the active slots. */
2302 ci_masked = ci_saved & *active;
2303 while (ci_masked) {
2304 slot = ffs(ci_masked) - 1;
2305 ccb = &ap->ap_ccbs[slot];
2306 ci_masked &= ~(1 << slot);
2307 ccb->ccb_xa.state = ATA_S_ERROR;
2308 }
2309
2310 /* Run completion for all active slots. */
2311 ci_saved &= ~*active;
2312
2313 /* Don't restart the port if our problems were deemed fatal. */
2314 if (ap->ap_state == AP_S_FATAL_ERROR)
2315 need_restart = 0;
2316 }
2317
2318 /*
2319 * CCB completion is detected by noticing its slot's bit in CI has
2320 * changed to zero some time after we activated it.
2321 * If we are polling, we may only be interested in particular slot(s).
2322 */
2323 ci_masked = ~ci_saved & *active & ci_mask;
2324 while (ci_masked) {
2325 slot = ffs(ci_masked) - 1;
2326 ccb = &ap->ap_ccbs[slot];
2327 ci_masked &= ~(1 << slot);
2328
2329 DPRINTF(AHCI_D_INTR, "%s: slot %d is complete%s\n",
2330 PORTNAME(ap), slot, ccb->ccb_xa.state == ATA_S_ERROR ?
2331 " (error)" : "");
2332
2333 bus_dmamap_sync(sc->sc_dmat,
2334 AHCI_DMA_MAP(ap->ap_dmamem_cmd_list),
2335 ccb->ccb_slot * sizeof(struct ahci_cmd_hdr),
2336 sizeof(struct ahci_cmd_hdr), BUS_DMASYNC_POSTWRITE);
2337
2338 bus_dmamap_sync(sc->sc_dmat,
2339 AHCI_DMA_MAP(ap->ap_dmamem_cmd_table),
2340 ccb->ccb_slot * sizeof(struct ahci_cmd_table),
2341 sizeof(struct ahci_cmd_table), BUS_DMASYNC_POSTWRITE);
2342
2343 bus_dmamap_sync(sc->sc_dmat,
2344 AHCI_DMA_MAP(ap->ap_dmamem_rfis), 0,
2345 sizeof(struct ahci_rfis), BUS_DMASYNC_POSTREAD);
2346
2347 *active &= ~(1 << ccb->ccb_slot);
2348 /* Copy the rfis into the ccb if we were asked for it */
2349 if (ccb->ccb_xa.state == ATA_S_ONCHIP &&
2350 ccb->ccb_xa.flags & ATA_F_GET_RFIS) {
2351 memcpy(&ccb->ccb_xa.rfis,
2352 ap->ap_rfis->rfis,
2353 sizeof(struct ata_fis_d2h));
2354 }
2355
2356 processed |= 1 << ccb->ccb_slot;
2357
2358 ccb->ccb_done(ccb);
2359 }
2360
2361 if (need_restart) {
2362 /* Restart command DMA on the port */
2363 ahci_port_start(ap, 0);
2364
2365 /* Re-enable outstanding commands on port. */
2366 if (ci_saved) {
2367 #ifdef DIAGNOSTIC
2368 tmp = ci_saved;
2369 while (tmp) {
2370 slot = ffs(tmp) - 1;
2371 tmp &= ~(1 << slot);
2372 ccb = &ap->ap_ccbs[slot];
2373 KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP);
2374 KASSERT((!!(ccb->ccb_xa.flags & ATA_F_NCQ)) ==
2375 (!!ap->ap_sactive));
2376 }
2377 #endif
2378 DPRINTF(AHCI_D_VERBOSE, "%s: ahci_port_intr "
2379 "re-enabling%s slots %08x\n", PORTNAME(ap),
2380 ap->ap_sactive ? " NCQ" : "", ci_saved);
2381
2382 if (ap->ap_sactive)
2383 ahci_pwrite(ap, AHCI_PREG_SACT, ci_saved);
2384 ahci_pwrite(ap, AHCI_PREG_CI, ci_saved);
2385 }
2386 }
2387
2388 return (processed);
2389 }
2390
2391 struct ahci_ccb *
ahci_get_ccb(struct ahci_port * ap)2392 ahci_get_ccb(struct ahci_port *ap)
2393 {
2394 struct ahci_ccb *ccb;
2395
2396 mtx_enter(&ap->ap_ccb_mtx);
2397 ccb = TAILQ_FIRST(&ap->ap_ccb_free);
2398 if (ccb != NULL) {
2399 KASSERT(ccb->ccb_xa.state == ATA_S_PUT);
2400 TAILQ_REMOVE(&ap->ap_ccb_free, ccb, ccb_entry);
2401 ccb->ccb_xa.state = ATA_S_SETUP;
2402 }
2403 mtx_leave(&ap->ap_ccb_mtx);
2404
2405 return (ccb);
2406 }
2407
2408 void
ahci_put_ccb(struct ahci_ccb * ccb)2409 ahci_put_ccb(struct ahci_ccb *ccb)
2410 {
2411 struct ahci_port *ap = ccb->ccb_port;
2412
2413 #ifdef DIAGNOSTIC
2414 if (ccb->ccb_xa.state != ATA_S_COMPLETE &&
2415 ccb->ccb_xa.state != ATA_S_TIMEOUT &&
2416 ccb->ccb_xa.state != ATA_S_ERROR) {
2417 printf("%s: invalid ata_xfer state %02x in ahci_put_ccb, "
2418 "slot %d\n", PORTNAME(ccb->ccb_port), ccb->ccb_xa.state,
2419 ccb->ccb_slot);
2420 }
2421 #endif
2422
2423 ccb->ccb_xa.state = ATA_S_PUT;
2424 mtx_enter(&ap->ap_ccb_mtx);
2425 TAILQ_INSERT_TAIL(&ap->ap_ccb_free, ccb, ccb_entry);
2426 mtx_leave(&ap->ap_ccb_mtx);
2427 }
2428
2429 struct ahci_ccb *
ahci_get_err_ccb(struct ahci_port * ap)2430 ahci_get_err_ccb(struct ahci_port *ap)
2431 {
2432 struct ahci_ccb *err_ccb;
2433 u_int32_t sact;
2434
2435 splassert(IPL_BIO);
2436
2437 /* No commands may be active on the chip. */
2438 sact = ahci_pread(ap, AHCI_PREG_SACT);
2439 if (sact != 0)
2440 printf("ahci_get_err_ccb but SACT %08x != 0?\n", sact);
2441 KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0);
2442
2443 #ifdef DIAGNOSTIC
2444 KASSERT(ap->ap_err_busy == 0);
2445 ap->ap_err_busy = 1;
2446 #endif
2447 /* Save outstanding command state. */
2448 ap->ap_err_saved_active = ap->ap_active;
2449 ap->ap_err_saved_active_cnt = ap->ap_active_cnt;
2450 ap->ap_err_saved_sactive = ap->ap_sactive;
2451
2452 /*
2453 * Pretend we have no commands outstanding, so that completions won't
2454 * run prematurely.
2455 */
2456 ap->ap_active = ap->ap_active_cnt = ap->ap_sactive = 0;
2457
2458 /*
2459 * Grab a CCB to use for error recovery. This should never fail, as
2460 * we ask atascsi to reserve one for us at init time.
2461 */
2462 err_ccb = ap->ap_ccb_err;
2463 err_ccb->ccb_xa.flags = 0;
2464 err_ccb->ccb_xa.state = ATA_S_SETUP;
2465 err_ccb->ccb_done = ahci_empty_done;
2466
2467 return (err_ccb);
2468 }
2469
2470 void
ahci_put_err_ccb(struct ahci_ccb * ccb)2471 ahci_put_err_ccb(struct ahci_ccb *ccb)
2472 {
2473 struct ahci_port *ap = ccb->ccb_port;
2474 u_int32_t sact;
2475
2476 splassert(IPL_BIO);
2477
2478 #ifdef DIAGNOSTIC
2479 KASSERT(ap->ap_err_busy);
2480 #endif
2481 /* No commands may be active on the chip */
2482 sact = ahci_pread(ap, AHCI_PREG_SACT);
2483 if (sact != 0)
2484 printf("ahci_put_err_ccb but SACT %08x != 0?\n", sact);
2485 KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0);
2486
2487 /* Done with the CCB */
2488 KASSERT(ccb == ap->ap_ccb_err);
2489
2490 /* Restore outstanding command state */
2491 ap->ap_sactive = ap->ap_err_saved_sactive;
2492 ap->ap_active_cnt = ap->ap_err_saved_active_cnt;
2493 ap->ap_active = ap->ap_err_saved_active;
2494
2495 #ifdef DIAGNOSTIC
2496 ap->ap_err_busy = 0;
2497 #endif
2498 }
2499
2500 struct ahci_ccb *
ahci_get_pmp_ccb(struct ahci_port * ap)2501 ahci_get_pmp_ccb(struct ahci_port *ap)
2502 {
2503 struct ahci_ccb *ccb;
2504 u_int32_t sact;
2505
2506 /* some PMP commands need to be issued on slot 1,
2507 * particularly the command that clears SRST and
2508 * fetches the device signature.
2509 *
2510 * ensure the chip is idle and ccb 1 is available.
2511 */
2512 splassert(IPL_BIO);
2513
2514 sact = ahci_pread(ap, AHCI_PREG_SACT);
2515 if (sact != 0)
2516 printf("ahci_get_pmp_ccb; SACT %08x != 0\n", sact);
2517 KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0);
2518
2519 ccb = &ap->ap_ccbs[1];
2520 KASSERT(ccb->ccb_xa.state == ATA_S_PUT);
2521 ccb->ccb_xa.flags = 0;
2522 ccb->ccb_done = ahci_pmp_cmd_done;
2523
2524 mtx_enter(&ap->ap_ccb_mtx);
2525 TAILQ_REMOVE(&ap->ap_ccb_free, ccb, ccb_entry);
2526 mtx_leave(&ap->ap_ccb_mtx);
2527
2528 return ccb;
2529 }
2530
2531 void
ahci_put_pmp_ccb(struct ahci_ccb * ccb)2532 ahci_put_pmp_ccb(struct ahci_ccb *ccb)
2533 {
2534 struct ahci_port *ap = ccb->ccb_port;
2535 u_int32_t sact;
2536
2537 /* make sure this is the right ccb */
2538 KASSERT(ccb == &ap->ap_ccbs[1]);
2539
2540 /* No commands may be active on the chip */
2541 sact = ahci_pread(ap, AHCI_PREG_SACT);
2542 if (sact != 0)
2543 printf("ahci_put_pmp_ccb but SACT %08x != 0?\n", sact);
2544 KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0);
2545
2546 ccb->ccb_xa.state = ATA_S_PUT;
2547 mtx_enter(&ap->ap_ccb_mtx);
2548 TAILQ_INSERT_TAIL(&ap->ap_ccb_free, ccb, ccb_entry);
2549 mtx_leave(&ap->ap_ccb_mtx);
2550 }
2551
2552 int
ahci_port_read_ncq_error(struct ahci_port * ap,int * err_slotp,int pmp_port)2553 ahci_port_read_ncq_error(struct ahci_port *ap, int *err_slotp, int pmp_port)
2554 {
2555 struct ahci_ccb *ccb;
2556 struct ahci_cmd_hdr *cmd_slot;
2557 u_int32_t cmd;
2558 struct ata_fis_h2d *fis;
2559 int rc = EIO, oldstate;
2560
2561 DPRINTF(AHCI_D_VERBOSE, "%s: read log page\n", PORTNAME(ap));
2562 oldstate = ap->ap_state;
2563 ap->ap_state = AP_S_ERROR_RECOVERY;
2564
2565 /* Save command register state. */
2566 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
2567
2568 /* Port should have been idled already. Start it. */
2569 KASSERT((cmd & AHCI_PREG_CMD_CR) == 0);
2570 ahci_port_start(ap, 0);
2571
2572 /* Prep error CCB for READ LOG EXT, page 10h, 1 sector. */
2573 ccb = ahci_get_err_ccb(ap);
2574 ccb->ccb_xa.flags = ATA_F_NOWAIT | ATA_F_READ | ATA_F_POLL;
2575 ccb->ccb_xa.data = ap->ap_err_scratch;
2576 ccb->ccb_xa.datalen = 512;
2577 cmd_slot = ccb->ccb_cmd_hdr;
2578 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table));
2579
2580 fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
2581 fis->type = ATA_FIS_TYPE_H2D;
2582 fis->flags = ATA_H2D_FLAGS_CMD | pmp_port;
2583 fis->command = ATA_C_READ_LOG_EXT;
2584 fis->lba_low = 0x10; /* queued error log page (10h) */
2585 fis->sector_count = 1; /* number of sectors (1) */
2586 fis->sector_count_exp = 0;
2587 fis->lba_mid = 0; /* starting offset */
2588 fis->lba_mid_exp = 0;
2589 fis->device = 0;
2590
2591 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ |
2592 (pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT));
2593
2594 if (ahci_load_prdt(ccb) != 0) {
2595 rc = ENOMEM; /* XXX caller must abort all commands */
2596 goto err;
2597 }
2598
2599 ccb->ccb_xa.state = ATA_S_PENDING;
2600 if (ahci_poll(ccb, 1000, NULL) != 0 ||
2601 ccb->ccb_xa.state == ATA_S_ERROR)
2602 goto err;
2603
2604 rc = 0;
2605 err:
2606 /* Abort our command, if it failed, by stopping command DMA. */
2607 if (rc != 0 && ISSET(ap->ap_active, 1 << ccb->ccb_slot)) {
2608 printf("%s: log page read failed, slot %d was still active.\n",
2609 PORTNAME(ap), ccb->ccb_slot);
2610 ahci_port_stop(ap, 0);
2611 }
2612
2613 /* Done with the error CCB now. */
2614 ahci_unload_prdt(ccb);
2615 ahci_put_err_ccb(ccb);
2616
2617 /* Extract failed register set and tags from the scratch space. */
2618 if (rc == 0) {
2619 struct ata_log_page_10h *log;
2620 int err_slot;
2621
2622 log = (struct ata_log_page_10h *)ap->ap_err_scratch;
2623 if (ISSET(log->err_regs.type, ATA_LOG_10H_TYPE_NOTQUEUED)) {
2624 /* Not queued bit was set - wasn't an NCQ error? */
2625 printf("%s: read NCQ error page, but not an NCQ "
2626 "error?\n", PORTNAME(ap));
2627 rc = ESRCH;
2628 } else {
2629 /* Copy back the log record as a D2H register FIS. */
2630 *err_slotp = err_slot = log->err_regs.type &
2631 ATA_LOG_10H_TYPE_TAG_MASK;
2632
2633 ccb = &ap->ap_ccbs[err_slot];
2634 memcpy(&ccb->ccb_xa.rfis, &log->err_regs,
2635 sizeof(struct ata_fis_d2h));
2636 ccb->ccb_xa.rfis.type = ATA_FIS_TYPE_D2H;
2637 ccb->ccb_xa.rfis.flags = 0;
2638 }
2639 }
2640
2641 /* Restore saved CMD register state */
2642 ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
2643 ap->ap_state = oldstate;
2644
2645 return (rc);
2646 }
2647
2648 struct ahci_dmamem *
ahci_dmamem_alloc(struct ahci_softc * sc,size_t size)2649 ahci_dmamem_alloc(struct ahci_softc *sc, size_t size)
2650 {
2651 struct ahci_dmamem *adm;
2652 int nsegs;
2653
2654 adm = malloc(sizeof(*adm), M_DEVBUF, M_NOWAIT | M_ZERO);
2655 if (adm == NULL)
2656 return (NULL);
2657
2658 adm->adm_size = size;
2659
2660 if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0,
2661 BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, &adm->adm_map) != 0)
2662 goto admfree;
2663
2664 if (bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, &adm->adm_seg,
2665 1, &nsegs, BUS_DMA_NOWAIT | BUS_DMA_ZERO) != 0)
2666 goto destroy;
2667
2668 if (bus_dmamem_map(sc->sc_dmat, &adm->adm_seg, nsegs, size,
2669 &adm->adm_kva, BUS_DMA_NOWAIT | BUS_DMA_COHERENT) != 0)
2670 goto free;
2671
2672 if (bus_dmamap_load(sc->sc_dmat, adm->adm_map, adm->adm_kva, size,
2673 NULL, BUS_DMA_NOWAIT) != 0)
2674 goto unmap;
2675
2676 return (adm);
2677
2678 unmap:
2679 bus_dmamem_unmap(sc->sc_dmat, adm->adm_kva, size);
2680 free:
2681 bus_dmamem_free(sc->sc_dmat, &adm->adm_seg, 1);
2682 destroy:
2683 bus_dmamap_destroy(sc->sc_dmat, adm->adm_map);
2684 admfree:
2685 free(adm, M_DEVBUF, sizeof(*adm));
2686
2687 return (NULL);
2688 }
2689
2690 void
ahci_dmamem_free(struct ahci_softc * sc,struct ahci_dmamem * adm)2691 ahci_dmamem_free(struct ahci_softc *sc, struct ahci_dmamem *adm)
2692 {
2693 bus_dmamap_unload(sc->sc_dmat, adm->adm_map);
2694 bus_dmamem_unmap(sc->sc_dmat, adm->adm_kva, adm->adm_size);
2695 bus_dmamem_free(sc->sc_dmat, &adm->adm_seg, 1);
2696 bus_dmamap_destroy(sc->sc_dmat, adm->adm_map);
2697 free(adm, M_DEVBUF, sizeof(*adm));
2698 }
2699
2700 u_int32_t
ahci_read(struct ahci_softc * sc,bus_size_t r)2701 ahci_read(struct ahci_softc *sc, bus_size_t r)
2702 {
2703 bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4,
2704 BUS_SPACE_BARRIER_READ);
2705 return (bus_space_read_4(sc->sc_iot, sc->sc_ioh, r));
2706 }
2707
2708 void
ahci_write(struct ahci_softc * sc,bus_size_t r,u_int32_t v)2709 ahci_write(struct ahci_softc *sc, bus_size_t r, u_int32_t v)
2710 {
2711 bus_space_write_4(sc->sc_iot, sc->sc_ioh, r, v);
2712 bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4,
2713 BUS_SPACE_BARRIER_WRITE);
2714 }
2715
2716 int
ahci_wait_ne(struct ahci_softc * sc,bus_size_t r,u_int32_t mask,u_int32_t target)2717 ahci_wait_ne(struct ahci_softc *sc, bus_size_t r, u_int32_t mask,
2718 u_int32_t target)
2719 {
2720 int i;
2721
2722 for (i = 0; i < 1000; i++) {
2723 if ((ahci_read(sc, r) & mask) != target)
2724 return (0);
2725 delay(1000);
2726 }
2727
2728 return (1);
2729 }
2730
2731 u_int32_t
ahci_pread(struct ahci_port * ap,bus_size_t r)2732 ahci_pread(struct ahci_port *ap, bus_size_t r)
2733 {
2734 bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4,
2735 BUS_SPACE_BARRIER_READ);
2736 return (bus_space_read_4(ap->ap_sc->sc_iot, ap->ap_ioh, r));
2737 }
2738
2739 void
ahci_pwrite(struct ahci_port * ap,bus_size_t r,u_int32_t v)2740 ahci_pwrite(struct ahci_port *ap, bus_size_t r, u_int32_t v)
2741 {
2742 bus_space_write_4(ap->ap_sc->sc_iot, ap->ap_ioh, r, v);
2743 bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4,
2744 BUS_SPACE_BARRIER_WRITE);
2745 }
2746
2747 int
ahci_pwait_eq(struct ahci_port * ap,bus_size_t r,u_int32_t mask,u_int32_t target,int n)2748 ahci_pwait_eq(struct ahci_port *ap, bus_size_t r, u_int32_t mask,
2749 u_int32_t target, int n)
2750 {
2751 int i;
2752
2753 for (i = 0; i < n * 1000; i++) {
2754 if ((ahci_pread(ap, r) & mask) == target)
2755 return (0);
2756 delay(1000);
2757 }
2758
2759 return (1);
2760 }
2761
2762 int
ahci_ata_probe(void * xsc,int port,int lun)2763 ahci_ata_probe(void *xsc, int port, int lun)
2764 {
2765 struct ahci_softc *sc = xsc;
2766 struct ahci_port *ap = sc->sc_ports[port];
2767
2768 if (ap == NULL)
2769 return (ATA_PORT_T_NONE);
2770
2771 if (lun != 0) {
2772 int pmp_port = lun - 1;
2773 if (pmp_port >= ap->ap_pmp_ports) {
2774 return (ATA_PORT_T_NONE);
2775 }
2776 return (ahci_pmp_port_probe(ap, pmp_port));
2777 } else {
2778 return (ahci_port_signature(ap));
2779 }
2780 }
2781
2782 void
ahci_ata_free(void * xsc,int port,int lun)2783 ahci_ata_free(void *xsc, int port, int lun)
2784 {
2785
2786 }
2787
2788 struct ata_xfer *
ahci_ata_get_xfer(void * aaa_cookie,int port)2789 ahci_ata_get_xfer(void *aaa_cookie, int port)
2790 {
2791 struct ahci_softc *sc = aaa_cookie;
2792 struct ahci_port *ap = sc->sc_ports[port];
2793 struct ahci_ccb *ccb;
2794
2795 ccb = ahci_get_ccb(ap);
2796 if (ccb == NULL) {
2797 DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer: NULL ccb\n",
2798 PORTNAME(ap));
2799 return (NULL);
2800 }
2801
2802 DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer got slot %d\n",
2803 PORTNAME(ap), ccb->ccb_slot);
2804
2805 return ((struct ata_xfer *)ccb);
2806 }
2807
2808 void
ahci_ata_put_xfer(struct ata_xfer * xa)2809 ahci_ata_put_xfer(struct ata_xfer *xa)
2810 {
2811 struct ahci_ccb *ccb = (struct ahci_ccb *)xa;
2812
2813 DPRINTF(AHCI_D_XFER, "ahci_ata_put_xfer slot %d\n", ccb->ccb_slot);
2814
2815 ahci_put_ccb(ccb);
2816 }
2817
2818 void
ahci_ata_cmd(struct ata_xfer * xa)2819 ahci_ata_cmd(struct ata_xfer *xa)
2820 {
2821 struct ahci_ccb *ccb = (struct ahci_ccb *)xa;
2822 struct ahci_cmd_hdr *cmd_slot;
2823 int s;
2824 u_int16_t flags;
2825
2826 if (ccb->ccb_port->ap_state == AP_S_FATAL_ERROR)
2827 goto failcmd;
2828
2829 ccb->ccb_done = ahci_ata_cmd_done;
2830
2831 cmd_slot = ccb->ccb_cmd_hdr;
2832 flags = 5 /* FIS length (in DWORDs) */;
2833 flags |= xa->pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT;
2834
2835 if (xa->flags & ATA_F_WRITE)
2836 flags |= AHCI_CMD_LIST_FLAG_W;
2837
2838 if (xa->flags & ATA_F_PACKET)
2839 flags |= AHCI_CMD_LIST_FLAG_A;
2840
2841 htolem16(&cmd_slot->flags, flags);
2842
2843 if (ahci_load_prdt(ccb) != 0)
2844 goto failcmd;
2845
2846 timeout_set(&xa->stimeout, ahci_ata_cmd_timeout, ccb);
2847
2848 xa->state = ATA_S_PENDING;
2849
2850 if (xa->flags & ATA_F_POLL)
2851 ahci_poll(ccb, xa->timeout, ahci_ata_cmd_timeout);
2852 else {
2853 s = splbio();
2854 timeout_add_msec(&xa->stimeout, xa->timeout);
2855 ahci_start(ccb);
2856 splx(s);
2857 }
2858
2859 return;
2860
2861 failcmd:
2862 s = splbio();
2863 xa->state = ATA_S_ERROR;
2864 ata_complete(xa);
2865 splx(s);
2866 }
2867
2868 void
ahci_pmp_cmd_done(struct ahci_ccb * ccb)2869 ahci_pmp_cmd_done(struct ahci_ccb *ccb)
2870 {
2871 struct ata_xfer *xa = &ccb->ccb_xa;
2872
2873 if (xa->state == ATA_S_ONCHIP || xa->state == ATA_S_ERROR)
2874 ahci_issue_pending_commands(ccb->ccb_port,
2875 xa->flags & ATA_F_NCQ);
2876
2877 xa->state = ATA_S_COMPLETE;
2878 }
2879
2880
2881 void
ahci_ata_cmd_done(struct ahci_ccb * ccb)2882 ahci_ata_cmd_done(struct ahci_ccb *ccb)
2883 {
2884 struct ata_xfer *xa = &ccb->ccb_xa;
2885
2886 timeout_del(&xa->stimeout);
2887
2888 if (xa->state == ATA_S_ONCHIP || xa->state == ATA_S_ERROR)
2889 ahci_issue_pending_commands(ccb->ccb_port,
2890 xa->flags & ATA_F_NCQ);
2891
2892 ahci_unload_prdt(ccb);
2893
2894 if (xa->state == ATA_S_ONCHIP)
2895 xa->state = ATA_S_COMPLETE;
2896 #ifdef DIAGNOSTIC
2897 else if (xa->state != ATA_S_ERROR && xa->state != ATA_S_TIMEOUT)
2898 printf("%s: invalid ata_xfer state %02x in ahci_ata_cmd_done, "
2899 "slot %d\n", PORTNAME(ccb->ccb_port), xa->state,
2900 ccb->ccb_slot);
2901 #endif
2902 if (xa->state != ATA_S_TIMEOUT)
2903 ata_complete(xa);
2904 }
2905
2906 void
ahci_ata_cmd_timeout(void * arg)2907 ahci_ata_cmd_timeout(void *arg)
2908 {
2909 struct ahci_ccb *ccb = arg;
2910 struct ata_xfer *xa = &ccb->ccb_xa;
2911 struct ahci_port *ap = ccb->ccb_port;
2912 int s, ccb_was_started, ncq_cmd;
2913 volatile u_int32_t *active;
2914
2915 s = splbio();
2916
2917 ncq_cmd = (xa->flags & ATA_F_NCQ);
2918 active = ncq_cmd ? &ap->ap_sactive : &ap->ap_active;
2919
2920 if (ccb->ccb_xa.state == ATA_S_PENDING) {
2921 DPRINTF(AHCI_D_TIMEOUT, "%s: command for slot %d timed out "
2922 "before it got on chip\n", PORTNAME(ap), ccb->ccb_slot);
2923 TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry);
2924 ccb_was_started = 0;
2925 } else if (ccb->ccb_xa.state == ATA_S_ONCHIP && ahci_port_intr(ap,
2926 1 << ccb->ccb_slot)) {
2927 DPRINTF(AHCI_D_TIMEOUT, "%s: final poll of port completed "
2928 "command in slot %d\n", PORTNAME(ap), ccb->ccb_slot);
2929 goto ret;
2930 } else if (ccb->ccb_xa.state != ATA_S_ONCHIP) {
2931 DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d already "
2932 "handled%s\n", PORTNAME(ap), ccb->ccb_slot,
2933 ISSET(*active, 1 << ccb->ccb_slot) ?
2934 " but slot is still active?" : ".");
2935 goto ret;
2936 } else if (!ISSET(ahci_pread(ap, ncq_cmd ? AHCI_PREG_SACT :
2937 AHCI_PREG_CI), 1 << ccb->ccb_slot) && ISSET(*active,
2938 1 << ccb->ccb_slot)) {
2939 DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d completed but "
2940 "IRQ handler didn't detect it. Why?\n", PORTNAME(ap),
2941 ccb->ccb_slot);
2942 *active &= ~(1 << ccb->ccb_slot);
2943 ccb->ccb_done(ccb);
2944 goto ret;
2945 } else {
2946 ccb_was_started = 1;
2947 }
2948
2949 /* Complete the slot with a timeout error. */
2950 ccb->ccb_xa.state = ATA_S_TIMEOUT;
2951 *active &= ~(1 << ccb->ccb_slot);
2952 DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (1)\n", PORTNAME(ap));
2953 ccb->ccb_done(ccb); /* This won't issue pending commands or run the
2954 atascsi completion. */
2955
2956 /* Reset port to abort running command. */
2957 if (ccb_was_started) {
2958 DPRINTF(AHCI_D_TIMEOUT, "%s: resetting port to abort%s command "
2959 "in slot %d, pmp port %d, active %08x\n", PORTNAME(ap),
2960 ncq_cmd ? " NCQ" : "", ccb->ccb_slot, xa->pmp_port, *active);
2961 if (ahci_port_softreset(ap) != 0 && ahci_port_portreset(ap, 0)
2962 != 0) {
2963 printf("%s: failed to reset port during timeout "
2964 "handling, disabling it\n", PORTNAME(ap));
2965 ap->ap_state = AP_S_FATAL_ERROR;
2966 }
2967
2968 /* Restart any other commands that were aborted by the reset. */
2969 if (*active) {
2970 DPRINTF(AHCI_D_TIMEOUT, "%s: re-enabling%s slots "
2971 "%08x\n", PORTNAME(ap), ncq_cmd ? " NCQ" : "",
2972 *active);
2973 if (ncq_cmd)
2974 ahci_pwrite(ap, AHCI_PREG_SACT, *active);
2975 ahci_pwrite(ap, AHCI_PREG_CI, *active);
2976 }
2977 }
2978
2979 /* Issue any pending commands now. */
2980 DPRINTF(AHCI_D_TIMEOUT, "%s: issue pending\n", PORTNAME(ap));
2981 if (ccb_was_started)
2982 ahci_issue_pending_commands(ap, ncq_cmd);
2983 else if (ap->ap_active == 0)
2984 ahci_issue_pending_ncq_commands(ap);
2985
2986 /* Complete the timed out ata_xfer I/O (may generate new I/O). */
2987 DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (2)\n", PORTNAME(ap));
2988 ata_complete(xa);
2989
2990 DPRINTF(AHCI_D_TIMEOUT, "%s: splx\n", PORTNAME(ap));
2991 ret:
2992 splx(s);
2993 }
2994
2995 void
ahci_empty_done(struct ahci_ccb * ccb)2996 ahci_empty_done(struct ahci_ccb *ccb)
2997 {
2998 if (ccb->ccb_xa.state != ATA_S_ERROR)
2999 ccb->ccb_xa.state = ATA_S_COMPLETE;
3000 }
3001
3002 int
ahci_pmp_read(struct ahci_port * ap,int target,int which,u_int32_t * datap)3003 ahci_pmp_read(struct ahci_port *ap, int target, int which, u_int32_t *datap)
3004 {
3005 struct ahci_ccb *ccb;
3006 struct ata_fis_h2d *fis;
3007 int error;
3008
3009 ccb = ahci_get_pmp_ccb(ap); /* Always returns non-NULL. */
3010 ccb->ccb_xa.flags = ATA_F_POLL | ATA_F_GET_RFIS;
3011 ccb->ccb_xa.pmp_port = SATA_PMP_CONTROL_PORT;
3012 ccb->ccb_xa.state = ATA_S_PENDING;
3013
3014 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table));
3015 fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
3016 fis->type = ATA_FIS_TYPE_H2D;
3017 fis->flags = ATA_H2D_FLAGS_CMD | SATA_PMP_CONTROL_PORT;
3018 fis->command = ATA_C_READ_PM;
3019 fis->features = which;
3020 fis->device = target | ATA_H2D_DEVICE_LBA;
3021 fis->control = ATA_FIS_CONTROL_4BIT;
3022
3023 if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) {
3024 error = 1;
3025 } else {
3026 *datap = ccb->ccb_xa.rfis.sector_count |
3027 (ccb->ccb_xa.rfis.lba_low << 8) |
3028 (ccb->ccb_xa.rfis.lba_mid << 16) |
3029 (ccb->ccb_xa.rfis.lba_high << 24);
3030 error = 0;
3031 }
3032 ahci_put_pmp_ccb(ccb);
3033 return (error);
3034 }
3035
3036 int
ahci_pmp_write(struct ahci_port * ap,int target,int which,u_int32_t data)3037 ahci_pmp_write(struct ahci_port *ap, int target, int which, u_int32_t data)
3038 {
3039 struct ahci_ccb *ccb;
3040 struct ata_fis_h2d *fis;
3041 int error;
3042
3043 ccb = ahci_get_pmp_ccb(ap); /* Always returns non-NULL. */
3044 ccb->ccb_xa.flags = ATA_F_POLL;
3045 ccb->ccb_xa.pmp_port = SATA_PMP_CONTROL_PORT;
3046 ccb->ccb_xa.state = ATA_S_PENDING;
3047
3048 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table));
3049 fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
3050 fis->type = ATA_FIS_TYPE_H2D;
3051 fis->flags = ATA_H2D_FLAGS_CMD | SATA_PMP_CONTROL_PORT;
3052 fis->command = ATA_C_WRITE_PM;
3053 fis->features = which;
3054 fis->device = target | ATA_H2D_DEVICE_LBA;
3055 fis->sector_count = (u_int8_t)data;
3056 fis->lba_low = (u_int8_t)(data >> 8);
3057 fis->lba_mid = (u_int8_t)(data >> 16);
3058 fis->lba_high = (u_int8_t)(data >> 24);
3059 fis->control = ATA_FIS_CONTROL_4BIT;
3060
3061 error = ahci_poll(ccb, 1000, ahci_pmp_probe_timeout);
3062 ahci_put_pmp_ccb(ccb);
3063 return (error);
3064 }
3065
3066 int
ahci_pmp_phy_status(struct ahci_port * ap,int target,u_int32_t * datap)3067 ahci_pmp_phy_status(struct ahci_port *ap, int target, u_int32_t *datap)
3068 {
3069 int error;
3070
3071 error = ahci_pmp_read(ap, target, SATA_PMREG_SSTS, datap);
3072 if (error == 0)
3073 error = ahci_pmp_write(ap, target, SATA_PMREG_SERR, -1);
3074 if (error)
3075 *datap = 0;
3076
3077 return (error);
3078 }
3079
3080 int
ahci_pmp_identify(struct ahci_port * ap,int * ret_nports)3081 ahci_pmp_identify(struct ahci_port *ap, int *ret_nports)
3082 {
3083 u_int32_t chipid;
3084 u_int32_t rev;
3085 u_int32_t nports;
3086 u_int32_t features;
3087 u_int32_t enabled;
3088 int s;
3089
3090 s = splbio();
3091
3092 if (ahci_pmp_read(ap, 15, 0, &chipid) ||
3093 ahci_pmp_read(ap, 15, 1, &rev) ||
3094 ahci_pmp_read(ap, 15, 2, &nports) ||
3095 ahci_pmp_read(ap, 15, SATA_PMREG_FEA, &features) ||
3096 ahci_pmp_read(ap, 15, SATA_PMREG_FEAEN, &enabled)) {
3097 printf("%s: port multiplier identification failed\n",
3098 PORTNAME(ap));
3099 splx(s);
3100 return (1);
3101 }
3102 splx(s);
3103
3104 nports &= 0x0F;
3105
3106 /* ignore SEMB port on SiI3726 port multiplier chips */
3107 if (chipid == 0x37261095) {
3108 nports--;
3109 }
3110
3111 printf("%s: port multiplier found: chip=%08x rev=0x%b nports=%d, "
3112 "features: 0x%b, enabled: 0x%b\n", PORTNAME(ap), chipid, rev,
3113 SATA_PFMT_PM_REV, nports, features, SATA_PFMT_PM_FEA, enabled,
3114 SATA_PFMT_PM_FEA);
3115
3116 *ret_nports = nports;
3117 return (0);
3118 }
3119
3120
3121 #ifdef HIBERNATE
3122 void
ahci_hibernate_io_start(struct ahci_port * ap,struct ahci_ccb * ccb)3123 ahci_hibernate_io_start(struct ahci_port *ap, struct ahci_ccb *ccb)
3124 {
3125 ccb->ccb_cmd_hdr->prdbc = 0;
3126 ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot);
3127 }
3128
3129 int
ahci_hibernate_io_poll(struct ahci_port * ap,struct ahci_ccb * ccb)3130 ahci_hibernate_io_poll(struct ahci_port *ap, struct ahci_ccb *ccb)
3131 {
3132 u_int32_t is, ci_saved;
3133 int process_error = 0;
3134
3135 is = ahci_pread(ap, AHCI_PREG_IS);
3136
3137 ci_saved = ahci_pread(ap, AHCI_PREG_CI);
3138
3139 if (is & AHCI_PREG_IS_DHRS) {
3140 u_int32_t tfd;
3141 u_int32_t cmd;
3142
3143 tfd = ahci_pread(ap, AHCI_PREG_TFD);
3144 cmd = ahci_pread(ap, AHCI_PREG_CMD);
3145 if ((tfd & AHCI_PREG_TFD_STS_ERR) &&
3146 (cmd & AHCI_PREG_CMD_CR) == 0) {
3147 process_error = 1;
3148 } else {
3149 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_DHRS);
3150 }
3151 } else if (is & (AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS |
3152 AHCI_PREG_IS_IFS | AHCI_PREG_IS_OFS | AHCI_PREG_IS_UFS)) {
3153 process_error = 1;
3154 }
3155
3156 /* Command failed. See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2. */
3157 if (process_error) {
3158
3159 /* Turn off ST to clear CI and SACT. */
3160 ahci_port_stop(ap, 0);
3161
3162 /* just return an error indicator? we can't meaningfully
3163 * recover, and on the way back out we'll DVACT_RESUME which
3164 * resets and reinits the port.
3165 */
3166 return (EIO);
3167 }
3168
3169 /* command is finished when the bit in CI for the slot goes to 0 */
3170 if (ci_saved & (1 << ccb->ccb_slot)) {
3171 return (EAGAIN);
3172 }
3173
3174 return (0);
3175 }
3176
3177 void
ahci_hibernate_load_prdt(struct ahci_ccb * ccb)3178 ahci_hibernate_load_prdt(struct ahci_ccb *ccb)
3179 {
3180 struct ata_xfer *xa = &ccb->ccb_xa;
3181 struct ahci_prdt *prdt = ccb->ccb_cmd_table->prdt;
3182 struct ahci_cmd_hdr *cmd_slot = ccb->ccb_cmd_hdr;
3183 int i;
3184 paddr_t data_phys;
3185 u_int64_t data_bus_phys;
3186 vaddr_t data_addr;
3187 size_t seglen;
3188 size_t buflen;
3189
3190 if (xa->datalen == 0) {
3191 ccb->ccb_cmd_hdr->prdtl = 0;
3192 return;
3193 }
3194
3195 /* derived from i386/amd64 _bus_dma_load_buffer;
3196 * for amd64 the buffer will always be dma safe.
3197 */
3198
3199 buflen = xa->datalen;
3200 data_addr = (vaddr_t)xa->data;
3201 for (i = 0; buflen > 0; i++) {
3202 pmap_extract(pmap_kernel(), data_addr, &data_phys);
3203 data_bus_phys = data_phys;
3204
3205 seglen = PAGE_SIZE - ((u_long)data_addr & PGOFSET);
3206 if (buflen < seglen)
3207 seglen = buflen;
3208
3209 ahci_load_prdt_seg(&prdt[i], data_bus_phys, seglen, 0);
3210
3211 data_addr += seglen;
3212 buflen -= seglen;
3213 }
3214
3215 htolem16(&cmd_slot->prdtl, i);
3216 }
3217
3218 int
ahci_hibernate_io(dev_t dev,daddr_t blkno,vaddr_t addr,size_t size,int op,void * page)3219 ahci_hibernate_io(dev_t dev, daddr_t blkno, vaddr_t addr, size_t size,
3220 int op, void *page)
3221 {
3222 /* we use the 'real' ahci_port and ahci_softc here, but
3223 * never write to them
3224 */
3225 struct {
3226 struct ahci_cmd_hdr cmd_hdr[32]; /* page aligned, 1024 bytes */
3227 struct ahci_rfis rfis; /* 1k aligned, 256 bytes */
3228 /* cmd table isn't actually used because of mysteries */
3229 struct ahci_cmd_table cmd_table; /* 256 aligned, 512 bytes */
3230 struct ahci_port *ap;
3231 struct ahci_ccb ccb_buf;
3232 struct ahci_ccb *ccb;
3233 struct ahci_cmd_hdr *hdr_buf;
3234 int pmp_port;
3235 daddr_t poffset;
3236 size_t psize;
3237 } *my = page;
3238 struct ata_fis_h2d *fis;
3239 u_int32_t sector_count;
3240 struct ahci_cmd_hdr *cmd_slot;
3241 int rc;
3242 int timeout;
3243 u_int16_t flags;
3244
3245 if (op == HIB_INIT) {
3246 struct device *disk;
3247 struct device *scsibus;
3248 struct ahci_softc *sc;
3249 extern struct cfdriver sd_cd;
3250 struct scsi_link *link;
3251 struct scsibus_softc *bus_sc;
3252 int port;
3253 paddr_t page_phys;
3254 u_int64_t item_phys;
3255 u_int32_t cmd;
3256
3257 my->poffset = blkno;
3258 my->psize = size;
3259
3260 /* map dev to an ahci port */
3261 disk = disk_lookup(&sd_cd, DISKUNIT(dev));
3262 scsibus = disk->dv_parent;
3263 sc = (struct ahci_softc *)disk->dv_parent->dv_parent;
3264
3265 /* find the scsi_link for the device, which has the port */
3266 port = -1;
3267 bus_sc = (struct scsibus_softc *)scsibus;
3268 SLIST_FOREACH(link, &bus_sc->sc_link_list, bus_list) {
3269 if (link->device_softc == disk) {
3270 port = link->target;
3271 if (link->lun > 0)
3272 my->pmp_port = link->lun - 1;
3273 else
3274 my->pmp_port = 0;
3275
3276 break;
3277 }
3278 }
3279 if (port == -1) {
3280 /* don't know where the disk is */
3281 return (EIO);
3282 }
3283
3284 my->ap = sc->sc_ports[port];
3285
3286 /* we're going to use the first command slot,
3287 * so ensure it's not already in use
3288 */
3289 if (my->ap->ap_ccbs[0].ccb_xa.state != ATA_S_PUT) {
3290 /* this shouldn't happen, we should be idle */
3291 return (EIO);
3292 }
3293
3294 /* stop the port so we can relocate to the hibernate page */
3295 if (ahci_port_stop(my->ap, 1)) {
3296 return (EIO);
3297 }
3298 ahci_pwrite(my->ap, AHCI_PREG_SCTL, 0);
3299
3300 pmap_extract(pmap_kernel(), (vaddr_t)page, &page_phys);
3301
3302 /* Setup RFIS base address */
3303 item_phys = page_phys + ((void *)&my->rfis - page);
3304 ahci_pwrite(my->ap, AHCI_PREG_FBU,
3305 (u_int32_t)(item_phys >> 32));
3306 ahci_pwrite(my->ap, AHCI_PREG_FB, (u_int32_t)item_phys);
3307
3308 /* Enable FIS reception and activate port. */
3309 cmd = ahci_pread(my->ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
3310 cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD |
3311 AHCI_PREG_CMD_SUD;
3312 ahci_pwrite(my->ap, AHCI_PREG_CMD, cmd |
3313 AHCI_PREG_CMD_ICC_ACTIVE);
3314
3315 /* Check whether port activated. */
3316 cmd = ahci_pread(my->ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
3317 if (!ISSET(cmd, AHCI_PREG_CMD_FRE)) {
3318 return (EIO);
3319 }
3320
3321 /* Set up the single CCB */
3322 my->ccb = &my->ccb_buf;
3323 my->ccb->ccb_slot = 0;
3324 my->ccb->ccb_port = my->ap;
3325
3326 /* Setup command list base address */
3327 item_phys = page_phys + ((void *)&my->cmd_hdr - page);
3328 ahci_pwrite(my->ap, AHCI_PREG_CLBU,
3329 (u_int32_t)(item_phys >> 32));
3330 ahci_pwrite(my->ap, AHCI_PREG_CLB, (u_int32_t)item_phys);
3331
3332 my->ccb->ccb_cmd_hdr = &my->cmd_hdr[0];
3333
3334 /* use existing cmd table - moving to a new one fails */
3335 my->ccb->ccb_cmd_table = my->ap->ap_ccbs[0].ccb_cmd_table;
3336 pmap_extract(pmap_kernel(),
3337 (vaddr_t)AHCI_DMA_KVA(my->ap->ap_dmamem_cmd_table),
3338 &page_phys);
3339 item_phys = page_phys;
3340 #if 0
3341 /* use cmd table in hibernate page (doesn't work) */
3342 my->ccb->ccb_cmd_table = &my->cmd_table;
3343 item_phys = page_phys + ((void *)&my->cmd_table - page);
3344 #endif
3345 htolem64(&my->ccb->ccb_cmd_hdr->ctba, item_phys);
3346
3347 my->ccb->ccb_xa.fis =
3348 (struct ata_fis_h2d *)my->ccb->ccb_cmd_table->cfis;
3349 my->ccb->ccb_xa.packetcmd = my->ccb->ccb_cmd_table->acmd;
3350 my->ccb->ccb_xa.tag = 0;
3351
3352 /* Wait for ICC change to complete */
3353 ahci_pwait_clr(my->ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1);
3354
3355 if (ahci_port_start(my->ap, 0)) {
3356 return (EIO);
3357 }
3358
3359 /* Flush interrupts for port */
3360 ahci_pwrite(my->ap, AHCI_PREG_IS, ahci_pread(my->ap,
3361 AHCI_PREG_IS));
3362 ahci_write(sc, AHCI_REG_IS, 1 << port);
3363
3364 ahci_enable_interrupts(my->ap);
3365 return (0);
3366 } else if (op == HIB_DONE) {
3367 ahci_activate(&my->ap->ap_sc->sc_dev, DVACT_RESUME);
3368 return (0);
3369 }
3370
3371 if (blkno > my->psize)
3372 return (E2BIG);
3373 blkno += my->poffset;
3374
3375 /* build fis */
3376 sector_count = size / 512; /* dlg promises this is okay */
3377 my->ccb->ccb_xa.flags = op == HIB_W ? ATA_F_WRITE : ATA_F_READ;
3378 fis = my->ccb->ccb_xa.fis;
3379 fis->flags = ATA_H2D_FLAGS_CMD | my->pmp_port;
3380 fis->lba_low = blkno & 0xff;
3381 fis->lba_mid = (blkno >> 8) & 0xff;
3382 fis->lba_high = (blkno >> 16) & 0xff;
3383
3384 if (sector_count > 0x100 || blkno > 0xfffffff) {
3385 /* Use LBA48 */
3386 fis->command = op == HIB_W ? ATA_C_WRITEDMA_EXT :
3387 ATA_C_READDMA_EXT;
3388 fis->device = ATA_H2D_DEVICE_LBA;
3389 fis->lba_low_exp = (blkno >> 24) & 0xff;
3390 fis->lba_mid_exp = (blkno >> 32) & 0xff;
3391 fis->lba_high_exp = (blkno >> 40) & 0xff;
3392 fis->sector_count = sector_count & 0xff;
3393 fis->sector_count_exp = (sector_count >> 8) & 0xff;
3394 } else {
3395 /* Use LBA */
3396 fis->command = op == HIB_W ? ATA_C_WRITEDMA : ATA_C_READDMA;
3397 fis->device = ATA_H2D_DEVICE_LBA | ((blkno >> 24) & 0x0f);
3398 fis->sector_count = sector_count & 0xff;
3399 }
3400
3401 my->ccb->ccb_xa.data = (void *)addr;
3402 my->ccb->ccb_xa.datalen = size;
3403 my->ccb->ccb_xa.pmp_port = my->pmp_port;
3404 my->ccb->ccb_xa.flags |= ATA_F_POLL;
3405
3406 cmd_slot = my->ccb->ccb_cmd_hdr;
3407 flags = 5; /* FIS length (in DWORDs) */
3408 flags |= my->pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT;
3409
3410 if (op == HIB_W)
3411 flags |= AHCI_CMD_LIST_FLAG_W;
3412
3413 htolem16(&cmd_slot->flags, flags);
3414
3415 ahci_hibernate_load_prdt(my->ccb);
3416
3417 ahci_hibernate_io_start(my->ap, my->ccb);
3418 timeout = 1000000;
3419 while ((rc = ahci_hibernate_io_poll(my->ap, my->ccb)) == EAGAIN) {
3420 delay(1);
3421 timeout--;
3422 if (timeout == 0) {
3423 return (EIO);
3424 }
3425 }
3426
3427 return (0);
3428 }
3429
3430 #endif
3431