1 /* $OpenBSD: ahci.c,v 1.21 2015/03/21 13:42:06 mpi Exp $ */ 2 3 /* 4 * Copyright (c) 2006 David Gwynne <dlg@openbsd.org> 5 * Copyright (c) 2010 Conformal Systems LLC <info@conformal.com> 6 * Copyright (c) 2010 Jonathan Matthew <jonathan@d14n.org> 7 * 8 * Permission to use, copy, modify, and distribute this software for any 9 * purpose with or without fee is hereby granted, provided that the above 10 * copyright notice and this permission notice appear in all copies. 11 * 12 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 13 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 14 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 15 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 16 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 17 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 18 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 19 */ 20 21 #include <sys/param.h> 22 #include <sys/systm.h> 23 #include <sys/buf.h> 24 #include <sys/kernel.h> 25 #include <sys/malloc.h> 26 #include <sys/device.h> 27 #include <sys/queue.h> 28 #include <sys/mutex.h> 29 #include <sys/pool.h> 30 31 #include <machine/bus.h> 32 33 #include <dev/ic/ahcireg.h> 34 #include <dev/ic/ahcivar.h> 35 36 #ifdef AHCI_DEBUG 37 #define DPRINTF(m, f...) do { if ((ahcidebug & (m)) == (m)) printf(f); } \ 38 while (0) 39 #define AHCI_D_TIMEOUT 0x00 40 #define AHCI_D_VERBOSE 0x01 41 #define AHCI_D_INTR 0x02 42 #define AHCI_D_XFER 0x08 43 int ahcidebug = AHCI_D_VERBOSE; 44 #else 45 #define DPRINTF(m, f...) 46 #endif 47 48 #ifdef HIBERNATE 49 #include <uvm/uvm_extern.h> 50 #include <sys/hibernate.h> 51 #include <sys/disk.h> 52 #include <sys/disklabel.h> 53 54 #include <scsi/scsi_all.h> 55 #include <scsi/scsiconf.h> 56 57 void ahci_hibernate_io_start(struct ahci_port *, 58 struct ahci_ccb *); 59 int ahci_hibernate_io_poll(struct ahci_port *, 60 struct ahci_ccb *); 61 void ahci_hibernate_load_prdt(struct ahci_ccb *); 62 63 int ahci_hibernate_io(dev_t dev, daddr_t blkno, 64 vaddr_t addr, size_t size, int wr, void *page); 65 #endif 66 67 struct cfdriver ahci_cd = { 68 NULL, "ahci", DV_DULL 69 }; 70 71 void ahci_enable_interrupts(struct ahci_port *); 72 73 int ahci_init(struct ahci_softc *); 74 int ahci_port_alloc(struct ahci_softc *, u_int); 75 void ahci_port_free(struct ahci_softc *, u_int); 76 int ahci_port_init(struct ahci_softc *, u_int); 77 78 int ahci_default_port_start(struct ahci_port *, int); 79 int ahci_port_stop(struct ahci_port *, int); 80 int ahci_port_clo(struct ahci_port *); 81 int ahci_port_softreset(struct ahci_port *); 82 int ahci_port_portreset(struct ahci_port *, int); 83 int ahci_port_signature(struct ahci_port *); 84 int ahci_pmp_port_softreset(struct ahci_port *, int); 85 int ahci_pmp_port_portreset(struct ahci_port *, int); 86 int ahci_pmp_port_probe(struct ahci_port *ap, int pmp_port); 87 88 int ahci_load_prdt(struct ahci_ccb *); 89 void ahci_load_prdt_seg(struct ahci_prdt *, u_int64_t, 90 u_int32_t, u_int32_t); 91 void ahci_unload_prdt(struct ahci_ccb *); 92 93 int ahci_poll(struct ahci_ccb *, int, void (*)(void *)); 94 void ahci_start(struct ahci_ccb *); 95 96 void ahci_issue_pending_ncq_commands(struct ahci_port *); 97 void ahci_issue_pending_commands(struct ahci_port *, int); 98 99 int ahci_intr(void *); 100 u_int32_t ahci_port_intr(struct ahci_port *, u_int32_t); 101 102 struct ahci_ccb *ahci_get_ccb(struct ahci_port *); 103 void ahci_put_ccb(struct ahci_ccb *); 104 105 struct ahci_ccb *ahci_get_err_ccb(struct ahci_port *); 106 void ahci_put_err_ccb(struct ahci_ccb *); 107 108 struct ahci_ccb *ahci_get_pmp_ccb(struct ahci_port *); 109 void ahci_put_pmp_ccb(struct ahci_ccb *); 110 111 int ahci_port_read_ncq_error(struct ahci_port *, int *, int); 112 113 struct ahci_dmamem *ahci_dmamem_alloc(struct ahci_softc *, size_t); 114 void ahci_dmamem_free(struct ahci_softc *, 115 struct ahci_dmamem *); 116 117 u_int32_t ahci_read(struct ahci_softc *, bus_size_t); 118 void ahci_write(struct ahci_softc *, bus_size_t, u_int32_t); 119 int ahci_wait_ne(struct ahci_softc *, bus_size_t, 120 u_int32_t, u_int32_t); 121 122 u_int32_t ahci_pread(struct ahci_port *, bus_size_t); 123 void ahci_pwrite(struct ahci_port *, bus_size_t, u_int32_t); 124 int ahci_pwait_eq(struct ahci_port *, bus_size_t, 125 u_int32_t, u_int32_t, int); 126 void ahci_flush_tfd(struct ahci_port *ap); 127 u_int32_t ahci_active_mask(struct ahci_port *); 128 int ahci_port_detect_pmp(struct ahci_port *); 129 void ahci_pmp_probe_timeout(void *); 130 131 /* pmp operations */ 132 int ahci_pmp_read(struct ahci_port *, int, int, 133 u_int32_t *); 134 int ahci_pmp_write(struct ahci_port *, int, int, u_int32_t); 135 int ahci_pmp_phy_status(struct ahci_port *, int, 136 u_int32_t *); 137 int ahci_pmp_identify(struct ahci_port *, int *); 138 139 140 /* Wait for all bits in _b to be cleared */ 141 #define ahci_pwait_clr(_ap, _r, _b, _n) \ 142 ahci_pwait_eq((_ap), (_r), (_b), 0, (_n)) 143 144 /* Wait for all bits in _b to be set */ 145 #define ahci_pwait_set(_ap, _r, _b, _n) \ 146 ahci_pwait_eq((_ap), (_r), (_b), (_b), (_n)) 147 148 149 150 /* provide methods for atascsi to call */ 151 int ahci_ata_probe(void *, int, int); 152 void ahci_ata_free(void *, int, int); 153 struct ata_xfer * ahci_ata_get_xfer(void *, int); 154 void ahci_ata_put_xfer(struct ata_xfer *); 155 void ahci_ata_cmd(struct ata_xfer *); 156 157 struct atascsi_methods ahci_atascsi_methods = { 158 ahci_ata_probe, 159 ahci_ata_free, 160 ahci_ata_get_xfer, 161 ahci_ata_put_xfer, 162 ahci_ata_cmd 163 }; 164 165 /* ccb completions */ 166 void ahci_ata_cmd_done(struct ahci_ccb *); 167 void ahci_pmp_cmd_done(struct ahci_ccb *); 168 void ahci_ata_cmd_timeout(void *); 169 void ahci_empty_done(struct ahci_ccb *); 170 171 int 172 ahci_attach(struct ahci_softc *sc) 173 { 174 struct atascsi_attach_args aaa; 175 u_int32_t pi; 176 int i; 177 178 if (sc->sc_port_start == NULL) 179 sc->sc_port_start = ahci_default_port_start; 180 181 if (ahci_init(sc) != 0) { 182 /* error already printed by ahci_init */ 183 goto unmap; 184 } 185 186 printf("\n"); 187 188 sc->sc_cap = ahci_read(sc, AHCI_REG_CAP); 189 sc->sc_ncmds = AHCI_REG_CAP_NCS(sc->sc_cap); 190 #ifdef AHCI_DEBUG 191 if (ahcidebug & AHCI_D_VERBOSE) { 192 const char *gen; 193 194 switch (sc->sc_cap & AHCI_REG_CAP_ISS) { 195 case AHCI_REG_CAP_ISS_G1: 196 gen = "1 (1.5Gbps)"; 197 break; 198 case AHCI_REG_CAP_ISS_G2: 199 gen = "2 (3.0Gb/s)"; 200 break; 201 case AHCI_REG_CAP_ISS_G3: 202 gen = "3 (6.0Gb/s)"; 203 break; 204 default: 205 gen = "unknown"; 206 break; 207 } 208 209 printf("%s: capabilities 0x%b, %d ports, %d cmds, gen %s\n", 210 DEVNAME(sc), sc->sc_cap, AHCI_FMT_CAP, 211 AHCI_REG_CAP_NP(sc->sc_cap), sc->sc_ncmds, gen); 212 printf("%s: extended capabilities 0x%b\n", DEVNAME(sc), 213 ahci_read(sc, AHCI_REG_CAP2), AHCI_FMT_CAP2); 214 } 215 #endif 216 217 pi = ahci_read(sc, AHCI_REG_PI); 218 DPRINTF(AHCI_D_VERBOSE, "%s: ports implemented: 0x%08x\n", 219 DEVNAME(sc), pi); 220 221 #ifdef AHCI_COALESCE 222 /* Naive coalescing support - enable for all ports. */ 223 if (sc->sc_cap & AHCI_REG_CAP_CCCS) { 224 u_int16_t ccc_timeout = 20; 225 u_int8_t ccc_numcomplete = 12; 226 u_int32_t ccc_ctl; 227 228 /* disable coalescing during reconfiguration. */ 229 ccc_ctl = ahci_read(sc, AHCI_REG_CCC_CTL); 230 ccc_ctl &= ~0x00000001; 231 ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl); 232 233 sc->sc_ccc_mask = 1 << AHCI_REG_CCC_CTL_INT(ccc_ctl); 234 if (pi & sc->sc_ccc_mask) { 235 /* A conflict with the implemented port list? */ 236 printf("%s: coalescing interrupt/implemented port list " 237 "conflict, PI: %08x, ccc_mask: %08x\n", 238 DEVNAME(sc), pi, sc->sc_ccc_mask); 239 sc->sc_ccc_mask = 0; 240 goto noccc; 241 } 242 243 /* ahci_port_start will enable each port when it starts. */ 244 sc->sc_ccc_ports = pi; 245 sc->sc_ccc_ports_cur = 0; 246 247 /* program thresholds and enable overall coalescing. */ 248 ccc_ctl &= ~0xffffff00; 249 ccc_ctl |= (ccc_timeout << 16) | (ccc_numcomplete << 8); 250 ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl); 251 ahci_write(sc, AHCI_REG_CCC_PORTS, 0); 252 ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl | 1); 253 } 254 noccc: 255 #endif 256 for (i = 0; i < AHCI_MAX_PORTS; i++) { 257 if (!ISSET(pi, 1 << i)) { 258 /* dont allocate stuff if the port isnt implemented */ 259 continue; 260 } 261 262 if (ahci_port_alloc(sc, i) == ENOMEM) 263 goto freeports; 264 } 265 266 memset(&aaa, 0, sizeof(aaa)); 267 aaa.aaa_cookie = sc; 268 aaa.aaa_methods = &ahci_atascsi_methods; 269 aaa.aaa_minphys = NULL; 270 aaa.aaa_nports = AHCI_MAX_PORTS; 271 aaa.aaa_ncmds = sc->sc_ncmds - 1; 272 if (!(sc->sc_flags & AHCI_F_NO_NCQ) && 273 (sc->sc_cap & AHCI_REG_CAP_SNCQ)) { 274 aaa.aaa_capability |= ASAA_CAP_NCQ | ASAA_CAP_PMP_NCQ; 275 /* XXX enabling ASAA_CAP_PMP_NCQ with FBS: 276 * - some error recovery work required (single device vs port 277 * errors) 278 * - probably need to look at storing our active ccb queue 279 * differently so we can group ncq and non-ncq commands 280 * for different ports. as long as we preserve the order for 281 * each port, we can reorder commands to get more ncq 282 * commands to run in parallel. 283 */ 284 } 285 286 sc->sc_atascsi = atascsi_attach(&sc->sc_dev, &aaa); 287 288 /* Enable interrupts */ 289 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE | AHCI_REG_GHC_IE); 290 291 return 0; 292 293 freeports: 294 for (i = 0; i < AHCI_MAX_PORTS; i++) { 295 if (sc->sc_ports[i] != NULL) 296 ahci_port_free(sc, i); 297 } 298 unmap: 299 /* Disable controller */ 300 ahci_write(sc, AHCI_REG_GHC, 0); 301 return 1; 302 } 303 304 int 305 ahci_detach(struct ahci_softc *sc, int flags) 306 { 307 int rv, i; 308 309 if (sc->sc_atascsi != NULL) { 310 rv = atascsi_detach(sc->sc_atascsi, flags); 311 if (rv != 0) 312 return (rv); 313 } 314 315 for (i = 0; i < AHCI_MAX_PORTS; i++) { 316 if (sc->sc_ports[i] != NULL) 317 ahci_port_free(sc, i); 318 } 319 320 return (0); 321 } 322 323 int 324 ahci_activate(struct device *self, int act) 325 { 326 struct ahci_softc *sc = (struct ahci_softc *)self; 327 int i, rv = 0; 328 329 switch (act) { 330 case DVACT_RESUME: 331 /* enable ahci (global interrupts disabled) */ 332 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE); 333 334 /* restore BIOS initialised parameters */ 335 ahci_write(sc, AHCI_REG_CAP, sc->sc_cap); 336 337 for (i = 0; i < AHCI_MAX_PORTS; i++) { 338 if (sc->sc_ports[i] != NULL) 339 ahci_port_init(sc, i); 340 } 341 342 /* Enable interrupts */ 343 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE | AHCI_REG_GHC_IE); 344 345 rv = config_activate_children(self, act); 346 break; 347 case DVACT_POWERDOWN: 348 rv = config_activate_children(self, act); 349 for (i = 0; i < AHCI_MAX_PORTS; i++) { 350 if (sc->sc_ports[i] != NULL) 351 ahci_port_stop(sc->sc_ports[i], 1); 352 } 353 break; 354 default: 355 rv = config_activate_children(self, act); 356 break; 357 } 358 return (rv); 359 } 360 361 int 362 ahci_init(struct ahci_softc *sc) 363 { 364 u_int32_t reg, cap, pi; 365 const char *revision; 366 367 DPRINTF(AHCI_D_VERBOSE, " GHC 0x%b", ahci_read(sc, AHCI_REG_GHC), 368 AHCI_FMT_GHC); 369 370 /* save BIOS initialised parameters, enable staggered spin up */ 371 cap = ahci_read(sc, AHCI_REG_CAP); 372 cap &= AHCI_REG_CAP_SMPS; 373 cap |= AHCI_REG_CAP_SSS; 374 pi = ahci_read(sc, AHCI_REG_PI); 375 376 if (ISSET(AHCI_REG_GHC_AE, ahci_read(sc, AHCI_REG_GHC))) { 377 /* reset the controller */ 378 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_HR); 379 if (ahci_wait_ne(sc, AHCI_REG_GHC, AHCI_REG_GHC_HR, 380 AHCI_REG_GHC_HR) != 0) { 381 printf(" unable to reset controller\n"); 382 return (1); 383 } 384 } 385 386 /* enable ahci (global interrupts disabled) */ 387 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE); 388 389 /* restore parameters */ 390 ahci_write(sc, AHCI_REG_CAP, cap); 391 ahci_write(sc, AHCI_REG_PI, pi); 392 393 /* check the revision */ 394 reg = ahci_read(sc, AHCI_REG_VS); 395 switch (reg) { 396 case AHCI_REG_VS_0_95: 397 revision = "0.95"; 398 break; 399 case AHCI_REG_VS_1_0: 400 revision = "1.0"; 401 break; 402 case AHCI_REG_VS_1_1: 403 revision = "1.1"; 404 break; 405 case AHCI_REG_VS_1_2: 406 revision = "1.2"; 407 break; 408 case AHCI_REG_VS_1_3: 409 revision = "1.3"; 410 break; 411 case AHCI_REG_VS_1_3_1: 412 revision = "1.3.1"; 413 break; 414 415 default: 416 printf(" unsupported AHCI revision 0x%08x\n", reg); 417 return (1); 418 } 419 420 printf(" AHCI %s", revision); 421 422 return (0); 423 } 424 425 void 426 ahci_enable_interrupts(struct ahci_port *ap) 427 { 428 ahci_pwrite(ap, AHCI_PREG_IE, AHCI_PREG_IE_TFEE | AHCI_PREG_IE_HBFE | 429 AHCI_PREG_IE_IFE | AHCI_PREG_IE_OFE | AHCI_PREG_IE_DPE | 430 AHCI_PREG_IE_UFE | 431 ((ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF) ? AHCI_PREG_IE_IPME : 0) | 432 #ifdef AHCI_COALESCE 433 ((ap->ap_sc->sc_ccc_ports & (1 << ap->ap_port)) ? 0 : 434 (AHCI_PREG_IE_SDBE | AHCI_PREG_IE_DHRE)) 435 #else 436 AHCI_PREG_IE_SDBE | AHCI_PREG_IE_DHRE 437 #endif 438 ); 439 } 440 441 int 442 ahci_port_alloc(struct ahci_softc *sc, u_int port) 443 { 444 struct ahci_port *ap; 445 struct ahci_ccb *ccb; 446 u_int64_t dva; 447 u_int32_t cmd; 448 struct ahci_cmd_hdr *hdr; 449 struct ahci_cmd_table *table; 450 const char *speed; 451 int i, rc = ENOMEM; 452 453 ap = malloc(sizeof(*ap), M_DEVBUF, M_NOWAIT | M_ZERO); 454 if (ap == NULL) { 455 printf("%s: unable to allocate memory for port %d\n", 456 DEVNAME(sc), port); 457 goto reterr; 458 } 459 ap->ap_err_scratch = dma_alloc(DEV_BSIZE, PR_NOWAIT | PR_ZERO); 460 if (ap->ap_err_scratch == NULL) { 461 printf("%s: unable to allocate DMA scratch buf for port %d\n", 462 DEVNAME(sc), port); 463 goto freeport; 464 } 465 466 #ifdef AHCI_DEBUG 467 snprintf(ap->ap_name, sizeof(ap->ap_name), "%s.%d", 468 DEVNAME(sc), port); 469 #endif 470 ap->ap_port = port; 471 sc->sc_ports[port] = ap; 472 473 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, 474 AHCI_PORT_REGION(port), AHCI_PORT_SIZE, &ap->ap_ioh) != 0) { 475 printf("%s: unable to create register window for port %d\n", 476 DEVNAME(sc), port); 477 goto freeport; 478 } 479 480 ap->ap_sc = sc; 481 #ifdef AHCI_COALESCE 482 ap->ap_num = port; 483 #endif 484 TAILQ_INIT(&ap->ap_ccb_free); 485 TAILQ_INIT(&ap->ap_ccb_pending); 486 mtx_init(&ap->ap_ccb_mtx, IPL_BIO); 487 488 /* Disable port interrupts */ 489 ahci_pwrite(ap, AHCI_PREG_IE, 0); 490 491 /* Sec 10.1.2 - deinitialise port if it is already running */ 492 cmd = ahci_pread(ap, AHCI_PREG_CMD); 493 if (ISSET(cmd, (AHCI_PREG_CMD_ST | AHCI_PREG_CMD_CR | 494 AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_FR)) || 495 ISSET(ahci_pread(ap, AHCI_PREG_SCTL), AHCI_PREG_SCTL_DET)) { 496 int r; 497 498 r = ahci_port_stop(ap, 1); 499 if (r) { 500 printf("%s: unable to disable %s, ignoring port %d\n", 501 DEVNAME(sc), r == 2 ? "CR" : "FR", port); 502 rc = ENXIO; 503 goto freeport; 504 } 505 506 /* Write DET to zero */ 507 ahci_pwrite(ap, AHCI_PREG_SCTL, 0); 508 } 509 510 /* XXX FBS - need to allocate 16x ahci_rfis struct? - but we don't 511 * know if there's a PMP attached or if the HBA supports FBS yet.. 512 * reallocate when we enable FBS? 513 */ 514 515 /* Allocate RFIS */ 516 ap->ap_dmamem_rfis = ahci_dmamem_alloc(sc, sizeof(struct ahci_rfis)); 517 if (ap->ap_dmamem_rfis == NULL) 518 goto nomem; 519 520 /* Setup RFIS base address */ 521 ap->ap_rfis = (struct ahci_rfis *) AHCI_DMA_KVA(ap->ap_dmamem_rfis); 522 dva = AHCI_DMA_DVA(ap->ap_dmamem_rfis); 523 ahci_pwrite(ap, AHCI_PREG_FBU, (u_int32_t)(dva >> 32)); 524 ahci_pwrite(ap, AHCI_PREG_FB, (u_int32_t)dva); 525 526 /* Enable FIS reception and activate port. */ 527 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 528 cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD | AHCI_PREG_CMD_SUD; 529 ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_ICC_ACTIVE); 530 531 /* Check whether port activated. Skip it if not. */ 532 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 533 if (!ISSET(cmd, AHCI_PREG_CMD_FRE)) { 534 rc = ENXIO; 535 goto freeport; 536 } 537 538 /* Allocate a CCB for each command slot */ 539 ap->ap_ccbs = mallocarray(sc->sc_ncmds, sizeof(struct ahci_ccb), 540 M_DEVBUF, M_NOWAIT | M_ZERO); 541 if (ap->ap_ccbs == NULL) { 542 printf("%s: unable to allocate command list for port %d\n", 543 DEVNAME(sc), port); 544 goto freeport; 545 } 546 547 /* Command List Structures and Command Tables */ 548 ap->ap_dmamem_cmd_list = ahci_dmamem_alloc(sc, 549 sc->sc_ncmds * sizeof(struct ahci_cmd_hdr)); 550 ap->ap_dmamem_cmd_table = ahci_dmamem_alloc(sc, 551 sc->sc_ncmds * sizeof(struct ahci_cmd_table)); 552 if (ap->ap_dmamem_cmd_table == NULL || ap->ap_dmamem_cmd_list == NULL) { 553 nomem: 554 printf("%s: unable to allocate DMA memory for port %d\n", 555 DEVNAME(sc), port); 556 goto freeport; 557 } 558 559 /* Setup command list base address */ 560 dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_list); 561 ahci_pwrite(ap, AHCI_PREG_CLBU, (u_int32_t)(dva >> 32)); 562 ahci_pwrite(ap, AHCI_PREG_CLB, (u_int32_t)dva); 563 564 /* Split CCB allocation into CCBs and assign to command header/table */ 565 hdr = AHCI_DMA_KVA(ap->ap_dmamem_cmd_list); 566 table = AHCI_DMA_KVA(ap->ap_dmamem_cmd_table); 567 for (i = 0; i < sc->sc_ncmds; i++) { 568 ccb = &ap->ap_ccbs[i]; 569 570 if (bus_dmamap_create(sc->sc_dmat, MAXPHYS, AHCI_MAX_PRDT, 571 (4 * 1024 * 1024), 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, 572 &ccb->ccb_dmamap) != 0) { 573 printf("%s: unable to create dmamap for port %d " 574 "ccb %d\n", DEVNAME(sc), port, i); 575 goto freeport; 576 } 577 578 ccb->ccb_slot = i; 579 ccb->ccb_port = ap; 580 ccb->ccb_cmd_hdr = &hdr[i]; 581 ccb->ccb_cmd_table = &table[i]; 582 htolem64(&ccb->ccb_cmd_hdr->ctba, 583 AHCI_DMA_DVA(ap->ap_dmamem_cmd_table) + 584 ccb->ccb_slot * sizeof(struct ahci_cmd_table)); 585 586 ccb->ccb_xa.fis = 587 (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis; 588 ccb->ccb_xa.packetcmd = ccb->ccb_cmd_table->acmd; 589 ccb->ccb_xa.tag = i; 590 591 ccb->ccb_xa.state = ATA_S_COMPLETE; 592 ahci_put_ccb(ccb); 593 } 594 595 /* grab a ccb for use during error recovery */ 596 ap->ap_ccb_err = &ap->ap_ccbs[sc->sc_ncmds - 1]; 597 TAILQ_REMOVE(&ap->ap_ccb_free, ap->ap_ccb_err, ccb_entry); 598 ap->ap_ccb_err->ccb_xa.state = ATA_S_COMPLETE; 599 600 /* Wait for ICC change to complete */ 601 ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1); 602 603 /* Reset port */ 604 rc = ahci_port_portreset(ap, 1); 605 606 switch (rc) { 607 case ENODEV: 608 switch (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) { 609 case AHCI_PREG_SSTS_DET_DEV_NE: 610 printf("%s: device not communicating on port %d\n", 611 DEVNAME(sc), port); 612 break; 613 case AHCI_PREG_SSTS_DET_PHYOFFLINE: 614 printf("%s: PHY offline on port %d\n", DEVNAME(sc), 615 port); 616 break; 617 default: 618 DPRINTF(AHCI_D_VERBOSE, "%s: no device detected " 619 "on port %d\n", DEVNAME(sc), port); 620 break; 621 } 622 goto freeport; 623 624 case EBUSY: 625 printf("%s: device on port %d didn't come ready, " 626 "TFD: 0x%b\n", DEVNAME(sc), port, 627 ahci_pread(ap, AHCI_PREG_TFD), AHCI_PFMT_TFD_STS); 628 629 /* Try a soft reset to clear busy */ 630 rc = ahci_port_softreset(ap); 631 if (rc) { 632 printf("%s: unable to communicate " 633 "with device on port %d\n", DEVNAME(sc), port); 634 goto freeport; 635 } 636 break; 637 638 default: 639 break; 640 } 641 642 DPRINTF(AHCI_D_VERBOSE, "%s: detected device on port %d; %d\n", 643 DEVNAME(sc), port, rc); 644 645 /* Read current link speed */ 646 switch(ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_SPD) { 647 case AHCI_PREG_SSTS_SPD_GEN1: 648 speed = "1.5Gb/s"; 649 break; 650 case AHCI_PREG_SSTS_SPD_GEN2: 651 speed = "3.0Gb/s"; 652 break; 653 case AHCI_PREG_SSTS_SPD_GEN3: 654 speed = "6.0Gb/s"; 655 break; 656 default: 657 speed = NULL; 658 break; 659 } 660 if (speed != NULL) 661 printf("%s: port %d: %s\n", PORTNAME(ap), port, speed); 662 663 /* Enable command transfers on port */ 664 if (ahci_port_start(ap, 0)) { 665 printf("%s: failed to start command DMA on port %d, " 666 "disabling\n", DEVNAME(sc), port); 667 rc = ENXIO; /* couldn't start port */ 668 } 669 670 /* Flush interrupts for port */ 671 ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS)); 672 ahci_write(sc, AHCI_REG_IS, 1 << port); 673 674 ahci_enable_interrupts(ap); 675 676 freeport: 677 if (rc != 0) 678 ahci_port_free(sc, port); 679 reterr: 680 return (rc); 681 } 682 683 void 684 ahci_port_free(struct ahci_softc *sc, u_int port) 685 { 686 struct ahci_port *ap = sc->sc_ports[port]; 687 struct ahci_ccb *ccb; 688 689 /* Ensure port is disabled and its interrupts are flushed */ 690 if (ap->ap_sc) { 691 ahci_pwrite(ap, AHCI_PREG_CMD, 0); 692 ahci_pwrite(ap, AHCI_PREG_IE, 0); 693 ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS)); 694 ahci_write(sc, AHCI_REG_IS, 1 << port); 695 } 696 697 if (ap->ap_ccb_err) 698 ahci_put_ccb(ap->ap_ccb_err); 699 700 if (ap->ap_ccbs) { 701 while ((ccb = ahci_get_ccb(ap)) != NULL) 702 bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap); 703 free(ap->ap_ccbs, M_DEVBUF, 0); 704 } 705 706 if (ap->ap_dmamem_cmd_list) 707 ahci_dmamem_free(sc, ap->ap_dmamem_cmd_list); 708 if (ap->ap_dmamem_rfis) 709 ahci_dmamem_free(sc, ap->ap_dmamem_rfis); 710 if (ap->ap_dmamem_cmd_table) 711 ahci_dmamem_free(sc, ap->ap_dmamem_cmd_table); 712 if (ap->ap_err_scratch) 713 dma_free(ap->ap_err_scratch, DEV_BSIZE); 714 715 /* bus_space(9) says we dont free the subregions handle */ 716 717 free(ap, M_DEVBUF, 0); 718 sc->sc_ports[port] = NULL; 719 } 720 721 int 722 ahci_port_init(struct ahci_softc *sc, u_int port) 723 { 724 struct ahci_port *ap; 725 u_int64_t dva; 726 u_int32_t cmd; 727 int rc = ENOMEM; 728 729 ap = sc->sc_ports[port]; 730 #ifdef AHCI_DEBUG 731 snprintf(ap->ap_name, sizeof(ap->ap_name), "%s.%d", 732 DEVNAME(sc), port); 733 #endif 734 735 /* Disable port interrupts */ 736 ahci_pwrite(ap, AHCI_PREG_IE, 0); 737 738 /* Sec 10.1.2 - deinitialise port if it is already running */ 739 cmd = ahci_pread(ap, AHCI_PREG_CMD); 740 if (ISSET(cmd, (AHCI_PREG_CMD_ST | AHCI_PREG_CMD_CR | 741 AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_FR)) || 742 ISSET(ahci_pread(ap, AHCI_PREG_SCTL), AHCI_PREG_SCTL_DET)) { 743 int r; 744 745 r = ahci_port_stop(ap, 1); 746 if (r) { 747 printf("%s: unable to disable %s, ignoring port %d\n", 748 DEVNAME(sc), r == 2 ? "CR" : "FR", port); 749 rc = ENXIO; 750 goto reterr; 751 } 752 753 /* Write DET to zero */ 754 ahci_pwrite(ap, AHCI_PREG_SCTL, 0); 755 } 756 757 /* Setup RFIS base address */ 758 ap->ap_rfis = (struct ahci_rfis *) AHCI_DMA_KVA(ap->ap_dmamem_rfis); 759 dva = AHCI_DMA_DVA(ap->ap_dmamem_rfis); 760 ahci_pwrite(ap, AHCI_PREG_FBU, (u_int32_t)(dva >> 32)); 761 ahci_pwrite(ap, AHCI_PREG_FB, (u_int32_t)dva); 762 763 /* Enable FIS reception and activate port. */ 764 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 765 cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD | AHCI_PREG_CMD_SUD; 766 ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_ICC_ACTIVE); 767 768 /* Check whether port activated. Skip it if not. */ 769 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 770 if (!ISSET(cmd, AHCI_PREG_CMD_FRE)) { 771 rc = ENXIO; 772 goto reterr; 773 } 774 775 /* Setup command list base address */ 776 dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_list); 777 ahci_pwrite(ap, AHCI_PREG_CLBU, (u_int32_t)(dva >> 32)); 778 ahci_pwrite(ap, AHCI_PREG_CLB, (u_int32_t)dva); 779 780 /* Wait for ICC change to complete */ 781 ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1); 782 783 /* Reset port */ 784 rc = ahci_port_portreset(ap, 1); 785 switch (rc) { 786 case ENODEV: 787 switch (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) { 788 case AHCI_PREG_SSTS_DET_DEV_NE: 789 printf("%s: device not communicating on port %d\n", 790 DEVNAME(sc), port); 791 break; 792 case AHCI_PREG_SSTS_DET_PHYOFFLINE: 793 printf("%s: PHY offline on port %d\n", DEVNAME(sc), 794 port); 795 break; 796 default: 797 DPRINTF(AHCI_D_VERBOSE, "%s: no device detected " 798 "on port %d\n", DEVNAME(sc), port); 799 break; 800 } 801 goto reterr; 802 803 case EBUSY: 804 printf("%s: device on port %d didn't come ready, " 805 "TFD: 0x%b\n", DEVNAME(sc), port, 806 ahci_pread(ap, AHCI_PREG_TFD), AHCI_PFMT_TFD_STS); 807 808 /* Try a soft reset to clear busy */ 809 rc = ahci_port_softreset(ap); 810 if (rc) { 811 printf("%s: unable to communicate " 812 "with device on port %d\n", DEVNAME(sc), port); 813 goto reterr; 814 } 815 break; 816 817 default: 818 break; 819 } 820 DPRINTF(AHCI_D_VERBOSE, "%s: detected device on port %d\n", 821 DEVNAME(sc), port); 822 823 if (ap->ap_pmp_ports > 0) { 824 int p; 825 826 for (p = 0; p < ap->ap_pmp_ports; p++) { 827 int sig; 828 829 /* might need to do a portreset first here? */ 830 831 /* softreset the port */ 832 if (ahci_pmp_port_softreset(ap, p)) { 833 printf("%s.%d: unable to probe PMP port due to" 834 " softreset failure\n", PORTNAME(ap), p); 835 continue; 836 } 837 838 sig = ahci_port_signature(ap); 839 printf("%s.%d: port signature returned %d\n", 840 PORTNAME(ap), p, sig); 841 } 842 } 843 844 /* Enable command transfers on port */ 845 if (ahci_port_start(ap, 0)) { 846 printf("%s: failed to start command DMA on port %d, " 847 "disabling\n", DEVNAME(sc), port); 848 rc = ENXIO; /* couldn't start port */ 849 } 850 851 /* Flush interrupts for port */ 852 ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS)); 853 ahci_write(sc, AHCI_REG_IS, 1 << port); 854 855 ahci_enable_interrupts(ap); 856 857 reterr: 858 return (rc); 859 } 860 861 int 862 ahci_default_port_start(struct ahci_port *ap, int fre_only) 863 { 864 u_int32_t r; 865 866 /* XXX FBS: possibly turn FBS on here */ 867 868 /* Turn on FRE (and ST) */ 869 r = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 870 r |= AHCI_PREG_CMD_FRE; 871 if (!fre_only) 872 r |= AHCI_PREG_CMD_ST; 873 ahci_pwrite(ap, AHCI_PREG_CMD, r); 874 875 #ifdef AHCI_COALESCE 876 /* (Re-)enable coalescing on the port. */ 877 if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) { 878 ap->ap_sc->sc_ccc_ports_cur |= (1 << ap->ap_num); 879 ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS, 880 ap->ap_sc->sc_ccc_ports_cur); 881 } 882 #endif 883 884 /* Wait for CR to come on */ 885 if (!fre_only && 886 ahci_pwait_set(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CR, 1)) 887 return (1); 888 889 return (0); 890 } 891 892 int 893 ahci_port_stop(struct ahci_port *ap, int stop_fis_rx) 894 { 895 u_int32_t r; 896 897 #ifdef AHCI_COALESCE 898 /* Disable coalescing on the port while it is stopped. */ 899 if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) { 900 ap->ap_sc->sc_ccc_ports_cur &= ~(1 << ap->ap_num); 901 ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS, 902 ap->ap_sc->sc_ccc_ports_cur); 903 } 904 #endif 905 906 /* Turn off ST (and FRE) */ 907 r = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 908 r &= ~AHCI_PREG_CMD_ST; 909 if (stop_fis_rx) 910 r &= ~AHCI_PREG_CMD_FRE; 911 ahci_pwrite(ap, AHCI_PREG_CMD, r); 912 913 /* Wait for CR to go off */ 914 if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CR, 1)) 915 return (1); 916 917 /* Wait for FR to go off */ 918 if (stop_fis_rx && 919 ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_FR, 1)) 920 return (2); 921 922 /* XXX FBS: possibly disable FBS here? */ 923 924 return (0); 925 } 926 927 /* AHCI command list override -> forcibly clear TFD.STS.{BSY,DRQ} */ 928 int 929 ahci_port_clo(struct ahci_port *ap) 930 { 931 struct ahci_softc *sc = ap->ap_sc; 932 u_int32_t cmd; 933 934 /* Only attempt CLO if supported by controller */ 935 if (!ISSET(ahci_read(sc, AHCI_REG_CAP), AHCI_REG_CAP_SCLO)) 936 return (1); 937 938 /* Issue CLO */ 939 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 940 #ifdef DIAGNOSTIC 941 if (ISSET(cmd, AHCI_PREG_CMD_ST)) 942 printf("%s: CLO requested while port running\n", PORTNAME(ap)); 943 #endif 944 ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_CLO); 945 946 /* Wait for completion */ 947 if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CLO, 1)) { 948 printf("%s: CLO did not complete\n", PORTNAME(ap)); 949 return (1); 950 } 951 952 return (0); 953 } 954 955 /* AHCI soft reset, Section 10.4.1 */ 956 int 957 ahci_port_softreset(struct ahci_port *ap) 958 { 959 struct ahci_ccb *ccb = NULL; 960 struct ahci_cmd_hdr *cmd_slot; 961 u_int8_t *fis; 962 int s, rc = EIO, oldstate; 963 u_int32_t cmd; 964 965 DPRINTF(AHCI_D_VERBOSE, "%s: soft reset\n", PORTNAME(ap)); 966 967 s = splbio(); 968 oldstate = ap->ap_state; 969 ap->ap_state = AP_S_ERROR_RECOVERY; 970 971 /* Save previous command register state */ 972 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 973 974 /* Idle port */ 975 if (ahci_port_stop(ap, 0)) { 976 printf("%s: failed to stop port, cannot softreset\n", 977 PORTNAME(ap)); 978 goto err; 979 } 980 981 /* Request CLO if device appears hung */ 982 if (ISSET(ahci_pread(ap, AHCI_PREG_TFD), AHCI_PREG_TFD_STS_BSY | 983 AHCI_PREG_TFD_STS_DRQ)) 984 ahci_port_clo(ap); 985 986 /* Clear port errors to permit TFD transfer */ 987 ahci_pwrite(ap, AHCI_PREG_SERR, ahci_pread(ap, AHCI_PREG_SERR)); 988 989 /* XXX FBS - need to ensure we don't enable FBS here, since we're 990 * resetting stuff 991 * (AHCI spec 9.3.8) 992 */ 993 /* Restart port */ 994 if (ahci_port_start(ap, 0)) { 995 printf("%s: failed to start port, cannot softreset\n", 996 PORTNAME(ap)); 997 goto err; 998 } 999 1000 /* Check whether CLO worked */ 1001 if (ahci_pwait_clr(ap, AHCI_PREG_TFD, 1002 AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)) { 1003 printf("%s: CLO %s, need port reset\n", PORTNAME(ap), 1004 ISSET(ahci_read(ap->ap_sc, AHCI_REG_CAP), AHCI_REG_CAP_SCLO) 1005 ? "failed" : "unsupported"); 1006 rc = EBUSY; 1007 goto err; 1008 } 1009 1010 /* Prep first D2H command with SRST feature & clear busy/reset flags */ 1011 ccb = ahci_get_err_ccb(ap); 1012 cmd_slot = ccb->ccb_cmd_hdr; 1013 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table)); 1014 1015 fis = ccb->ccb_cmd_table->cfis; 1016 fis[0] = ATA_FIS_TYPE_H2D; 1017 fis[15] = ATA_FIS_CONTROL_SRST; 1018 1019 cmd_slot->prdtl = 0; 1020 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ | 1021 AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R | 1022 AHCI_CMD_LIST_FLAG_W); 1023 1024 ccb->ccb_xa.state = ATA_S_PENDING; 1025 if (ahci_poll(ccb, 1000, NULL) != 0) 1026 goto err; 1027 1028 /* Prep second D2H command to read status and complete reset sequence */ 1029 fis[0] = ATA_FIS_TYPE_H2D; 1030 fis[15] = 0; 1031 1032 cmd_slot->prdtl = 0; 1033 htolem16(&cmd_slot->flags, 5 | AHCI_CMD_LIST_FLAG_W); 1034 1035 ccb->ccb_xa.state = ATA_S_PENDING; 1036 if (ahci_poll(ccb, 1000, NULL) != 0) 1037 goto err; 1038 1039 if (ahci_pwait_clr(ap, AHCI_PREG_TFD, AHCI_PREG_TFD_STS_BSY | 1040 AHCI_PREG_TFD_STS_DRQ | AHCI_PREG_TFD_STS_ERR, 1)) { 1041 printf("%s: device didn't come ready after reset, TFD: 0x%b\n", 1042 PORTNAME(ap), ahci_pread(ap, AHCI_PREG_TFD), 1043 AHCI_PFMT_TFD_STS); 1044 rc = EBUSY; 1045 goto err; 1046 } 1047 1048 rc = 0; 1049 err: 1050 if (ccb != NULL) { 1051 /* Abort our command, if it failed, by stopping command DMA. */ 1052 if (rc != 0 && ISSET(ap->ap_active, 1 << ccb->ccb_slot)) { 1053 printf("%s: stopping the port, softreset slot %d was " 1054 "still active.\n", PORTNAME(ap), ccb->ccb_slot); 1055 ahci_port_stop(ap, 0); 1056 } 1057 ccb->ccb_xa.state = ATA_S_ERROR; 1058 ahci_put_err_ccb(ccb); 1059 } 1060 1061 /* Restore saved CMD register state */ 1062 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 1063 ap->ap_state = oldstate; 1064 1065 splx(s); 1066 1067 return (rc); 1068 } 1069 1070 int 1071 ahci_pmp_port_softreset(struct ahci_port *ap, int pmp_port) 1072 { 1073 struct ahci_ccb *ccb = NULL; 1074 u_int32_t data; 1075 int count; 1076 int rc; 1077 int s; 1078 struct ahci_cmd_hdr *cmd_slot; 1079 u_int8_t *fis; 1080 1081 /* XXX FBS: ensure fbs is disabled on ap, since we're resetting 1082 * devices (AHCI section 9.3.8) 1083 */ 1084 1085 s = splbio(); 1086 /* ignore spurious IFS errors while resetting */ 1087 DPRINTF(AHCI_D_VERBOSE, "%s: now ignoring IFS\n", PORTNAME(ap)); 1088 ap->ap_pmp_ignore_ifs = 1; 1089 1090 count = 2; 1091 rc = 0; 1092 do { 1093 if (ccb != NULL) { 1094 ahci_put_pmp_ccb(ccb); 1095 ccb = NULL; 1096 } 1097 1098 if (ahci_pmp_phy_status(ap, pmp_port, &data)) { 1099 printf("%s.%d: unable to clear PHY status\n", 1100 PORTNAME(ap), pmp_port); 1101 } 1102 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 1103 /* maybe don't do this on the first loop: */ 1104 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS); 1105 ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1); 1106 1107 /* send first softreset FIS */ 1108 ccb = ahci_get_pmp_ccb(ap); 1109 cmd_slot = ccb->ccb_cmd_hdr; 1110 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table)); 1111 1112 fis = ccb->ccb_cmd_table->cfis; 1113 fis[0] = ATA_FIS_TYPE_H2D; 1114 fis[1] = pmp_port; 1115 fis[15] = ATA_FIS_CONTROL_SRST | ATA_FIS_CONTROL_4BIT; 1116 1117 cmd_slot->prdtl = 0; 1118 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ | 1119 AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R | 1120 (pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT)); 1121 1122 ccb->ccb_xa.state = ATA_S_PENDING; 1123 1124 DPRINTF(AHCI_D_VERBOSE, "%s.%d: sending PMP softreset cmd\n", 1125 PORTNAME(ap), pmp_port); 1126 if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) { 1127 printf("%s.%d: PMP port softreset cmd failed\n", 1128 PORTNAME(ap), pmp_port); 1129 rc = EBUSY; 1130 if (count > 0) { 1131 /* probably delay a while to allow 1132 * it to settle down? 1133 */ 1134 } 1135 continue; 1136 } 1137 1138 /* send signature FIS */ 1139 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table)); 1140 fis[0] = ATA_FIS_TYPE_H2D; 1141 fis[1] = pmp_port; 1142 fis[15] = ATA_FIS_CONTROL_4BIT; 1143 1144 cmd_slot->prdtl = 0; 1145 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ | 1146 (pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT)); 1147 1148 DPRINTF(AHCI_D_VERBOSE, "%s.%d: sending PMP probe status cmd\n", 1149 PORTNAME(ap), pmp_port); 1150 ccb->ccb_xa.state = ATA_S_PENDING; 1151 if (ahci_poll(ccb, 5000, ahci_pmp_probe_timeout) != 0) { 1152 DPRINTF(AHCI_D_VERBOSE, "%s.%d: PMP probe status cmd " 1153 "failed\n", PORTNAME(ap), pmp_port); 1154 rc = EBUSY; 1155 if (count > 0) { 1156 /* sleep a while? */ 1157 } 1158 continue; 1159 } 1160 1161 fis[15] = 0; 1162 break; 1163 } while (count--); 1164 1165 if (ccb != NULL) { 1166 ahci_put_pmp_ccb(ccb); 1167 ccb = NULL; 1168 } 1169 1170 /* clean up a bit */ 1171 ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1); 1172 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 1173 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS); 1174 ap->ap_pmp_ignore_ifs = 0; 1175 DPRINTF(AHCI_D_VERBOSE, "%s: no longer ignoring IFS\n", PORTNAME(ap)); 1176 splx(s); 1177 1178 return (rc); 1179 } 1180 1181 int 1182 ahci_pmp_port_probe(struct ahci_port *ap, int pmp_port) 1183 { 1184 int sig; 1185 1186 ap->ap_state = AP_S_PMP_PORT_PROBE; 1187 1188 DPRINTF(AHCI_D_VERBOSE, "%s.%d: probing pmp port\n", PORTNAME(ap), 1189 pmp_port); 1190 if (ahci_pmp_port_portreset(ap, pmp_port)) { 1191 printf("%s.%d: unable to probe PMP port; portreset failed\n", 1192 PORTNAME(ap), pmp_port); 1193 ap->ap_state = AP_S_NORMAL; 1194 return (ATA_PORT_T_NONE); 1195 } 1196 1197 if (ahci_pmp_port_softreset(ap, pmp_port)) { 1198 printf("%s.%d: unable to probe PMP port due to softreset " 1199 "failure\n", PORTNAME(ap), pmp_port); 1200 ap->ap_state = AP_S_NORMAL; 1201 return (ATA_PORT_T_NONE); 1202 } 1203 1204 sig = ahci_port_signature(ap); 1205 DPRINTF(AHCI_D_VERBOSE, "%s.%d: port signature returned %d\n", 1206 PORTNAME(ap), pmp_port, sig); 1207 ap->ap_state = AP_S_NORMAL; 1208 return (sig); 1209 } 1210 1211 1212 void 1213 ahci_flush_tfd(struct ahci_port *ap) 1214 { 1215 u_int32_t r; 1216 1217 r = ahci_pread(ap, AHCI_PREG_SERR); 1218 if (r & AHCI_PREG_SERR_DIAG_X) 1219 ahci_pwrite(ap, AHCI_PREG_SERR, AHCI_PREG_SERR_DIAG_X); 1220 } 1221 1222 u_int32_t 1223 ahci_active_mask(struct ahci_port *ap) 1224 { 1225 u_int32_t mask; 1226 1227 mask = ahci_pread(ap, AHCI_PREG_CI); 1228 if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SNCQ) 1229 mask |= ahci_pread(ap, AHCI_PREG_SACT); 1230 return mask; 1231 } 1232 1233 void 1234 ahci_pmp_probe_timeout(void *cookie) 1235 { 1236 struct ahci_ccb *ccb = cookie; 1237 struct ahci_port *ap = ccb->ccb_port; 1238 u_int32_t mask; 1239 1240 DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe cmd timed out\n", PORTNAME(ap)); 1241 switch (ccb->ccb_xa.state) { 1242 case ATA_S_PENDING: 1243 TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry); 1244 ccb->ccb_xa.state = ATA_S_TIMEOUT; 1245 break; 1246 1247 case ATA_S_ONCHIP: 1248 case ATA_S_ERROR: /* currently mostly here for the ATI SBx00 quirk */ 1249 /* clear the command on-chip */ 1250 KASSERT(ap->ap_active == (1 << ccb->ccb_slot) && 1251 ap->ap_sactive == 0); 1252 ahci_port_stop(ap, 0); 1253 ahci_port_start(ap, 0); 1254 1255 if (ahci_active_mask(ap) != 0) { 1256 ahci_port_stop(ap, 0); 1257 ahci_port_start(ap, 0); 1258 mask = ahci_active_mask(ap); 1259 if (mask != 0) { 1260 printf("%s: ahci_pmp_probe_timeout: failed to " 1261 "clear active cmds: %08x\n", PORTNAME(ap), 1262 mask); 1263 } 1264 } 1265 1266 ccb->ccb_xa.state = ATA_S_TIMEOUT; 1267 ap->ap_active &= ~(1 << ccb->ccb_slot); 1268 KASSERT(ap->ap_active_cnt > 0); 1269 --ap->ap_active_cnt; 1270 DPRINTF(AHCI_D_VERBOSE, "%s: timed out %d, active %x, count %d\n", 1271 PORTNAME(ap), ccb->ccb_slot, ap->ap_active, ap->ap_active_cnt); 1272 break; 1273 1274 default: 1275 panic("%s: ahci_pmp_probe_timeout: ccb in bad state %d", 1276 PORTNAME(ap), ccb->ccb_xa.state); 1277 } 1278 } 1279 1280 int 1281 ahci_port_signature(struct ahci_port *ap) 1282 { 1283 u_int32_t sig; 1284 1285 sig = ahci_pread(ap, AHCI_PREG_SIG); 1286 if ((sig & 0xffff0000) == (SATA_SIGNATURE_ATAPI & 0xffff0000)) 1287 return (ATA_PORT_T_ATAPI); 1288 else if ((sig & 0xffff0000) == (SATA_SIGNATURE_PORT_MULTIPLIER & 1289 0xffff0000)) 1290 return (ATA_PORT_T_PM); 1291 else 1292 return (ATA_PORT_T_DISK); 1293 } 1294 1295 int 1296 ahci_pmp_port_portreset(struct ahci_port *ap, int pmp_port) 1297 { 1298 u_int32_t cmd, data; 1299 int loop; 1300 int rc = 1; 1301 int s; 1302 1303 s = splbio(); 1304 DPRINTF(AHCI_D_VERBOSE, "%s.%d: PMP port reset\n", PORTNAME(ap), 1305 pmp_port); 1306 1307 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 1308 1309 /* turn off power management and disable the PHY */ 1310 data = AHCI_PREG_SCTL_IPM_DISABLED; 1311 /* maybe add AHCI_PREG_SCTL_DET_DISABLE */ 1312 if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1)) 1313 goto err; 1314 if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL, data)) 1315 goto err; 1316 delay(10000); 1317 1318 /* start COMRESET */ 1319 data = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_INIT; 1320 if ((ap->ap_sc->sc_dev.dv_cfdata->cf_flags & 0x01) != 0) { 1321 DPRINTF(AHCI_D_VERBOSE, "%s.%d: forcing GEN1\n", PORTNAME(ap), 1322 pmp_port); 1323 data |= AHCI_PREG_SCTL_SPD_GEN1; 1324 } else 1325 data |= AHCI_PREG_SCTL_SPD_ANY; 1326 1327 if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL, data)) 1328 goto err; 1329 1330 /* give it a while to settle down */ 1331 delay(100000); 1332 1333 if (ahci_pmp_phy_status(ap, pmp_port, &data)) { 1334 printf("%s.%d: cannot clear PHY status\n", PORTNAME(ap), 1335 pmp_port); 1336 } 1337 1338 /* start trying to negotiate */ 1339 ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1); 1340 data = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_NONE; 1341 if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL, data)) 1342 goto err; 1343 1344 /* give it a while to detect */ 1345 for (loop = 3; loop; --loop) { 1346 if (ahci_pmp_read(ap, pmp_port, SATA_PMREG_SSTS, &data)) 1347 goto err; 1348 if (data & AHCI_PREG_SSTS_DET) 1349 break; 1350 delay(100000); 1351 } 1352 if (loop == 0) { 1353 printf("%s.%d: port is unplugged\n", PORTNAME(ap), pmp_port); 1354 goto err; 1355 } 1356 1357 /* give it even longer to fully negotiate */ 1358 for (loop = 30; loop; --loop) { 1359 if (ahci_pmp_read(ap, pmp_port, SATA_PMREG_SSTS, &data)) 1360 goto err; 1361 if ((data & AHCI_PREG_SSTS_DET) == AHCI_PREG_SSTS_DET_DEV) 1362 break; 1363 delay(100000); 1364 } 1365 1366 if (loop == 0) { 1367 printf("%s.%d: device is not negotiating\n", PORTNAME(ap), 1368 pmp_port); 1369 goto err; 1370 } 1371 1372 /* device detected */ 1373 DPRINTF(AHCI_D_VERBOSE, "%s.%d: device detected\n", PORTNAME(ap), 1374 pmp_port); 1375 1376 /* clean up a bit */ 1377 delay(100000); 1378 ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1); 1379 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 1380 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS); 1381 1382 rc = 0; 1383 err: 1384 splx(s); 1385 return (rc); 1386 } 1387 1388 /* AHCI port reset, Section 10.4.2 */ 1389 int 1390 ahci_port_portreset(struct ahci_port *ap, int pmp) 1391 { 1392 u_int32_t cmd, r; 1393 int rc, s, retries = 0; 1394 1395 s = splbio(); 1396 DPRINTF(AHCI_D_VERBOSE, "%s: port reset\n", PORTNAME(ap)); 1397 1398 /* Save previous command register state */ 1399 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 1400 1401 /* Clear ST, ignoring failure */ 1402 ahci_port_stop(ap, 0); 1403 1404 /* Perform device detection */ 1405 ahci_pwrite(ap, AHCI_PREG_SCTL, 0); 1406 retry: 1407 delay(10000); 1408 r = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_INIT; 1409 if ((ap->ap_sc->sc_dev.dv_cfdata->cf_flags & 0x01) != 0) { 1410 DPRINTF(AHCI_D_VERBOSE, "%s: forcing GEN1\n", PORTNAME(ap)); 1411 r |= AHCI_PREG_SCTL_SPD_GEN1; 1412 } else 1413 r |= AHCI_PREG_SCTL_SPD_ANY; 1414 ahci_pwrite(ap, AHCI_PREG_SCTL, r); 1415 delay(10000); /* wait at least 1ms for COMRESET to be sent */ 1416 r &= ~AHCI_PREG_SCTL_DET_INIT; 1417 r |= AHCI_PREG_SCTL_DET_NONE; 1418 ahci_pwrite(ap, AHCI_PREG_SCTL, r); 1419 delay(10000); 1420 1421 /* Wait for device to be detected and communications established */ 1422 if (ahci_pwait_eq(ap, AHCI_PREG_SSTS, AHCI_PREG_SSTS_DET, 1423 AHCI_PREG_SSTS_DET_DEV, 1)) { 1424 rc = ENODEV; 1425 if (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) { 1426 /* this may be a port multiplier with no device 1427 * on port 0, so still do the pmp check if requested. 1428 */ 1429 } else { 1430 goto err; 1431 } 1432 } else { 1433 /* Clear SERR (incl X bit), so TFD can update */ 1434 ahci_pwrite(ap, AHCI_PREG_SERR, ahci_pread(ap, AHCI_PREG_SERR)); 1435 1436 /* Wait for device to become ready */ 1437 if (ahci_pwait_clr(ap, AHCI_PREG_TFD, AHCI_PREG_TFD_STS_BSY | 1438 AHCI_PREG_TFD_STS_DRQ | AHCI_PREG_TFD_STS_ERR, 3)) { 1439 /* even if the device doesn't wake up, check if there's 1440 * a port multiplier there 1441 */ 1442 if (retries == 0) { 1443 retries = 1; 1444 goto retry; 1445 } 1446 rc = EBUSY; 1447 } else { 1448 rc = 0; 1449 } 1450 } 1451 1452 if (pmp != 0) { 1453 if (ahci_port_detect_pmp(ap) != 0) { 1454 rc = EBUSY; 1455 } 1456 } 1457 1458 err: 1459 /* Restore preserved port state */ 1460 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 1461 splx(s); 1462 1463 return (rc); 1464 } 1465 1466 int 1467 ahci_port_detect_pmp(struct ahci_port *ap) 1468 { 1469 int count, pmp_rc, rc; 1470 u_int32_t r, cmd; 1471 struct ahci_cmd_hdr *cmd_slot; 1472 struct ahci_ccb *ccb = NULL; 1473 u_int8_t *fis = NULL; 1474 1475 if ((ap->ap_sc->sc_flags & AHCI_F_NO_PMP) || 1476 !ISSET(ahci_read(ap->ap_sc, AHCI_REG_CAP), AHCI_REG_CAP_SPM)) { 1477 return 0; 1478 } 1479 1480 rc = 0; 1481 pmp_rc = 0; 1482 count = 2; 1483 do { 1484 DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe %d\n", PORTNAME(ap), 1485 count); 1486 if (ccb != NULL) { 1487 ahci_put_pmp_ccb(ccb); 1488 ccb = NULL; 1489 } 1490 ahci_port_stop(ap, 0); 1491 ap->ap_state = AP_S_PMP_PROBE; 1492 1493 /* set PMA in cmd reg */ 1494 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 1495 if ((cmd & AHCI_PREG_CMD_PMA) == 0) { 1496 cmd |= AHCI_PREG_CMD_PMA; 1497 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 1498 } 1499 1500 /* Flush errors and request CLO unconditionally, 1501 * then start the port 1502 */ 1503 r = ahci_pread(ap, AHCI_PREG_SERR); 1504 if (r & AHCI_PREG_SERR_DIAG_X) 1505 ahci_pwrite(ap, AHCI_PREG_SERR, 1506 AHCI_PREG_SERR_DIAG_X); 1507 1508 /* Request CLO */ 1509 ahci_port_clo(ap); 1510 1511 /* Clear port errors to permit TFD transfer */ 1512 r = ahci_pread(ap, AHCI_PREG_SERR); 1513 ahci_pwrite(ap, AHCI_PREG_SERR, r); 1514 1515 /* XXX FBS: ensure we don't enable FBS here, since 1516 * we're resetting the port 1517 * (AHCI section 9.3.8) 1518 */ 1519 /* Restart port */ 1520 if (ahci_port_start(ap, 0)) { 1521 rc = EBUSY; 1522 printf("%s: failed to start port, cannot probe PMP\n", 1523 PORTNAME(ap)); 1524 break; 1525 } 1526 1527 /* Check whether CLO worked */ 1528 if (ahci_pwait_clr(ap, AHCI_PREG_TFD, 1529 AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)) { 1530 u_int32_t cap; 1531 1532 cap = ahci_read(ap->ap_sc, AHCI_REG_CAP); 1533 printf("%s: CLO %s, need port reset\n", 1534 PORTNAME(ap), 1535 ISSET(cap, AHCI_REG_CAP_SCLO) 1536 ? "failed" : "unsupported"); 1537 pmp_rc = EBUSY; 1538 break; 1539 } 1540 1541 /* Prep first command with SRST feature & 1542 * clear busy/reset flags 1543 */ 1544 ccb = ahci_get_pmp_ccb(ap); 1545 cmd_slot = ccb->ccb_cmd_hdr; 1546 memset(ccb->ccb_cmd_table, 0, 1547 sizeof(struct ahci_cmd_table)); 1548 1549 fis = ccb->ccb_cmd_table->cfis; 1550 fis[0] = ATA_FIS_TYPE_H2D; 1551 fis[1] = SATA_PMP_CONTROL_PORT; 1552 fis[15] = ATA_FIS_CONTROL_SRST | ATA_FIS_CONTROL_4BIT; 1553 1554 cmd_slot->prdtl = 0; 1555 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ | 1556 AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R | 1557 AHCI_CMD_LIST_FLAG_PMP); 1558 1559 DPRINTF(AHCI_D_VERBOSE, "%s: sending PMP reset cmd\n", 1560 PORTNAME(ap)); 1561 ccb->ccb_xa.state = ATA_S_PENDING; 1562 if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) { 1563 DPRINTF(AHCI_D_VERBOSE, "%s: PMP reset cmd failed\n", 1564 PORTNAME(ap)); 1565 pmp_rc = EBUSY; 1566 continue; 1567 } 1568 1569 if (ahci_pwait_clr(ap, AHCI_PREG_TFD, 1570 AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)) { 1571 printf("%s: port busy after first PMP probe FIS\n", 1572 PORTNAME(ap)); 1573 } 1574 1575 /* clear errors in case the device 1576 * didn't reset cleanly 1577 */ 1578 ahci_flush_tfd(ap); 1579 r = ahci_pread(ap, AHCI_PREG_SERR); 1580 ahci_pwrite(ap, AHCI_PREG_SERR, r); 1581 1582 /* Prep second command to read status and 1583 * complete reset sequence 1584 */ 1585 memset(ccb->ccb_cmd_table, 0, 1586 sizeof(struct ahci_cmd_table)); 1587 fis[0] = ATA_FIS_TYPE_H2D; 1588 fis[1] = SATA_PMP_CONTROL_PORT; 1589 fis[15] = ATA_FIS_CONTROL_4BIT; 1590 1591 cmd_slot->prdtl = 0; 1592 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ | 1593 AHCI_CMD_LIST_FLAG_PMP); 1594 1595 DPRINTF(AHCI_D_VERBOSE, "%s: sending PMP probe status cmd\n", 1596 PORTNAME(ap)); 1597 ccb->ccb_xa.state = ATA_S_PENDING; 1598 if (ahci_poll(ccb, 5000, ahci_pmp_probe_timeout) != 0) { 1599 DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe status " 1600 "cmd failed\n", PORTNAME(ap)); 1601 pmp_rc = EBUSY; 1602 continue; 1603 } 1604 1605 /* apparently we need to retry at least once 1606 * to get the right signature 1607 */ 1608 fis[15] = 0; 1609 pmp_rc = 0; 1610 } while (--count); 1611 1612 if (ccb != NULL) { 1613 ahci_put_pmp_ccb(ccb); 1614 ccb = NULL; 1615 } 1616 1617 if (ap->ap_state == AP_S_PMP_PROBE) { 1618 ap->ap_state = AP_S_NORMAL; 1619 } 1620 1621 if (pmp_rc == 0) { 1622 if (ahci_port_signature(ap) != ATA_PORT_T_PM) { 1623 DPRINTF(AHCI_D_VERBOSE, "%s: device is not a PMP\n", 1624 PORTNAME(ap)); 1625 pmp_rc = EBUSY; 1626 } else { 1627 DPRINTF(AHCI_D_VERBOSE, "%s: PMP found\n", 1628 PORTNAME(ap)); 1629 } 1630 } 1631 1632 if (pmp_rc == 0) { 1633 if (ahci_pmp_identify(ap, &ap->ap_pmp_ports)) { 1634 pmp_rc = EBUSY; 1635 } else { 1636 /* XXX enable FBS if available */ 1637 rc = 0; 1638 } 1639 } 1640 1641 /* if PMP detection failed, so turn off the PMA bit and 1642 * reset the port again 1643 */ 1644 if (pmp_rc != 0) { 1645 DPRINTF(AHCI_D_VERBOSE, "%s: no PMP found, resetting " 1646 "the port\n", PORTNAME(ap)); 1647 ahci_port_stop(ap, 0); 1648 ahci_port_clo(ap); 1649 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 1650 cmd &= ~AHCI_PREG_CMD_PMA; 1651 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 1652 1653 ahci_pwrite(ap, AHCI_PREG_IE, 0); 1654 ahci_port_stop(ap, 0); 1655 if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF) 1656 ahci_pwrite(ap, AHCI_PREG_SNTF, -1); 1657 ahci_flush_tfd(ap); 1658 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 1659 1660 ahci_pwrite(ap, AHCI_PREG_IS, -1); 1661 1662 ahci_enable_interrupts(ap); 1663 1664 ahci_port_portreset(ap, 0); 1665 } 1666 1667 return (rc); 1668 } 1669 1670 void 1671 ahci_load_prdt_seg(struct ahci_prdt *prd, u_int64_t addr, u_int32_t len, 1672 u_int32_t flags) 1673 { 1674 flags |= len - 1; 1675 1676 htolem64(&prd->dba, addr); 1677 htolem32(&prd->flags, flags); 1678 } 1679 1680 int 1681 ahci_load_prdt(struct ahci_ccb *ccb) 1682 { 1683 struct ahci_port *ap = ccb->ccb_port; 1684 struct ahci_softc *sc = ap->ap_sc; 1685 struct ata_xfer *xa = &ccb->ccb_xa; 1686 struct ahci_prdt *prdt = ccb->ccb_cmd_table->prdt; 1687 bus_dmamap_t dmap = ccb->ccb_dmamap; 1688 struct ahci_cmd_hdr *cmd_slot = ccb->ccb_cmd_hdr; 1689 int i, error; 1690 1691 if (xa->datalen == 0) { 1692 ccb->ccb_cmd_hdr->prdtl = 0; 1693 return (0); 1694 } 1695 1696 error = bus_dmamap_load(sc->sc_dmat, dmap, xa->data, xa->datalen, NULL, 1697 (xa->flags & ATA_F_NOWAIT) ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK); 1698 if (error != 0) { 1699 printf("%s: error %d loading dmamap\n", PORTNAME(ap), error); 1700 return (1); 1701 } 1702 1703 for (i = 0; i < dmap->dm_nsegs - 1; i++) { 1704 ahci_load_prdt_seg(&prdt[i], dmap->dm_segs[i].ds_addr, 1705 dmap->dm_segs[i].ds_len, 0); 1706 } 1707 1708 ahci_load_prdt_seg(&prdt[i], 1709 dmap->dm_segs[i].ds_addr, dmap->dm_segs[i].ds_len, 1710 ISSET(xa->flags, ATA_F_PIO) ? AHCI_PRDT_FLAG_INTR : 0); 1711 1712 htolem16(&cmd_slot->prdtl, dmap->dm_nsegs); 1713 1714 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 1715 (xa->flags & ATA_F_READ) ? BUS_DMASYNC_PREREAD : 1716 BUS_DMASYNC_PREWRITE); 1717 1718 return (0); 1719 } 1720 1721 void 1722 ahci_unload_prdt(struct ahci_ccb *ccb) 1723 { 1724 struct ahci_port *ap = ccb->ccb_port; 1725 struct ahci_softc *sc = ap->ap_sc; 1726 struct ata_xfer *xa = &ccb->ccb_xa; 1727 bus_dmamap_t dmap = ccb->ccb_dmamap; 1728 1729 if (xa->datalen != 0) { 1730 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 1731 (xa->flags & ATA_F_READ) ? BUS_DMASYNC_POSTREAD : 1732 BUS_DMASYNC_POSTWRITE); 1733 1734 bus_dmamap_unload(sc->sc_dmat, dmap); 1735 1736 if (ccb->ccb_xa.flags & ATA_F_NCQ) 1737 xa->resid = 0; 1738 else 1739 xa->resid = xa->datalen - 1740 lemtoh32(&ccb->ccb_cmd_hdr->prdbc); 1741 } 1742 } 1743 1744 int 1745 ahci_poll(struct ahci_ccb *ccb, int timeout, void (*timeout_fn)(void *)) 1746 { 1747 struct ahci_port *ap = ccb->ccb_port; 1748 int s; 1749 1750 s = splbio(); 1751 ahci_start(ccb); 1752 do { 1753 if (ISSET(ahci_port_intr(ap, AHCI_PREG_CI_ALL_SLOTS), 1754 1 << ccb->ccb_slot)) { 1755 splx(s); 1756 return (0); 1757 } 1758 if (ccb->ccb_xa.state == ATA_S_ERROR) { 1759 DPRINTF(AHCI_D_VERBOSE, "%s: ccb in slot %d errored\n", 1760 PORTNAME(ap), ccb->ccb_slot); 1761 /* pretend it timed out? */ 1762 if (timeout_fn != NULL) { 1763 timeout_fn(ccb); 1764 } 1765 splx(s); 1766 return (1); 1767 } 1768 1769 delay(1000); 1770 } while (--timeout > 0); 1771 1772 /* Run timeout while at splbio, otherwise ahci_intr could interfere. */ 1773 if (timeout_fn != NULL) 1774 timeout_fn(ccb); 1775 1776 splx(s); 1777 1778 return (1); 1779 } 1780 1781 void 1782 ahci_start(struct ahci_ccb *ccb) 1783 { 1784 struct ahci_port *ap = ccb->ccb_port; 1785 struct ahci_softc *sc = ap->ap_sc; 1786 1787 /* Zero transferred byte count before transfer */ 1788 ccb->ccb_cmd_hdr->prdbc = 0; 1789 1790 /* Sync command list entry and corresponding command table entry */ 1791 bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_cmd_list), 1792 ccb->ccb_slot * sizeof(struct ahci_cmd_hdr), 1793 sizeof(struct ahci_cmd_hdr), BUS_DMASYNC_PREWRITE); 1794 bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_cmd_table), 1795 ccb->ccb_slot * sizeof(struct ahci_cmd_table), 1796 sizeof(struct ahci_cmd_table), BUS_DMASYNC_PREWRITE); 1797 1798 /* Prepare RFIS area for write by controller */ 1799 bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_rfis), 0, 1800 sizeof(struct ahci_rfis), BUS_DMASYNC_PREREAD); 1801 1802 /* XXX FBS: need to figure out whether we still need to keep NCQ and 1803 * non-queued commands separate when FBS is in use. I guess probably 1804 * not? it's not particularly clear from the spec.. 1805 */ 1806 1807 if (ccb->ccb_xa.flags & ATA_F_NCQ) { 1808 /* Issue NCQ commands only when there are no outstanding 1809 * standard commands. */ 1810 if (ap->ap_active != 0 || !TAILQ_EMPTY(&ap->ap_ccb_pending) || 1811 (ap->ap_sactive != 0 && 1812 ap->ap_pmp_ncq_port != ccb->ccb_xa.pmp_port)) { 1813 TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry); 1814 } else { 1815 /* XXX FBS: if using FBS, set AHCI_PREG_FBS_DEV 1816 * to the port number 1817 */ 1818 1819 KASSERT(ap->ap_active_cnt == 0); 1820 ap->ap_sactive |= (1 << ccb->ccb_slot); 1821 ccb->ccb_xa.state = ATA_S_ONCHIP; 1822 ahci_pwrite(ap, AHCI_PREG_SACT, 1 << ccb->ccb_slot); 1823 ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot); 1824 ap->ap_pmp_ncq_port = ccb->ccb_xa.pmp_port; 1825 } 1826 } else { 1827 /* Wait for all NCQ commands to finish before issuing standard 1828 * command. */ 1829 if (ap->ap_sactive != 0 || ap->ap_active_cnt == 2) 1830 TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry); 1831 else if (ap->ap_active_cnt < 2) { 1832 /* XXX FBS: if using FBS, set AHCI_PREG_FBS_DEV to the 1833 * port number 1834 */ 1835 1836 ap->ap_active |= 1 << ccb->ccb_slot; 1837 ccb->ccb_xa.state = ATA_S_ONCHIP; 1838 ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot); 1839 ap->ap_active_cnt++; 1840 } 1841 } 1842 } 1843 1844 void 1845 ahci_issue_pending_ncq_commands(struct ahci_port *ap) 1846 { 1847 struct ahci_ccb *nextccb; 1848 u_int32_t sact_change = 0; 1849 1850 KASSERT(ap->ap_active_cnt == 0); 1851 1852 nextccb = TAILQ_FIRST(&ap->ap_ccb_pending); 1853 if (nextccb == NULL || !(nextccb->ccb_xa.flags & ATA_F_NCQ)) 1854 return; 1855 1856 /* XXX FBS: 1857 * - set AHCI_PREG_FBS_DEV for each command 1858 * - one write to AHCI_PREG_CI per command 1859 */ 1860 1861 /* Start all the NCQ commands at the head of the pending list. 1862 * If a port multiplier is attached to the port, we can only 1863 * issue commands for one of its ports at a time. 1864 */ 1865 if (ap->ap_sactive != 0 && 1866 ap->ap_pmp_ncq_port != nextccb->ccb_xa.pmp_port) { 1867 return; 1868 } 1869 1870 ap->ap_pmp_ncq_port = nextccb->ccb_xa.pmp_port; 1871 do { 1872 TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry); 1873 sact_change |= 1 << nextccb->ccb_slot; 1874 nextccb->ccb_xa.state = ATA_S_ONCHIP; 1875 nextccb = TAILQ_FIRST(&ap->ap_ccb_pending); 1876 } while (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ) && 1877 (nextccb->ccb_xa.pmp_port == ap->ap_pmp_ncq_port)); 1878 1879 ap->ap_sactive |= sact_change; 1880 ahci_pwrite(ap, AHCI_PREG_SACT, sact_change); 1881 ahci_pwrite(ap, AHCI_PREG_CI, sact_change); 1882 } 1883 1884 void 1885 ahci_issue_pending_commands(struct ahci_port *ap, int last_was_ncq) 1886 { 1887 struct ahci_ccb *nextccb; 1888 1889 nextccb = TAILQ_FIRST(&ap->ap_ccb_pending); 1890 if (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ)) { 1891 if (last_was_ncq) { 1892 KASSERT(nextccb->ccb_xa.pmp_port != 1893 ap->ap_pmp_ncq_port); 1894 /* otherwise it should have been started already */ 1895 } else { 1896 ap->ap_active_cnt--; 1897 } 1898 1899 /* Issue NCQ commands only when there are no outstanding 1900 * standard commands, and previous NCQ commands for other 1901 * PMP ports have finished. 1902 */ 1903 if (ap->ap_active == 0) 1904 ahci_issue_pending_ncq_commands(ap); 1905 else 1906 KASSERT(ap->ap_active_cnt == 1); 1907 } else if (nextccb) { 1908 if (ap->ap_sactive != 0 || last_was_ncq) 1909 KASSERT(ap->ap_active_cnt == 0); 1910 1911 /* Wait for all NCQ commands to finish before issuing standard 1912 * command. */ 1913 if (ap->ap_sactive != 0) 1914 return; 1915 1916 /* Keep up to 2 standard commands on-chip at a time. */ 1917 do { 1918 TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry); 1919 ap->ap_active |= 1 << nextccb->ccb_slot; 1920 nextccb->ccb_xa.state = ATA_S_ONCHIP; 1921 /* XXX FBS: set AHCI_PREG_FBS_DEV here */ 1922 ahci_pwrite(ap, AHCI_PREG_CI, 1 << nextccb->ccb_slot); 1923 if (last_was_ncq) 1924 ap->ap_active_cnt++; 1925 if (ap->ap_active_cnt == 2) 1926 break; 1927 KASSERT(ap->ap_active_cnt == 1); 1928 nextccb = TAILQ_FIRST(&ap->ap_ccb_pending); 1929 } while (nextccb && !(nextccb->ccb_xa.flags & ATA_F_NCQ)); 1930 } else if (!last_was_ncq) { 1931 KASSERT(ap->ap_active_cnt == 1 || ap->ap_active_cnt == 2); 1932 1933 /* Standard command finished, none waiting to start. */ 1934 ap->ap_active_cnt--; 1935 } else { 1936 KASSERT(ap->ap_active_cnt == 0); 1937 1938 /* NCQ command finished. */ 1939 } 1940 } 1941 1942 int 1943 ahci_intr(void *arg) 1944 { 1945 struct ahci_softc *sc = arg; 1946 u_int32_t is, ack = 0; 1947 int port; 1948 1949 /* Read global interrupt status */ 1950 is = ahci_read(sc, AHCI_REG_IS); 1951 if (is == 0 || is == 0xffffffff) 1952 return (0); 1953 ack = is; 1954 1955 #ifdef AHCI_COALESCE 1956 /* Check coalescing interrupt first */ 1957 if (is & sc->sc_ccc_mask) { 1958 DPRINTF(AHCI_D_INTR, "%s: command coalescing interrupt\n", 1959 DEVNAME(sc)); 1960 is &= ~sc->sc_ccc_mask; 1961 is |= sc->sc_ccc_ports_cur; 1962 } 1963 #endif 1964 1965 /* Process interrupts for each port */ 1966 while (is) { 1967 port = ffs(is) - 1; 1968 if (sc->sc_ports[port]) 1969 ahci_port_intr(sc->sc_ports[port], 1970 AHCI_PREG_CI_ALL_SLOTS); 1971 is &= ~(1 << port); 1972 } 1973 1974 /* Finally, acknowledge global interrupt */ 1975 ahci_write(sc, AHCI_REG_IS, ack); 1976 1977 return (1); 1978 } 1979 1980 u_int32_t 1981 ahci_port_intr(struct ahci_port *ap, u_int32_t ci_mask) 1982 { 1983 struct ahci_softc *sc = ap->ap_sc; 1984 u_int32_t is, ci_saved, ci_masked, processed = 0; 1985 int slot, need_restart = 0; 1986 int process_error = 0; 1987 struct ahci_ccb *ccb; 1988 volatile u_int32_t *active; 1989 #ifdef DIAGNOSTIC 1990 u_int32_t tmp; 1991 #endif 1992 1993 is = ahci_pread(ap, AHCI_PREG_IS); 1994 1995 /* Ack port interrupt only if checking all command slots. */ 1996 if (ci_mask == AHCI_PREG_CI_ALL_SLOTS) 1997 ahci_pwrite(ap, AHCI_PREG_IS, is); 1998 1999 if (is) 2000 DPRINTF(AHCI_D_INTR, "%s: interrupt: %b\n", PORTNAME(ap), 2001 is, AHCI_PFMT_IS); 2002 2003 if (ap->ap_sactive) { 2004 /* Active NCQ commands - use SActive instead of CI */ 2005 KASSERT(ap->ap_active == 0); 2006 KASSERT(ap->ap_active_cnt == 0); 2007 ci_saved = ahci_pread(ap, AHCI_PREG_SACT); 2008 active = &ap->ap_sactive; 2009 } else { 2010 /* Save CI */ 2011 ci_saved = ahci_pread(ap, AHCI_PREG_CI); 2012 active = &ap->ap_active; 2013 } 2014 2015 if (is & AHCI_PREG_IS_TFES) { 2016 process_error = 1; 2017 } else if (is & AHCI_PREG_IS_DHRS) { 2018 u_int32_t tfd; 2019 u_int32_t cmd; 2020 u_int32_t serr; 2021 2022 tfd = ahci_pread(ap, AHCI_PREG_TFD); 2023 cmd = ahci_pread(ap, AHCI_PREG_CMD); 2024 serr = ahci_pread(ap, AHCI_PREG_SERR); 2025 if ((tfd & AHCI_PREG_TFD_STS_ERR) && 2026 (cmd & AHCI_PREG_CMD_CR) == 0) { 2027 DPRINTF(AHCI_D_VERBOSE, "%s: DHRS error, TFD: %b, SERR:" 2028 " %b, DIAG: %b\n", PORTNAME(ap), tfd, 2029 AHCI_PFMT_TFD_STS, AHCI_PREG_SERR_ERR(serr), 2030 AHCI_PFMT_SERR_ERR, AHCI_PREG_SERR_DIAG(serr), 2031 AHCI_PFMT_SERR_DIAG); 2032 process_error = 1; 2033 } else { 2034 /* rfis copy back is in the normal execution path */ 2035 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_DHRS); 2036 } 2037 } 2038 2039 /* Command failed. See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2. */ 2040 if (process_error) { 2041 u_int32_t tfd, serr; 2042 int err_slot; 2043 2044 tfd = ahci_pread(ap, AHCI_PREG_TFD); 2045 serr = ahci_pread(ap, AHCI_PREG_SERR); 2046 2047 if (ap->ap_sactive == 0) { 2048 /* Errored slot is easy to determine from CMD. */ 2049 err_slot = AHCI_PREG_CMD_CCS(ahci_pread(ap, 2050 AHCI_PREG_CMD)); 2051 2052 if ((ci_saved & (1 << err_slot)) == 0) { 2053 /* 2054 * Hardware doesn't seem to report correct 2055 * slot number. If there's only one 2056 * outstanding command we can cope, 2057 * otherwise fail all active commands. 2058 */ 2059 if (ap->ap_active_cnt == 1) 2060 err_slot = ffs(ap->ap_active) - 1; 2061 else 2062 goto failall; 2063 } 2064 2065 ccb = &ap->ap_ccbs[err_slot]; 2066 2067 /* Preserve received taskfile data from the RFIS. */ 2068 memcpy(&ccb->ccb_xa.rfis, ap->ap_rfis->rfis, 2069 sizeof(struct ata_fis_d2h)); 2070 } else 2071 err_slot = -1; /* Must extract error from log page */ 2072 2073 DPRINTF(AHCI_D_VERBOSE, "%s: errored slot %d, TFD: %b, SERR:" 2074 " %b, DIAG: %b\n", PORTNAME(ap), err_slot, tfd, 2075 AHCI_PFMT_TFD_STS, AHCI_PREG_SERR_ERR(serr), 2076 AHCI_PFMT_SERR_ERR, AHCI_PREG_SERR_DIAG(serr), 2077 AHCI_PFMT_SERR_DIAG); 2078 2079 /* Turn off ST to clear CI and SACT. */ 2080 ahci_port_stop(ap, 0); 2081 need_restart = 1; 2082 2083 /* Clear SERR to enable capturing new errors. */ 2084 ahci_pwrite(ap, AHCI_PREG_SERR, serr); 2085 2086 /* Acknowledge the interrupts we can recover from. */ 2087 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_TFES | 2088 AHCI_PREG_IS_IFS); 2089 is = ahci_pread(ap, AHCI_PREG_IS); 2090 2091 /* If device hasn't cleared its busy status, try to idle it. */ 2092 if (ISSET(tfd, AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) { 2093 2094 if ((ap->ap_state == AP_S_PMP_PORT_PROBE) || 2095 (ap->ap_state == AP_S_ERROR_RECOVERY)) { 2096 /* can't reset the port here, just make sure 2097 * the operation fails and the port still works. 2098 */ 2099 } else if (ap->ap_pmp_ports != 0 && err_slot != -1) { 2100 printf("%s: error on PMP port %d, idling " 2101 "device\n", PORTNAME(ap), 2102 ccb->ccb_xa.pmp_port); 2103 if (ahci_pmp_port_softreset(ap, 2104 ccb->ccb_xa.pmp_port) == 0) { 2105 printf("%s: unable to softreset port " 2106 "%d\n", PORTNAME(ap), 2107 ccb->ccb_xa.pmp_port); 2108 if (ahci_pmp_port_portreset(ap, 2109 ccb->ccb_xa.pmp_port)) { 2110 printf("%s: failed to port " 2111 " reset %d, giving up on " 2112 "it\n", PORTNAME(ap), 2113 ccb->ccb_xa.pmp_port); 2114 goto fatal; 2115 } 2116 } 2117 } else { 2118 printf("%s: attempting to idle device\n", 2119 PORTNAME(ap)); 2120 if (ahci_port_softreset(ap)) { 2121 printf("%s: failed to soft reset " 2122 "device\n", PORTNAME(ap)); 2123 if (ahci_port_portreset(ap, 0)) { 2124 printf("%s: failed to port " 2125 "reset device, give up on " 2126 "it\n", PORTNAME(ap)); 2127 goto fatal; 2128 } 2129 } 2130 } 2131 2132 /* Had to reset device, can't gather extended info. */ 2133 } else if (ap->ap_sactive) { 2134 /* Recover the NCQ error from log page 10h. 2135 * XXX FBS: need to do things to figure out where the 2136 * error came from. without FBS, we know the PMP port 2137 * responsible because we can only have queued commands 2138 * active for one port at a time. 2139 */ 2140 ahci_port_read_ncq_error(ap, &err_slot, 2141 ap->ap_pmp_ncq_port); 2142 if (err_slot < 0) 2143 goto failall; 2144 2145 DPRINTF(AHCI_D_VERBOSE, "%s: NCQ errored slot %d\n", 2146 PORTNAME(ap), err_slot); 2147 2148 ccb = &ap->ap_ccbs[err_slot]; 2149 } else { 2150 /* Didn't reset, could gather extended info from log. */ 2151 } 2152 2153 /* 2154 * If we couldn't determine the errored slot, reset the port 2155 * and fail all the active slots. 2156 */ 2157 if (err_slot == -1) { 2158 if (ahci_port_softreset(ap) != 0 && 2159 ahci_port_portreset(ap, 0) != 0) { 2160 printf("%s: couldn't reset after NCQ error, " 2161 "disabling device.\n", PORTNAME(ap)); 2162 goto fatal; 2163 } 2164 printf("%s: couldn't recover NCQ error, failing " 2165 "all outstanding commands.\n", PORTNAME(ap)); 2166 goto failall; 2167 } 2168 2169 /* Clear the failed command in saved CI so completion runs. */ 2170 ci_saved &= ~(1 << err_slot); 2171 2172 /* Note the error in the ata_xfer. */ 2173 KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP); 2174 ccb->ccb_xa.state = ATA_S_ERROR; 2175 2176 #ifdef DIAGNOSTIC 2177 /* There may only be one outstanding standard command now. */ 2178 if (ap->ap_sactive == 0) { 2179 tmp = ci_saved; 2180 if (tmp) { 2181 slot = ffs(tmp) - 1; 2182 tmp &= ~(1 << slot); 2183 KASSERT(tmp == 0); 2184 } 2185 } 2186 #endif 2187 } 2188 2189 /* ATI SBx00 AHCI controllers respond to PMP probes with IPMS interrupts 2190 * when there's a normal SATA device attached. 2191 */ 2192 if ((ap->ap_state == AP_S_PMP_PROBE) && 2193 (ap->ap_sc->sc_flags & AHCI_F_IPMS_PROBE) && 2194 (is & AHCI_PREG_IS_IPMS)) { 2195 slot = AHCI_PREG_CMD_CCS(ahci_pread(ap, AHCI_PREG_CMD)); 2196 DPRINTF(AHCI_D_INTR, "%s: slot %d received IPMS\n", 2197 PORTNAME(ap), slot); 2198 2199 ccb = &ap->ap_ccbs[slot]; 2200 ccb->ccb_xa.state = ATA_S_ERROR; 2201 2202 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IPMS); 2203 is &= ~AHCI_PREG_IS_IPMS; 2204 } 2205 2206 /* ignore IFS errors while resetting a PMP port */ 2207 if ((is & AHCI_PREG_IS_IFS) /*&& ap->ap_pmp_ignore_ifs*/) { 2208 DPRINTF(AHCI_D_INTR, "%s: ignoring IFS while resetting PMP " 2209 "port\n", PORTNAME(ap)); 2210 2211 need_restart = 1; 2212 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 2213 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS); 2214 is &= ~AHCI_PREG_IS_IFS; 2215 goto failall; 2216 } 2217 2218 /* Check for remaining errors - they are fatal. */ 2219 if (is & (AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS | AHCI_PREG_IS_IFS | 2220 AHCI_PREG_IS_OFS | AHCI_PREG_IS_UFS)) { 2221 printf("%s: unrecoverable errors (IS: %b), disabling port.\n", 2222 PORTNAME(ap), is, AHCI_PFMT_IS); 2223 2224 /* XXX try recovery first */ 2225 goto fatal; 2226 } 2227 2228 /* Fail all outstanding commands if we know the port won't recover. */ 2229 if (ap->ap_state == AP_S_FATAL_ERROR) { 2230 fatal: 2231 ap->ap_state = AP_S_FATAL_ERROR; 2232 failall: 2233 2234 /* Ensure port is shut down. */ 2235 ahci_port_stop(ap, 1); 2236 2237 /* Error all the active slots. */ 2238 ci_masked = ci_saved & *active; 2239 while (ci_masked) { 2240 slot = ffs(ci_masked) - 1; 2241 ccb = &ap->ap_ccbs[slot]; 2242 ci_masked &= ~(1 << slot); 2243 ccb->ccb_xa.state = ATA_S_ERROR; 2244 } 2245 2246 /* Run completion for all active slots. */ 2247 ci_saved &= ~*active; 2248 2249 /* Don't restart the port if our problems were deemed fatal. */ 2250 if (ap->ap_state == AP_S_FATAL_ERROR) 2251 need_restart = 0; 2252 } 2253 2254 /* 2255 * CCB completion is detected by noticing its slot's bit in CI has 2256 * changed to zero some time after we activated it. 2257 * If we are polling, we may only be interested in particular slot(s). 2258 */ 2259 ci_masked = ~ci_saved & *active & ci_mask; 2260 while (ci_masked) { 2261 slot = ffs(ci_masked) - 1; 2262 ccb = &ap->ap_ccbs[slot]; 2263 ci_masked &= ~(1 << slot); 2264 2265 DPRINTF(AHCI_D_INTR, "%s: slot %d is complete%s\n", 2266 PORTNAME(ap), slot, ccb->ccb_xa.state == ATA_S_ERROR ? 2267 " (error)" : ""); 2268 2269 bus_dmamap_sync(sc->sc_dmat, 2270 AHCI_DMA_MAP(ap->ap_dmamem_cmd_list), 2271 ccb->ccb_slot * sizeof(struct ahci_cmd_hdr), 2272 sizeof(struct ahci_cmd_hdr), BUS_DMASYNC_POSTWRITE); 2273 2274 bus_dmamap_sync(sc->sc_dmat, 2275 AHCI_DMA_MAP(ap->ap_dmamem_cmd_table), 2276 ccb->ccb_slot * sizeof(struct ahci_cmd_table), 2277 sizeof(struct ahci_cmd_table), BUS_DMASYNC_POSTWRITE); 2278 2279 bus_dmamap_sync(sc->sc_dmat, 2280 AHCI_DMA_MAP(ap->ap_dmamem_rfis), 0, 2281 sizeof(struct ahci_rfis), BUS_DMASYNC_POSTREAD); 2282 2283 *active &= ~(1 << ccb->ccb_slot); 2284 /* Copy the rfis into the ccb if we were asked for it */ 2285 if (ccb->ccb_xa.state == ATA_S_ONCHIP && 2286 ccb->ccb_xa.flags & ATA_F_GET_RFIS) { 2287 memcpy(&ccb->ccb_xa.rfis, 2288 ap->ap_rfis->rfis, 2289 sizeof(struct ata_fis_d2h)); 2290 } 2291 2292 ccb->ccb_done(ccb); 2293 2294 processed |= 1 << ccb->ccb_slot; 2295 } 2296 2297 if (need_restart) { 2298 /* Restart command DMA on the port */ 2299 ahci_port_start(ap, 0); 2300 2301 /* Re-enable outstanding commands on port. */ 2302 if (ci_saved) { 2303 #ifdef DIAGNOSTIC 2304 tmp = ci_saved; 2305 while (tmp) { 2306 slot = ffs(tmp) - 1; 2307 tmp &= ~(1 << slot); 2308 ccb = &ap->ap_ccbs[slot]; 2309 KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP); 2310 KASSERT((!!(ccb->ccb_xa.flags & ATA_F_NCQ)) == 2311 (!!ap->ap_sactive)); 2312 } 2313 #endif 2314 DPRINTF(AHCI_D_VERBOSE, "%s: ahci_port_intr " 2315 "re-enabling%s slots %08x\n", PORTNAME(ap), 2316 ap->ap_sactive ? " NCQ" : "", ci_saved); 2317 2318 /* XXX FBS: 2319 * - need to set AHCI_PREG_FBS_DEV for each command 2320 * - can't do multiple commands with a single write to 2321 * AHCI_PREG_CI 2322 */ 2323 if (ap->ap_sactive) 2324 ahci_pwrite(ap, AHCI_PREG_SACT, ci_saved); 2325 ahci_pwrite(ap, AHCI_PREG_CI, ci_saved); 2326 } 2327 } 2328 2329 return (processed); 2330 } 2331 2332 struct ahci_ccb * 2333 ahci_get_ccb(struct ahci_port *ap) 2334 { 2335 struct ahci_ccb *ccb; 2336 2337 mtx_enter(&ap->ap_ccb_mtx); 2338 ccb = TAILQ_FIRST(&ap->ap_ccb_free); 2339 if (ccb != NULL) { 2340 KASSERT(ccb->ccb_xa.state == ATA_S_PUT); 2341 TAILQ_REMOVE(&ap->ap_ccb_free, ccb, ccb_entry); 2342 ccb->ccb_xa.state = ATA_S_SETUP; 2343 } 2344 mtx_leave(&ap->ap_ccb_mtx); 2345 2346 return (ccb); 2347 } 2348 2349 void 2350 ahci_put_ccb(struct ahci_ccb *ccb) 2351 { 2352 struct ahci_port *ap = ccb->ccb_port; 2353 2354 #ifdef DIAGNOSTIC 2355 if (ccb->ccb_xa.state != ATA_S_COMPLETE && 2356 ccb->ccb_xa.state != ATA_S_TIMEOUT && 2357 ccb->ccb_xa.state != ATA_S_ERROR) { 2358 printf("%s: invalid ata_xfer state %02x in ahci_put_ccb, " 2359 "slot %d\n", PORTNAME(ccb->ccb_port), ccb->ccb_xa.state, 2360 ccb->ccb_slot); 2361 } 2362 #endif 2363 2364 ccb->ccb_xa.state = ATA_S_PUT; 2365 mtx_enter(&ap->ap_ccb_mtx); 2366 TAILQ_INSERT_TAIL(&ap->ap_ccb_free, ccb, ccb_entry); 2367 mtx_leave(&ap->ap_ccb_mtx); 2368 } 2369 2370 struct ahci_ccb * 2371 ahci_get_err_ccb(struct ahci_port *ap) 2372 { 2373 struct ahci_ccb *err_ccb; 2374 u_int32_t sact; 2375 2376 splassert(IPL_BIO); 2377 2378 /* No commands may be active on the chip. */ 2379 sact = ahci_pread(ap, AHCI_PREG_SACT); 2380 if (sact != 0) 2381 printf("ahci_get_err_ccb but SACT %08x != 0?\n", sact); 2382 KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0); 2383 2384 #ifdef DIAGNOSTIC 2385 KASSERT(ap->ap_err_busy == 0); 2386 ap->ap_err_busy = 1; 2387 #endif 2388 /* Save outstanding command state. */ 2389 ap->ap_err_saved_active = ap->ap_active; 2390 ap->ap_err_saved_active_cnt = ap->ap_active_cnt; 2391 ap->ap_err_saved_sactive = ap->ap_sactive; 2392 2393 /* 2394 * Pretend we have no commands outstanding, so that completions won't 2395 * run prematurely. 2396 */ 2397 ap->ap_active = ap->ap_active_cnt = ap->ap_sactive = 0; 2398 2399 /* 2400 * Grab a CCB to use for error recovery. This should never fail, as 2401 * we ask atascsi to reserve one for us at init time. 2402 */ 2403 err_ccb = ap->ap_ccb_err; 2404 err_ccb->ccb_xa.flags = 0; 2405 err_ccb->ccb_xa.state = ATA_S_SETUP; 2406 err_ccb->ccb_done = ahci_empty_done; 2407 2408 return (err_ccb); 2409 } 2410 2411 void 2412 ahci_put_err_ccb(struct ahci_ccb *ccb) 2413 { 2414 struct ahci_port *ap = ccb->ccb_port; 2415 u_int32_t sact; 2416 2417 splassert(IPL_BIO); 2418 2419 #ifdef DIAGNOSTIC 2420 KASSERT(ap->ap_err_busy); 2421 #endif 2422 /* No commands may be active on the chip */ 2423 sact = ahci_pread(ap, AHCI_PREG_SACT); 2424 if (sact != 0) 2425 printf("ahci_port_err_ccb_restore but SACT %08x != 0?\n", sact); 2426 KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0); 2427 2428 #ifdef DIAGNOSTIC 2429 /* Done with the CCB */ 2430 KASSERT(ccb == ap->ap_ccb_err); 2431 #endif 2432 2433 /* Restore outstanding command state */ 2434 ap->ap_sactive = ap->ap_err_saved_sactive; 2435 ap->ap_active_cnt = ap->ap_err_saved_active_cnt; 2436 ap->ap_active = ap->ap_err_saved_active; 2437 2438 #ifdef DIAGNOSTIC 2439 ap->ap_err_busy = 0; 2440 #endif 2441 } 2442 2443 struct ahci_ccb * 2444 ahci_get_pmp_ccb(struct ahci_port *ap) 2445 { 2446 struct ahci_ccb *ccb; 2447 u_int32_t sact; 2448 2449 /* some PMP commands need to be issued on slot 1, 2450 * particularly the command that clears SRST and 2451 * fetches the device signature. 2452 * 2453 * ensure the chip is idle and ccb 1 is available. 2454 */ 2455 splassert(IPL_BIO); 2456 2457 sact = ahci_pread(ap, AHCI_PREG_SACT); 2458 if (sact != 0) 2459 printf("ahci_get_pmp_ccb; SACT %08x != 0\n", sact); 2460 KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0); 2461 2462 ccb = &ap->ap_ccbs[1]; 2463 KASSERT(ccb->ccb_xa.state == ATA_S_PUT); 2464 ccb->ccb_xa.flags = 0; 2465 ccb->ccb_done = ahci_pmp_cmd_done; 2466 2467 mtx_enter(&ap->ap_ccb_mtx); 2468 TAILQ_REMOVE(&ap->ap_ccb_free, ccb, ccb_entry); 2469 mtx_leave(&ap->ap_ccb_mtx); 2470 2471 return ccb; 2472 } 2473 2474 void 2475 ahci_put_pmp_ccb(struct ahci_ccb *ccb) 2476 { 2477 struct ahci_port *ap = ccb->ccb_port; 2478 u_int32_t sact; 2479 2480 /* make sure this is the right ccb */ 2481 KASSERT(ccb == &ap->ap_ccbs[1]); 2482 2483 /* No commands may be active on the chip */ 2484 sact = ahci_pread(ap, AHCI_PREG_SACT); 2485 if (sact != 0) 2486 printf("ahci_port_err_ccb_restore but SACT %08x != 0?\n", sact); 2487 KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0); 2488 2489 ccb->ccb_xa.state = ATA_S_PUT; 2490 mtx_enter(&ap->ap_ccb_mtx); 2491 TAILQ_INSERT_TAIL(&ap->ap_ccb_free, ccb, ccb_entry); 2492 mtx_leave(&ap->ap_ccb_mtx); 2493 } 2494 2495 int 2496 ahci_port_read_ncq_error(struct ahci_port *ap, int *err_slotp, int pmp_port) 2497 { 2498 struct ahci_ccb *ccb; 2499 struct ahci_cmd_hdr *cmd_slot; 2500 u_int32_t cmd; 2501 struct ata_fis_h2d *fis; 2502 int rc = EIO, oldstate; 2503 2504 DPRINTF(AHCI_D_VERBOSE, "%s: read log page\n", PORTNAME(ap)); 2505 oldstate = ap->ap_state; 2506 ap->ap_state = AP_S_ERROR_RECOVERY; 2507 2508 /* Save command register state. */ 2509 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 2510 2511 /* Port should have been idled already. Start it. */ 2512 KASSERT((cmd & AHCI_PREG_CMD_CR) == 0); 2513 ahci_port_start(ap, 0); 2514 2515 /* Prep error CCB for READ LOG EXT, page 10h, 1 sector. */ 2516 ccb = ahci_get_err_ccb(ap); 2517 ccb->ccb_xa.flags = ATA_F_NOWAIT | ATA_F_READ | ATA_F_POLL; 2518 ccb->ccb_xa.data = ap->ap_err_scratch; 2519 ccb->ccb_xa.datalen = 512; 2520 cmd_slot = ccb->ccb_cmd_hdr; 2521 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table)); 2522 2523 fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis; 2524 fis->type = ATA_FIS_TYPE_H2D; 2525 fis->flags = ATA_H2D_FLAGS_CMD | pmp_port; 2526 fis->command = ATA_C_READ_LOG_EXT; 2527 fis->lba_low = 0x10; /* queued error log page (10h) */ 2528 fis->sector_count = 1; /* number of sectors (1) */ 2529 fis->sector_count_exp = 0; 2530 fis->lba_mid = 0; /* starting offset */ 2531 fis->lba_mid_exp = 0; 2532 fis->device = 0; 2533 2534 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ | 2535 (pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT)); 2536 2537 if (ahci_load_prdt(ccb) != 0) { 2538 rc = ENOMEM; /* XXX caller must abort all commands */ 2539 goto err; 2540 } 2541 2542 ccb->ccb_xa.state = ATA_S_PENDING; 2543 if (ahci_poll(ccb, 1000, NULL) != 0 || 2544 ccb->ccb_xa.state == ATA_S_ERROR) 2545 goto err; 2546 2547 rc = 0; 2548 err: 2549 /* Abort our command, if it failed, by stopping command DMA. */ 2550 if (rc != 0 && ISSET(ap->ap_active, 1 << ccb->ccb_slot)) { 2551 printf("%s: log page read failed, slot %d was still active.\n", 2552 PORTNAME(ap), ccb->ccb_slot); 2553 ahci_port_stop(ap, 0); 2554 } 2555 2556 /* Done with the error CCB now. */ 2557 ahci_unload_prdt(ccb); 2558 ahci_put_err_ccb(ccb); 2559 2560 /* Extract failed register set and tags from the scratch space. */ 2561 if (rc == 0) { 2562 struct ata_log_page_10h *log; 2563 int err_slot; 2564 2565 log = (struct ata_log_page_10h *)ap->ap_err_scratch; 2566 if (ISSET(log->err_regs.type, ATA_LOG_10H_TYPE_NOTQUEUED)) { 2567 /* Not queued bit was set - wasn't an NCQ error? */ 2568 printf("%s: read NCQ error page, but not an NCQ " 2569 "error?\n", PORTNAME(ap)); 2570 rc = ESRCH; 2571 } else { 2572 /* Copy back the log record as a D2H register FIS. */ 2573 *err_slotp = err_slot = log->err_regs.type & 2574 ATA_LOG_10H_TYPE_TAG_MASK; 2575 2576 ccb = &ap->ap_ccbs[err_slot]; 2577 memcpy(&ccb->ccb_xa.rfis, &log->err_regs, 2578 sizeof(struct ata_fis_d2h)); 2579 ccb->ccb_xa.rfis.type = ATA_FIS_TYPE_D2H; 2580 ccb->ccb_xa.rfis.flags = 0; 2581 } 2582 } 2583 2584 /* Restore saved CMD register state */ 2585 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 2586 ap->ap_state = oldstate; 2587 2588 return (rc); 2589 } 2590 2591 struct ahci_dmamem * 2592 ahci_dmamem_alloc(struct ahci_softc *sc, size_t size) 2593 { 2594 struct ahci_dmamem *adm; 2595 int nsegs; 2596 2597 adm = malloc(sizeof(*adm), M_DEVBUF, M_NOWAIT | M_ZERO); 2598 if (adm == NULL) 2599 return (NULL); 2600 2601 adm->adm_size = size; 2602 2603 if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, 2604 BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, &adm->adm_map) != 0) 2605 goto admfree; 2606 2607 if (bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, &adm->adm_seg, 2608 1, &nsegs, BUS_DMA_NOWAIT | BUS_DMA_ZERO) != 0) 2609 goto destroy; 2610 2611 if (bus_dmamem_map(sc->sc_dmat, &adm->adm_seg, nsegs, size, 2612 &adm->adm_kva, BUS_DMA_NOWAIT) != 0) 2613 goto free; 2614 2615 if (bus_dmamap_load(sc->sc_dmat, adm->adm_map, adm->adm_kva, size, 2616 NULL, BUS_DMA_NOWAIT) != 0) 2617 goto unmap; 2618 2619 return (adm); 2620 2621 unmap: 2622 bus_dmamem_unmap(sc->sc_dmat, adm->adm_kva, size); 2623 free: 2624 bus_dmamem_free(sc->sc_dmat, &adm->adm_seg, 1); 2625 destroy: 2626 bus_dmamap_destroy(sc->sc_dmat, adm->adm_map); 2627 admfree: 2628 free(adm, M_DEVBUF, 0); 2629 2630 return (NULL); 2631 } 2632 2633 void 2634 ahci_dmamem_free(struct ahci_softc *sc, struct ahci_dmamem *adm) 2635 { 2636 bus_dmamap_unload(sc->sc_dmat, adm->adm_map); 2637 bus_dmamem_unmap(sc->sc_dmat, adm->adm_kva, adm->adm_size); 2638 bus_dmamem_free(sc->sc_dmat, &adm->adm_seg, 1); 2639 bus_dmamap_destroy(sc->sc_dmat, adm->adm_map); 2640 free(adm, M_DEVBUF, 0); 2641 } 2642 2643 u_int32_t 2644 ahci_read(struct ahci_softc *sc, bus_size_t r) 2645 { 2646 bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4, 2647 BUS_SPACE_BARRIER_READ); 2648 return (bus_space_read_4(sc->sc_iot, sc->sc_ioh, r)); 2649 } 2650 2651 void 2652 ahci_write(struct ahci_softc *sc, bus_size_t r, u_int32_t v) 2653 { 2654 bus_space_write_4(sc->sc_iot, sc->sc_ioh, r, v); 2655 bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4, 2656 BUS_SPACE_BARRIER_WRITE); 2657 } 2658 2659 int 2660 ahci_wait_ne(struct ahci_softc *sc, bus_size_t r, u_int32_t mask, 2661 u_int32_t target) 2662 { 2663 int i; 2664 2665 for (i = 0; i < 1000; i++) { 2666 if ((ahci_read(sc, r) & mask) != target) 2667 return (0); 2668 delay(1000); 2669 } 2670 2671 return (1); 2672 } 2673 2674 u_int32_t 2675 ahci_pread(struct ahci_port *ap, bus_size_t r) 2676 { 2677 bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4, 2678 BUS_SPACE_BARRIER_READ); 2679 return (bus_space_read_4(ap->ap_sc->sc_iot, ap->ap_ioh, r)); 2680 } 2681 2682 void 2683 ahci_pwrite(struct ahci_port *ap, bus_size_t r, u_int32_t v) 2684 { 2685 bus_space_write_4(ap->ap_sc->sc_iot, ap->ap_ioh, r, v); 2686 bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4, 2687 BUS_SPACE_BARRIER_WRITE); 2688 } 2689 2690 int 2691 ahci_pwait_eq(struct ahci_port *ap, bus_size_t r, u_int32_t mask, 2692 u_int32_t target, int n) 2693 { 2694 int i; 2695 2696 for (i = 0; i < n * 1000; i++) { 2697 if ((ahci_pread(ap, r) & mask) == target) 2698 return (0); 2699 delay(1000); 2700 } 2701 2702 return (1); 2703 } 2704 2705 int 2706 ahci_ata_probe(void *xsc, int port, int lun) 2707 { 2708 struct ahci_softc *sc = xsc; 2709 struct ahci_port *ap = sc->sc_ports[port]; 2710 2711 if (ap == NULL) 2712 return (ATA_PORT_T_NONE); 2713 2714 if (lun != 0) { 2715 int pmp_port = lun - 1; 2716 if (pmp_port >= ap->ap_pmp_ports) { 2717 return (ATA_PORT_T_NONE); 2718 } 2719 return (ahci_pmp_port_probe(ap, pmp_port)); 2720 } else { 2721 return (ahci_port_signature(ap)); 2722 } 2723 } 2724 2725 void 2726 ahci_ata_free(void *xsc, int port, int lun) 2727 { 2728 2729 } 2730 2731 struct ata_xfer * 2732 ahci_ata_get_xfer(void *aaa_cookie, int port) 2733 { 2734 struct ahci_softc *sc = aaa_cookie; 2735 struct ahci_port *ap = sc->sc_ports[port]; 2736 struct ahci_ccb *ccb; 2737 2738 ccb = ahci_get_ccb(ap); 2739 if (ccb == NULL) { 2740 DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer: NULL ccb\n", 2741 PORTNAME(ap)); 2742 return (NULL); 2743 } 2744 2745 DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer got slot %d\n", 2746 PORTNAME(ap), ccb->ccb_slot); 2747 2748 return ((struct ata_xfer *)ccb); 2749 } 2750 2751 void 2752 ahci_ata_put_xfer(struct ata_xfer *xa) 2753 { 2754 struct ahci_ccb *ccb = (struct ahci_ccb *)xa; 2755 2756 DPRINTF(AHCI_D_XFER, "ahci_ata_put_xfer slot %d\n", ccb->ccb_slot); 2757 2758 ahci_put_ccb(ccb); 2759 } 2760 2761 void 2762 ahci_ata_cmd(struct ata_xfer *xa) 2763 { 2764 struct ahci_ccb *ccb = (struct ahci_ccb *)xa; 2765 struct ahci_cmd_hdr *cmd_slot; 2766 int s; 2767 u_int16_t flags; 2768 2769 if (ccb->ccb_port->ap_state == AP_S_FATAL_ERROR) 2770 goto failcmd; 2771 2772 ccb->ccb_done = ahci_ata_cmd_done; 2773 2774 cmd_slot = ccb->ccb_cmd_hdr; 2775 flags = 5 /* FIS length (in DWORDs) */; 2776 flags |= xa->pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT; 2777 2778 if (xa->flags & ATA_F_WRITE) 2779 flags |= AHCI_CMD_LIST_FLAG_W; 2780 2781 if (xa->flags & ATA_F_PACKET) 2782 flags |= AHCI_CMD_LIST_FLAG_A; 2783 2784 htolem16(&cmd_slot->flags, flags); 2785 2786 if (ahci_load_prdt(ccb) != 0) 2787 goto failcmd; 2788 2789 timeout_set(&xa->stimeout, ahci_ata_cmd_timeout, ccb); 2790 2791 xa->state = ATA_S_PENDING; 2792 2793 if (xa->flags & ATA_F_POLL) 2794 ahci_poll(ccb, xa->timeout, ahci_ata_cmd_timeout); 2795 else { 2796 s = splbio(); 2797 timeout_add_msec(&xa->stimeout, xa->timeout); 2798 ahci_start(ccb); 2799 splx(s); 2800 } 2801 2802 return; 2803 2804 failcmd: 2805 s = splbio(); 2806 xa->state = ATA_S_ERROR; 2807 ata_complete(xa); 2808 splx(s); 2809 } 2810 2811 void 2812 ahci_pmp_cmd_done(struct ahci_ccb *ccb) 2813 { 2814 struct ata_xfer *xa = &ccb->ccb_xa; 2815 2816 if (xa->state == ATA_S_ONCHIP || xa->state == ATA_S_ERROR) 2817 ahci_issue_pending_commands(ccb->ccb_port, 2818 xa->flags & ATA_F_NCQ); 2819 2820 xa->state = ATA_S_COMPLETE; 2821 } 2822 2823 2824 void 2825 ahci_ata_cmd_done(struct ahci_ccb *ccb) 2826 { 2827 struct ata_xfer *xa = &ccb->ccb_xa; 2828 2829 timeout_del(&xa->stimeout); 2830 2831 if (xa->state == ATA_S_ONCHIP || xa->state == ATA_S_ERROR) 2832 ahci_issue_pending_commands(ccb->ccb_port, 2833 xa->flags & ATA_F_NCQ); 2834 2835 ahci_unload_prdt(ccb); 2836 2837 if (xa->state == ATA_S_ONCHIP) 2838 xa->state = ATA_S_COMPLETE; 2839 #ifdef DIAGNOSTIC 2840 else if (xa->state != ATA_S_ERROR && xa->state != ATA_S_TIMEOUT) 2841 printf("%s: invalid ata_xfer state %02x in ahci_ata_cmd_done, " 2842 "slot %d\n", PORTNAME(ccb->ccb_port), xa->state, 2843 ccb->ccb_slot); 2844 #endif 2845 if (xa->state != ATA_S_TIMEOUT) 2846 ata_complete(xa); 2847 } 2848 2849 void 2850 ahci_ata_cmd_timeout(void *arg) 2851 { 2852 struct ahci_ccb *ccb = arg; 2853 struct ata_xfer *xa = &ccb->ccb_xa; 2854 struct ahci_port *ap = ccb->ccb_port; 2855 int s, ccb_was_started, ncq_cmd; 2856 volatile u_int32_t *active; 2857 2858 s = splbio(); 2859 2860 ncq_cmd = (xa->flags & ATA_F_NCQ); 2861 active = ncq_cmd ? &ap->ap_sactive : &ap->ap_active; 2862 2863 if (ccb->ccb_xa.state == ATA_S_PENDING) { 2864 DPRINTF(AHCI_D_TIMEOUT, "%s: command for slot %d timed out " 2865 "before it got on chip\n", PORTNAME(ap), ccb->ccb_slot); 2866 TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry); 2867 ccb_was_started = 0; 2868 } else if (ccb->ccb_xa.state == ATA_S_ONCHIP && ahci_port_intr(ap, 2869 1 << ccb->ccb_slot)) { 2870 DPRINTF(AHCI_D_TIMEOUT, "%s: final poll of port completed " 2871 "command in slot %d\n", PORTNAME(ap), ccb->ccb_slot); 2872 goto ret; 2873 } else if (ccb->ccb_xa.state != ATA_S_ONCHIP) { 2874 DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d already " 2875 "handled%s\n", PORTNAME(ap), ccb->ccb_slot, 2876 ISSET(*active, 1 << ccb->ccb_slot) ? 2877 " but slot is still active?" : "."); 2878 goto ret; 2879 } else if (!ISSET(ahci_pread(ap, ncq_cmd ? AHCI_PREG_SACT : 2880 AHCI_PREG_CI), 1 << ccb->ccb_slot) && ISSET(*active, 2881 1 << ccb->ccb_slot)) { 2882 DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d completed but " 2883 "IRQ handler didn't detect it. Why?\n", PORTNAME(ap), 2884 ccb->ccb_slot); 2885 *active &= ~(1 << ccb->ccb_slot); 2886 ccb->ccb_done(ccb); 2887 goto ret; 2888 } else { 2889 ccb_was_started = 1; 2890 } 2891 2892 /* Complete the slot with a timeout error. */ 2893 ccb->ccb_xa.state = ATA_S_TIMEOUT; 2894 *active &= ~(1 << ccb->ccb_slot); 2895 DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (1)\n", PORTNAME(ap)); 2896 ccb->ccb_done(ccb); /* This won't issue pending commands or run the 2897 atascsi completion. */ 2898 2899 /* Reset port to abort running command. */ 2900 if (ccb_was_started) { 2901 DPRINTF(AHCI_D_TIMEOUT, "%s: resetting port to abort%s command " 2902 "in slot %d, pmp port %d, active %08x\n", PORTNAME(ap), 2903 ncq_cmd ? " NCQ" : "", ccb->ccb_slot, xa->pmp_port, *active); 2904 if (ahci_port_softreset(ap) != 0 && ahci_port_portreset(ap, 0) 2905 != 0) { 2906 printf("%s: failed to reset port during timeout " 2907 "handling, disabling it\n", PORTNAME(ap)); 2908 ap->ap_state = AP_S_FATAL_ERROR; 2909 } 2910 2911 /* Restart any other commands that were aborted by the reset. */ 2912 if (*active) { 2913 DPRINTF(AHCI_D_TIMEOUT, "%s: re-enabling%s slots " 2914 "%08x\n", PORTNAME(ap), ncq_cmd ? " NCQ" : "", 2915 *active); 2916 if (ncq_cmd) 2917 ahci_pwrite(ap, AHCI_PREG_SACT, *active); 2918 ahci_pwrite(ap, AHCI_PREG_CI, *active); 2919 } 2920 } 2921 2922 /* Issue any pending commands now. */ 2923 DPRINTF(AHCI_D_TIMEOUT, "%s: issue pending\n", PORTNAME(ap)); 2924 if (ccb_was_started) 2925 ahci_issue_pending_commands(ap, ncq_cmd); 2926 else if (ap->ap_active == 0) 2927 ahci_issue_pending_ncq_commands(ap); 2928 2929 /* Complete the timed out ata_xfer I/O (may generate new I/O). */ 2930 DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (2)\n", PORTNAME(ap)); 2931 ata_complete(xa); 2932 2933 DPRINTF(AHCI_D_TIMEOUT, "%s: splx\n", PORTNAME(ap)); 2934 ret: 2935 splx(s); 2936 } 2937 2938 void 2939 ahci_empty_done(struct ahci_ccb *ccb) 2940 { 2941 if (ccb->ccb_xa.state != ATA_S_ERROR) 2942 ccb->ccb_xa.state = ATA_S_COMPLETE; 2943 } 2944 2945 int 2946 ahci_pmp_read(struct ahci_port *ap, int target, int which, u_int32_t *datap) 2947 { 2948 struct ahci_ccb *ccb; 2949 struct ata_fis_h2d *fis; 2950 int error; 2951 2952 ccb = ahci_get_pmp_ccb(ap); 2953 if (ccb == NULL) { 2954 printf("%s: NULL ccb!\n", PORTNAME(ap)); 2955 return (1); 2956 } 2957 ccb->ccb_xa.flags = ATA_F_POLL | ATA_F_GET_RFIS; 2958 ccb->ccb_xa.pmp_port = SATA_PMP_CONTROL_PORT; 2959 ccb->ccb_xa.state = ATA_S_PENDING; 2960 2961 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table)); 2962 fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis; 2963 fis->type = ATA_FIS_TYPE_H2D; 2964 fis->flags = ATA_H2D_FLAGS_CMD | SATA_PMP_CONTROL_PORT; 2965 fis->command = ATA_C_READ_PM; 2966 fis->features = which; 2967 fis->device = target | ATA_H2D_DEVICE_LBA; 2968 fis->control = ATA_FIS_CONTROL_4BIT; 2969 2970 if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) { 2971 error = 1; 2972 } else { 2973 *datap = ccb->ccb_xa.rfis.sector_count | 2974 (ccb->ccb_xa.rfis.lba_low << 8) | 2975 (ccb->ccb_xa.rfis.lba_mid << 16) | 2976 (ccb->ccb_xa.rfis.lba_high << 24); 2977 error = 0; 2978 } 2979 ahci_put_pmp_ccb(ccb); 2980 return (error); 2981 } 2982 2983 int 2984 ahci_pmp_write(struct ahci_port *ap, int target, int which, u_int32_t data) 2985 { 2986 struct ahci_ccb *ccb; 2987 struct ata_fis_h2d *fis; 2988 int error; 2989 2990 ccb = ahci_get_pmp_ccb(ap); 2991 if (ccb == NULL) { 2992 printf("%s: NULL ccb!\n", PORTNAME(ap)); 2993 return (1); 2994 } 2995 ccb->ccb_xa.flags = ATA_F_POLL; 2996 ccb->ccb_xa.pmp_port = SATA_PMP_CONTROL_PORT; 2997 ccb->ccb_xa.state = ATA_S_PENDING; 2998 2999 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table)); 3000 fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis; 3001 fis->type = ATA_FIS_TYPE_H2D; 3002 fis->flags = ATA_H2D_FLAGS_CMD | SATA_PMP_CONTROL_PORT; 3003 fis->command = ATA_C_WRITE_PM; 3004 fis->features = which; 3005 fis->device = target | ATA_H2D_DEVICE_LBA; 3006 fis->sector_count = (u_int8_t)data; 3007 fis->lba_low = (u_int8_t)(data >> 8); 3008 fis->lba_mid = (u_int8_t)(data >> 16); 3009 fis->lba_high = (u_int8_t)(data >> 24); 3010 fis->control = ATA_FIS_CONTROL_4BIT; 3011 3012 error = ahci_poll(ccb, 1000, ahci_pmp_probe_timeout); 3013 ahci_put_pmp_ccb(ccb); 3014 return (error); 3015 } 3016 3017 int 3018 ahci_pmp_phy_status(struct ahci_port *ap, int target, u_int32_t *datap) 3019 { 3020 int error; 3021 3022 error = ahci_pmp_read(ap, target, SATA_PMREG_SSTS, datap); 3023 if (error == 0) 3024 error = ahci_pmp_write(ap, target, SATA_PMREG_SERR, -1); 3025 if (error) 3026 *datap = 0; 3027 3028 return (error); 3029 } 3030 3031 int 3032 ahci_pmp_identify(struct ahci_port *ap, int *ret_nports) 3033 { 3034 u_int32_t chipid; 3035 u_int32_t rev; 3036 u_int32_t nports; 3037 u_int32_t features; 3038 u_int32_t enabled; 3039 int s; 3040 3041 s = splbio(); 3042 3043 if (ahci_pmp_read(ap, 15, 0, &chipid) || 3044 ahci_pmp_read(ap, 15, 1, &rev) || 3045 ahci_pmp_read(ap, 15, 2, &nports) || 3046 ahci_pmp_read(ap, 15, SATA_PMREG_FEA, &features) || 3047 ahci_pmp_read(ap, 15, SATA_PMREG_FEAEN, &enabled)) { 3048 printf("%s: port multiplier identification failed\n", 3049 PORTNAME(ap)); 3050 splx(s); 3051 return (1); 3052 } 3053 splx(s); 3054 3055 nports &= 0x0F; 3056 3057 /* ignore SEMB port on SiI3726 port multiplier chips */ 3058 if (chipid == 0x37261095) { 3059 nports--; 3060 } 3061 3062 printf("%s: port multiplier found: chip=%08x rev=0x%b nports=%d, " 3063 "features: 0x%b, enabled: 0x%b\n", PORTNAME(ap), chipid, rev, 3064 SATA_PFMT_PM_REV, nports, features, SATA_PFMT_PM_FEA, enabled, 3065 SATA_PFMT_PM_FEA); 3066 3067 *ret_nports = nports; 3068 return (0); 3069 } 3070 3071 3072 #ifdef HIBERNATE 3073 void 3074 ahci_hibernate_io_start(struct ahci_port *ap, struct ahci_ccb *ccb) 3075 { 3076 ccb->ccb_cmd_hdr->prdbc = 0; 3077 ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot); 3078 } 3079 3080 int 3081 ahci_hibernate_io_poll(struct ahci_port *ap, struct ahci_ccb *ccb) 3082 { 3083 u_int32_t is, ci_saved; 3084 int process_error = 0; 3085 3086 is = ahci_pread(ap, AHCI_PREG_IS); 3087 3088 ci_saved = ahci_pread(ap, AHCI_PREG_CI); 3089 3090 if (is & AHCI_PREG_IS_DHRS) { 3091 u_int32_t tfd; 3092 u_int32_t cmd; 3093 u_int32_t serr; 3094 3095 tfd = ahci_pread(ap, AHCI_PREG_TFD); 3096 cmd = ahci_pread(ap, AHCI_PREG_CMD); 3097 serr = ahci_pread(ap, AHCI_PREG_SERR); 3098 if ((tfd & AHCI_PREG_TFD_STS_ERR) && 3099 (cmd & AHCI_PREG_CMD_CR) == 0) { 3100 process_error = 1; 3101 } else { 3102 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_DHRS); 3103 } 3104 } else if (is & (AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS | 3105 AHCI_PREG_IS_IFS | AHCI_PREG_IS_OFS | AHCI_PREG_IS_UFS)) { 3106 process_error = 1; 3107 } 3108 3109 /* Command failed. See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2. */ 3110 if (process_error) { 3111 3112 /* Turn off ST to clear CI and SACT. */ 3113 ahci_port_stop(ap, 0); 3114 3115 /* just return an error indicator? we can't meaningfully 3116 * recover, and on the way back out we'll DVACT_RESUME which 3117 * resets and reinits the port. 3118 */ 3119 return (EIO); 3120 } 3121 3122 /* command is finished when the bit in CI for the slot goes to 0 */ 3123 if (ci_saved & (1 << ccb->ccb_slot)) { 3124 return (EAGAIN); 3125 } 3126 3127 return (0); 3128 } 3129 3130 void 3131 ahci_hibernate_load_prdt(struct ahci_ccb *ccb) 3132 { 3133 struct ata_xfer *xa = &ccb->ccb_xa; 3134 struct ahci_prdt *prdt = ccb->ccb_cmd_table->prdt, *prd; 3135 struct ahci_cmd_hdr *cmd_slot = ccb->ccb_cmd_hdr; 3136 int i; 3137 paddr_t data_phys; 3138 u_int64_t data_bus_phys; 3139 vaddr_t data_addr; 3140 size_t seglen; 3141 size_t buflen; 3142 3143 if (xa->datalen == 0) { 3144 ccb->ccb_cmd_hdr->prdtl = 0; 3145 return; 3146 } 3147 3148 /* derived from i386/amd64 _bus_dma_load_buffer; 3149 * for amd64 the buffer will always be dma safe. 3150 */ 3151 3152 buflen = xa->datalen; 3153 data_addr = (vaddr_t)xa->data; 3154 for (i = 0; buflen > 0; i++) { 3155 prd = &prdt[i]; 3156 3157 pmap_extract(pmap_kernel(), data_addr, &data_phys); 3158 data_bus_phys = data_phys; 3159 3160 seglen = PAGE_SIZE - ((u_long)data_addr & PGOFSET); 3161 if (buflen < seglen) 3162 seglen = buflen; 3163 3164 ahci_load_prdt_seg(&prdt[i], data_bus_phys, seglen, 0); 3165 3166 data_addr += seglen; 3167 buflen -= seglen; 3168 } 3169 3170 htolem16(&cmd_slot->prdtl, i); 3171 } 3172 3173 int 3174 ahci_hibernate_io(dev_t dev, daddr_t blkno, vaddr_t addr, size_t size, 3175 int op, void *page) 3176 { 3177 /* we use the 'real' ahci_port and ahci_softc here, but 3178 * never write to them 3179 */ 3180 struct { 3181 struct ahci_cmd_hdr cmd_hdr[32]; /* page aligned, 1024 bytes */ 3182 struct ahci_rfis rfis; /* 1k aligned, 256 bytes */ 3183 /* cmd table isn't actually used because of mysteries */ 3184 struct ahci_cmd_table cmd_table; /* 256 aligned, 512 bytes */ 3185 struct ahci_port *ap; 3186 struct ahci_ccb ccb_buf; 3187 struct ahci_ccb *ccb; 3188 struct ahci_cmd_hdr *hdr_buf; 3189 int pmp_port; 3190 daddr_t poffset; 3191 size_t psize; 3192 } *my = page; 3193 struct ata_fis_h2d *fis; 3194 u_int32_t sector_count; 3195 struct ahci_cmd_hdr *cmd_slot; 3196 int rc; 3197 int timeout; 3198 u_int16_t flags; 3199 3200 if (op == HIB_INIT) { 3201 struct device *disk; 3202 struct device *scsibus; 3203 struct ahci_softc *sc; 3204 extern struct cfdriver sd_cd; 3205 struct scsi_link *link; 3206 struct scsibus_softc *bus_sc; 3207 int port; 3208 paddr_t page_phys; 3209 u_int64_t item_phys; 3210 u_int32_t cmd; 3211 3212 my->poffset = blkno; 3213 my->psize = size; 3214 3215 /* map dev to an ahci port */ 3216 disk = disk_lookup(&sd_cd, DISKUNIT(dev)); 3217 scsibus = disk->dv_parent; 3218 sc = (struct ahci_softc *)disk->dv_parent->dv_parent; 3219 3220 /* find the scsi_link for the device, which has the port */ 3221 port = -1; 3222 bus_sc = (struct scsibus_softc *)scsibus; 3223 SLIST_FOREACH(link, &bus_sc->sc_link, bus_list) { 3224 if (link->device_softc == disk) { 3225 /* link->adapter_softc == sc->sc_atascsi */ 3226 port = link->target; 3227 if (link->lun > 0) 3228 my->pmp_port = link->lun - 1; 3229 else 3230 my->pmp_port = 0; 3231 3232 break; 3233 } 3234 } 3235 if (port == -1) { 3236 /* don't know where the disk is */ 3237 return (EIO); 3238 } 3239 3240 my->ap = sc->sc_ports[port]; 3241 3242 /* we're going to use the first command slot, 3243 * so ensure it's not already in use 3244 */ 3245 if (my->ap->ap_ccbs[0].ccb_xa.state != ATA_S_PUT) { 3246 /* this shouldn't happen, we should be idle */ 3247 return (EIO); 3248 } 3249 3250 /* stop the port so we can relocate to the hibernate page */ 3251 if (ahci_port_stop(my->ap, 1)) { 3252 return (EIO); 3253 } 3254 ahci_pwrite(my->ap, AHCI_PREG_SCTL, 0); 3255 3256 pmap_extract(pmap_kernel(), (vaddr_t)page, &page_phys); 3257 3258 /* Setup RFIS base address */ 3259 item_phys = page_phys + ((void *)&my->rfis - page); 3260 ahci_pwrite(my->ap, AHCI_PREG_FBU, 3261 (u_int32_t)(item_phys >> 32)); 3262 ahci_pwrite(my->ap, AHCI_PREG_FB, (u_int32_t)item_phys); 3263 3264 /* Enable FIS reception and activate port. */ 3265 cmd = ahci_pread(my->ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 3266 cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD | 3267 AHCI_PREG_CMD_SUD; 3268 ahci_pwrite(my->ap, AHCI_PREG_CMD, cmd | 3269 AHCI_PREG_CMD_ICC_ACTIVE); 3270 3271 /* Check whether port activated. */ 3272 cmd = ahci_pread(my->ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 3273 if (!ISSET(cmd, AHCI_PREG_CMD_FRE)) { 3274 return (EIO); 3275 } 3276 3277 /* Set up the single CCB */ 3278 my->ccb = &my->ccb_buf; 3279 my->ccb->ccb_slot = 0; 3280 my->ccb->ccb_port = my->ap; 3281 3282 /* Setup command list base address */ 3283 item_phys = page_phys + ((void *)&my->cmd_hdr - page); 3284 ahci_pwrite(my->ap, AHCI_PREG_CLBU, 3285 (u_int32_t)(item_phys >> 32)); 3286 ahci_pwrite(my->ap, AHCI_PREG_CLB, (u_int32_t)item_phys); 3287 3288 my->ccb->ccb_cmd_hdr = &my->cmd_hdr[0]; 3289 3290 /* use existing cmd table - don't know why moving to a new one fails */ 3291 my->ccb->ccb_cmd_table = my->ap->ap_ccbs[0].ccb_cmd_table; 3292 pmap_extract(pmap_kernel(), 3293 (vaddr_t)AHCI_DMA_KVA(my->ap->ap_dmamem_cmd_table), 3294 &page_phys); 3295 item_phys = page_phys; 3296 #if 0 3297 /* use cmd table in hibernate page (doesn't work) */ 3298 my->ccb->ccb_cmd_table = &my->cmd_table; 3299 item_phys = page_phys + ((void *)&my->cmd_table - page); 3300 #endif 3301 htolem64(&my->ccb->ccb_cmd_hdr->ctba, item_phys); 3302 3303 my->ccb->ccb_xa.fis = 3304 (struct ata_fis_h2d *)my->ccb->ccb_cmd_table->cfis; 3305 my->ccb->ccb_xa.packetcmd = my->ccb->ccb_cmd_table->acmd; 3306 my->ccb->ccb_xa.tag = 0; 3307 3308 /* Wait for ICC change to complete */ 3309 ahci_pwait_clr(my->ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1); 3310 3311 if (ahci_port_start(my->ap, 0)) { 3312 return (EIO); 3313 } 3314 3315 /* Flush interrupts for port */ 3316 ahci_pwrite(my->ap, AHCI_PREG_IS, ahci_pread(my->ap, 3317 AHCI_PREG_IS)); 3318 ahci_write(sc, AHCI_REG_IS, 1 << port); 3319 3320 ahci_enable_interrupts(my->ap); 3321 return (0); 3322 } else if (op == HIB_DONE) { 3323 ahci_activate(&my->ap->ap_sc->sc_dev, DVACT_RESUME); 3324 return (0); 3325 } 3326 3327 if (blkno > my->psize) 3328 return (E2BIG); 3329 blkno += my->poffset; 3330 3331 /* build fis */ 3332 sector_count = size / 512; /* dlg promises this is okay */ 3333 my->ccb->ccb_xa.flags = op == HIB_W ? ATA_F_WRITE : ATA_F_READ; 3334 fis = my->ccb->ccb_xa.fis; 3335 fis->flags = ATA_H2D_FLAGS_CMD | my->pmp_port; 3336 fis->lba_low = blkno & 0xff; 3337 fis->lba_mid = (blkno >> 8) & 0xff; 3338 fis->lba_high = (blkno >> 16) & 0xff; 3339 3340 if (sector_count > 0x100 || blkno > 0xfffffff) { 3341 /* Use LBA48 */ 3342 fis->command = op == HIB_W ? ATA_C_WRITEDMA_EXT : 3343 ATA_C_READDMA_EXT; 3344 fis->device = ATA_H2D_DEVICE_LBA; 3345 fis->lba_low_exp = (blkno >> 24) & 0xff; 3346 fis->lba_mid_exp = (blkno >> 32) & 0xff; 3347 fis->lba_high_exp = (blkno >> 40) & 0xff; 3348 fis->sector_count = sector_count & 0xff; 3349 fis->sector_count_exp = (sector_count >> 8) & 0xff; 3350 } else { 3351 /* Use LBA */ 3352 fis->command = op == HIB_W ? ATA_C_WRITEDMA : ATA_C_READDMA; 3353 fis->device = ATA_H2D_DEVICE_LBA | ((blkno >> 24) & 0x0f); 3354 fis->sector_count = sector_count & 0xff; 3355 } 3356 3357 my->ccb->ccb_xa.data = (void *)addr; 3358 my->ccb->ccb_xa.datalen = size; 3359 my->ccb->ccb_xa.pmp_port = my->pmp_port; 3360 my->ccb->ccb_xa.flags |= ATA_F_POLL; 3361 3362 cmd_slot = my->ccb->ccb_cmd_hdr; 3363 flags = 5; /* FIS length (in DWORDs) */ 3364 flags |= my->pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT; 3365 3366 if (op == HIB_W) 3367 flags |= AHCI_CMD_LIST_FLAG_W; 3368 3369 htolem16(&cmd_slot->flags, flags); 3370 3371 ahci_hibernate_load_prdt(my->ccb); 3372 3373 ahci_hibernate_io_start(my->ap, my->ccb); 3374 timeout = 1000; 3375 while ((rc = ahci_hibernate_io_poll(my->ap, my->ccb)) == EAGAIN) { 3376 delay(1000); 3377 timeout--; 3378 if (timeout == 0) { 3379 return (EIO); 3380 } 3381 } 3382 3383 return (0); 3384 } 3385 3386 #endif 3387