1 /* 2 * Copyright (c) 2006 David Gwynne <dlg@openbsd.org> 3 * 4 * Permission to use, copy, modify, and distribute this software for any 5 * purpose with or without fee is hereby granted, provided that the above 6 * copyright notice and this permission notice appear in all copies. 7 * 8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 11 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 13 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 14 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 15 * 16 * 17 * Copyright (c) 2009 The DragonFly Project. All rights reserved. 18 * 19 * This code is derived from software contributed to The DragonFly Project 20 * by Matthew Dillon <dillon@backplane.com> 21 * 22 * Redistribution and use in source and binary forms, with or without 23 * modification, are permitted provided that the following conditions 24 * are met: 25 * 26 * 1. Redistributions of source code must retain the above copyright 27 * notice, this list of conditions and the following disclaimer. 28 * 2. Redistributions in binary form must reproduce the above copyright 29 * notice, this list of conditions and the following disclaimer in 30 * the documentation and/or other materials provided with the 31 * distribution. 32 * 3. Neither the name of The DragonFly Project nor the names of its 33 * contributors may be used to endorse or promote products derived 34 * from this software without specific, prior written permission. 35 * 36 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 37 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 38 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 39 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 40 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 41 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 42 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 43 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 44 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 45 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 46 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 47 * SUCH DAMAGE. 48 * 49 * $OpenBSD: ahci.c,v 1.147 2009/02/16 21:19:07 miod Exp $ 50 */ 51 52 #include "ahci.h" 53 54 void ahci_port_interrupt_enable(struct ahci_port *ap); 55 56 int ahci_load_prdt(struct ahci_ccb *); 57 void ahci_unload_prdt(struct ahci_ccb *); 58 static void ahci_load_prdt_callback(void *info, bus_dma_segment_t *segs, 59 int nsegs, int error); 60 void ahci_start(struct ahci_ccb *); 61 int ahci_port_softreset(struct ahci_port *ap); 62 int ahci_port_hardreset(struct ahci_port *ap, int hard); 63 void ahci_port_hardstop(struct ahci_port *ap); 64 65 static void ahci_ata_cmd_timeout_unserialized(void *); 66 void ahci_check_active_timeouts(struct ahci_port *ap); 67 68 void ahci_beg_exclusive_access(struct ahci_port *ap, struct ata_port *at); 69 void ahci_end_exclusive_access(struct ahci_port *ap, struct ata_port *at); 70 void ahci_issue_pending_commands(struct ahci_port *ap, struct ahci_ccb *ccb); 71 void ahci_issue_saved_commands(struct ahci_port *ap, u_int32_t mask); 72 73 int ahci_port_read_ncq_error(struct ahci_port *, int); 74 75 struct ahci_dmamem *ahci_dmamem_alloc(struct ahci_softc *, bus_dma_tag_t tag); 76 void ahci_dmamem_free(struct ahci_softc *, struct ahci_dmamem *); 77 static void ahci_dmamem_saveseg(void *info, bus_dma_segment_t *segs, int nsegs, int error); 78 79 static void ahci_dummy_done(struct ata_xfer *xa); 80 static void ahci_empty_done(struct ahci_ccb *ccb); 81 static void ahci_ata_cmd_done(struct ahci_ccb *ccb); 82 83 /* 84 * Initialize the global AHCI hardware. This code does not set up any of 85 * its ports. 86 */ 87 int 88 ahci_init(struct ahci_softc *sc) 89 { 90 u_int32_t cap, pi, pleft; 91 int i; 92 struct ahci_port *ap; 93 94 DPRINTF(AHCI_D_VERBOSE, " GHC 0x%b", 95 ahci_read(sc, AHCI_REG_GHC), AHCI_FMT_GHC); 96 97 /* 98 * save BIOS initialised parameters, enable staggered spin up 99 */ 100 cap = ahci_read(sc, AHCI_REG_CAP); 101 cap &= AHCI_REG_CAP_SMPS; 102 cap |= AHCI_REG_CAP_SSS; 103 pi = ahci_read(sc, AHCI_REG_PI); 104 105 /* 106 * Unconditionally reset the controller, do not conditionalize on 107 * trying to figure it if it was previously active or not. 108 * 109 * NOTE: On AE before HR. The AHCI-1.1 spec has a note in section 110 * 5.2.2.1 regarding this. HR should be set to 1 only after 111 * AE is set to 1. The reset sequence will clear HR when 112 * it completes, and will also clear AE if SAM is 0. AE must 113 * then be set again. When SAM is 1 the AE bit typically reads 114 * as 1 (and is read-only). 115 * 116 * NOTE: Avoid PCI[e] transaction burst by issuing dummy reads, 117 * otherwise the writes will only be separated by a few 118 * nanoseconds. 119 * 120 * NOTE BRICKS (1) 121 * 122 * If you have a port multiplier and it does not have a device 123 * in target 0, and it probes normally, but a later operation 124 * mis-probes a target behind that PM, it is possible for the 125 * port to brick such that only (a) a power cycle of the host 126 * or (b) placing a device in target 0 will fix the problem. 127 * Power cycling the PM has no effect (it works fine on another 128 * host port). This issue is unrelated to CLO. 129 */ 130 /* 131 * Wait for any prior reset sequence to complete 132 */ 133 if (ahci_wait_ne(sc, AHCI_REG_GHC, 134 AHCI_REG_GHC_HR, AHCI_REG_GHC_HR) != 0) { 135 device_printf(sc->sc_dev, "Controller is stuck in reset\n"); 136 return (1); 137 } 138 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE); 139 ahci_os_sleep(500); 140 ahci_read(sc, AHCI_REG_GHC); /* flush */ 141 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE | AHCI_REG_GHC_HR); 142 ahci_os_sleep(500); 143 ahci_read(sc, AHCI_REG_GHC); /* flush */ 144 if (ahci_wait_ne(sc, AHCI_REG_GHC, 145 AHCI_REG_GHC_HR, AHCI_REG_GHC_HR) != 0) { 146 device_printf(sc->sc_dev, "unable to reset controller\n"); 147 return (1); 148 } 149 if (ahci_read(sc, AHCI_REG_GHC) & AHCI_REG_GHC_AE) { 150 device_printf(sc->sc_dev, "AE did not auto-clear!\n"); 151 ahci_write(sc, AHCI_REG_GHC, 0); 152 ahci_os_sleep(500); 153 } 154 155 /* 156 * Enable ahci (global interrupts disabled) 157 * 158 * Restore saved parameters. Avoid pci transaction burst write 159 * by issuing dummy reads. 160 */ 161 ahci_os_sleep(500); 162 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE); 163 ahci_os_sleep(500); 164 165 ahci_read(sc, AHCI_REG_GHC); /* flush */ 166 ahci_write(sc, AHCI_REG_CAP, cap); 167 ahci_write(sc, AHCI_REG_PI, pi); 168 ahci_read(sc, AHCI_REG_GHC); /* flush */ 169 170 /* 171 * Intel hocus pocus in case the BIOS has not set the chip up 172 * properly for AHCI operation. 173 */ 174 if (pci_get_vendor(sc->sc_dev) == PCI_VENDOR_INTEL) { 175 if ((pci_read_config(sc->sc_dev, 0x92, 2) & 0x0F) != 0x0F) 176 device_printf(sc->sc_dev, "Intel hocus pocus\n"); 177 pci_write_config(sc->sc_dev, 0x92, 178 pci_read_config(sc->sc_dev, 0x92, 2) | 0x0F, 2); 179 } 180 181 /* 182 * This is a hack that currently does not appear to have 183 * a significant effect, but I noticed the port registers 184 * do not appear to be completely cleared after the host 185 * controller is reset. 186 * 187 * Use a temporary ap structure so we can call ahci_pwrite(). 188 * 189 * We must be sure to stop the port 190 */ 191 ap = kmalloc(sizeof(*ap), M_DEVBUF, M_WAITOK | M_ZERO); 192 ap->ap_sc = sc; 193 pleft = pi; 194 for (i = 0; i < AHCI_MAX_PORTS; ++i) { 195 if (pleft == 0) 196 break; 197 if ((pi & (1 << i)) == 0) 198 continue; 199 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, 200 AHCI_PORT_REGION(i), AHCI_PORT_SIZE, &ap->ap_ioh) != 0) { 201 device_printf(sc->sc_dev, "can't map port\n"); 202 return (1); 203 } 204 /* 205 * NOTE! Setting AHCI_PREG_SCTL_DET_DISABLE on AHCI1.0 or 206 * AHCI1.1 can brick the chipset. Not only brick it, 207 * but also crash the PC. The bit seems unreliable 208 * on AHCI1.2 as well. 209 */ 210 ahci_port_stop(ap, 1); 211 ahci_pwrite(ap, AHCI_PREG_SCTL, AHCI_PREG_SCTL_IPM_DISABLED); 212 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 213 ahci_pwrite(ap, AHCI_PREG_IE, 0); 214 ahci_write(ap->ap_sc, AHCI_REG_IS, 1 << i); 215 ahci_pwrite(ap, AHCI_PREG_CMD, 0); 216 ahci_pwrite(ap, AHCI_PREG_IS, -1); 217 sc->sc_portmask |= (1 << i); 218 pleft &= ~(1 << i); 219 } 220 sc->sc_numports = i; 221 kfree(ap, M_DEVBUF); 222 223 return (0); 224 } 225 226 /* 227 * Allocate and initialize an AHCI port. 228 */ 229 int 230 ahci_port_alloc(struct ahci_softc *sc, u_int port) 231 { 232 struct ahci_port *ap; 233 struct ata_port *at; 234 struct ahci_ccb *ccb; 235 u_int64_t dva; 236 u_int32_t cmd; 237 u_int32_t data; 238 struct ahci_cmd_hdr *hdr; 239 struct ahci_cmd_table *table; 240 int rc = ENOMEM; 241 int error; 242 int i; 243 244 ap = kmalloc(sizeof(*ap), M_DEVBUF, M_WAITOK | M_ZERO); 245 ap->ap_err_scratch = kmalloc(512, M_DEVBUF, M_WAITOK | M_ZERO); 246 247 ksnprintf(ap->ap_name, sizeof(ap->ap_name), "%s%d.%d", 248 device_get_name(sc->sc_dev), 249 device_get_unit(sc->sc_dev), 250 port); 251 sc->sc_ports[port] = ap; 252 253 /* 254 * Allocate enough so we never have to reallocate, it makes 255 * it easier. 256 * 257 * ap_pmcount will be reduced by the scan if we encounter the 258 * port multiplier port prior to target 15. 259 * 260 * kmalloc power-of-2 allocations are guaranteed not to cross 261 * a page boundary. Make sure the identify sub-structure in the 262 * at structure does not cross a page boundary, just in case the 263 * part is AHCI-1.1 and can't handle multiple DRQ blocks. 264 */ 265 if (ap->ap_ata[0] == NULL) { 266 int pw2; 267 268 for (pw2 = 1; pw2 < sizeof(*at); pw2 <<= 1) 269 ; 270 for (i = 0; i < AHCI_MAX_PMPORTS; ++i) { 271 at = kmalloc(pw2, M_DEVBUF, M_INTWAIT | M_ZERO); 272 ap->ap_ata[i] = at; 273 at->at_ahci_port = ap; 274 at->at_target = i; 275 at->at_probe = ATA_PROBE_NEED_INIT; 276 at->at_features |= ATA_PORT_F_RESCAN; 277 ksnprintf(at->at_name, sizeof(at->at_name), 278 "%s.%d", ap->ap_name, i); 279 } 280 } 281 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, 282 AHCI_PORT_REGION(port), AHCI_PORT_SIZE, &ap->ap_ioh) != 0) { 283 device_printf(sc->sc_dev, 284 "unable to create register window for port %d\n", 285 port); 286 goto freeport; 287 } 288 289 ap->ap_sc = sc; 290 ap->ap_num = port; 291 ap->ap_probe = ATA_PROBE_NEED_INIT; 292 ap->link_pwr_mgmt = AHCI_LINK_PWR_MGMT_NONE; 293 ap->sysctl_tree = NULL; 294 TAILQ_INIT(&ap->ap_ccb_free); 295 TAILQ_INIT(&ap->ap_ccb_pending); 296 lockinit(&ap->ap_ccb_lock, "ahcipo", 0, 0); 297 298 /* Disable port interrupts */ 299 ahci_pwrite(ap, AHCI_PREG_IE, 0); 300 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 301 302 /* 303 * Sec 10.1.2 - deinitialise port if it is already running 304 */ 305 cmd = ahci_pread(ap, AHCI_PREG_CMD); 306 kprintf("%s: Caps %b\n", PORTNAME(ap), cmd, AHCI_PFMT_CMD); 307 308 if ((cmd & (AHCI_PREG_CMD_ST | AHCI_PREG_CMD_CR | 309 AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_FR)) || 310 (ahci_pread(ap, AHCI_PREG_SCTL) & AHCI_PREG_SCTL_DET)) { 311 int r; 312 313 r = ahci_port_stop(ap, 1); 314 if (r) { 315 device_printf(sc->sc_dev, 316 "unable to disable %s, ignoring port %d\n", 317 ((r == 2) ? "CR" : "FR"), port); 318 rc = ENXIO; 319 goto freeport; 320 } 321 322 /* Write DET to zero */ 323 ahci_pwrite(ap, AHCI_PREG_SCTL, AHCI_PREG_SCTL_IPM_DISABLED); 324 } 325 326 /* Allocate RFIS */ 327 ap->ap_dmamem_rfis = ahci_dmamem_alloc(sc, sc->sc_tag_rfis); 328 if (ap->ap_dmamem_rfis == NULL) { 329 kprintf("%s: NORFIS\n", PORTNAME(ap)); 330 goto nomem; 331 } 332 333 /* Setup RFIS base address */ 334 ap->ap_rfis = (struct ahci_rfis *) AHCI_DMA_KVA(ap->ap_dmamem_rfis); 335 dva = AHCI_DMA_DVA(ap->ap_dmamem_rfis); 336 ahci_pwrite(ap, AHCI_PREG_FBU, (u_int32_t)(dva >> 32)); 337 ahci_pwrite(ap, AHCI_PREG_FB, (u_int32_t)dva); 338 339 /* Clear SERR before starting FIS reception or ST or anything */ 340 ahci_flush_tfd(ap); 341 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 342 343 /* Enable FIS reception and activate port. */ 344 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 345 cmd &= ~(AHCI_PREG_CMD_CLO | AHCI_PREG_CMD_PMA); 346 cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD | AHCI_PREG_CMD_SUD; 347 ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_ICC_ACTIVE); 348 349 /* Check whether port activated. Skip it if not. */ 350 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 351 if ((cmd & AHCI_PREG_CMD_FRE) == 0) { 352 kprintf("%s: NOT-ACTIVATED\n", PORTNAME(ap)); 353 rc = ENXIO; 354 goto freeport; 355 } 356 357 /* Allocate a CCB for each command slot */ 358 ap->ap_ccbs = kmalloc(sizeof(struct ahci_ccb) * sc->sc_ncmds, M_DEVBUF, 359 M_WAITOK | M_ZERO); 360 if (ap->ap_ccbs == NULL) { 361 device_printf(sc->sc_dev, 362 "unable to allocate command list for port %d\n", 363 port); 364 goto freeport; 365 } 366 367 /* Command List Structures and Command Tables */ 368 ap->ap_dmamem_cmd_list = ahci_dmamem_alloc(sc, sc->sc_tag_cmdh); 369 ap->ap_dmamem_cmd_table = ahci_dmamem_alloc(sc, sc->sc_tag_cmdt); 370 if (ap->ap_dmamem_cmd_table == NULL || 371 ap->ap_dmamem_cmd_list == NULL) { 372 nomem: 373 device_printf(sc->sc_dev, 374 "unable to allocate DMA memory for port %d\n", 375 port); 376 goto freeport; 377 } 378 379 /* Setup command list base address */ 380 dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_list); 381 ahci_pwrite(ap, AHCI_PREG_CLBU, (u_int32_t)(dva >> 32)); 382 ahci_pwrite(ap, AHCI_PREG_CLB, (u_int32_t)dva); 383 384 /* Split CCB allocation into CCBs and assign to command header/table */ 385 hdr = AHCI_DMA_KVA(ap->ap_dmamem_cmd_list); 386 table = AHCI_DMA_KVA(ap->ap_dmamem_cmd_table); 387 for (i = 0; i < sc->sc_ncmds; i++) { 388 ccb = &ap->ap_ccbs[i]; 389 390 error = bus_dmamap_create(sc->sc_tag_data, BUS_DMA_ALLOCNOW, 391 &ccb->ccb_dmamap); 392 if (error) { 393 device_printf(sc->sc_dev, 394 "unable to create dmamap for port %d " 395 "ccb %d\n", port, i); 396 goto freeport; 397 } 398 399 callout_init(&ccb->ccb_timeout); 400 ccb->ccb_slot = i; 401 ccb->ccb_port = ap; 402 ccb->ccb_cmd_hdr = &hdr[i]; 403 ccb->ccb_cmd_table = &table[i]; 404 dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_table) + 405 ccb->ccb_slot * sizeof(struct ahci_cmd_table); 406 ccb->ccb_cmd_hdr->ctba_hi = htole32((u_int32_t)(dva >> 32)); 407 ccb->ccb_cmd_hdr->ctba_lo = htole32((u_int32_t)dva); 408 409 ccb->ccb_xa.fis = 410 (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis; 411 ccb->ccb_xa.packetcmd = ccb->ccb_cmd_table->acmd; 412 ccb->ccb_xa.tag = i; 413 414 ccb->ccb_xa.state = ATA_S_COMPLETE; 415 416 /* 417 * CCB[1] is the error CCB and is not get or put. It is 418 * also used for probing. Numerous HBAs only load the 419 * signature from CCB[1] so it MUST be used for the second 420 * FIS. 421 */ 422 if (i == 1) 423 ap->ap_err_ccb = ccb; 424 else 425 ahci_put_ccb(ccb); 426 } 427 428 /* 429 * Wait for ICC change to complete 430 */ 431 ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC); 432 433 /* 434 * Calculate the interrupt mask 435 */ 436 data = AHCI_PREG_IE_TFEE | AHCI_PREG_IE_HBFE | 437 AHCI_PREG_IE_IFE | AHCI_PREG_IE_OFE | 438 AHCI_PREG_IE_DPE | AHCI_PREG_IE_UFE | 439 AHCI_PREG_IE_PCE | AHCI_PREG_IE_PRCE | 440 AHCI_PREG_IE_DHRE | AHCI_PREG_IE_SDBE; 441 if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF) 442 data |= AHCI_PREG_IE_IPME; 443 #ifdef AHCI_COALESCE 444 if (sc->sc_ccc_ports & (1 << port) 445 data &= ~(AHCI_PREG_IE_SDBE | AHCI_PREG_IE_DHRE); 446 #endif 447 ap->ap_intmask = data; 448 449 /* 450 * Start the port helper thread. The helper thread will call 451 * ahci_port_init() so the ports can all be started in parallel. 452 * A failure by ahci_port_init() does not deallocate the port 453 * since we still want hot-plug events. 454 */ 455 ahci_os_start_port(ap); 456 return(0); 457 freeport: 458 ahci_port_free(sc, port); 459 return (rc); 460 } 461 462 /* 463 * [re]initialize an idle port. No CCBs should be active. 464 * 465 * This function is called during the initial port allocation sequence 466 * and is also called on hot-plug insertion. We take no chances and 467 * use a portreset instead of a softreset. 468 * 469 * This function is the only way to move a failed port back to active 470 * status. 471 * 472 * Returns 0 if a device is successfully detected. 473 */ 474 int 475 ahci_port_init(struct ahci_port *ap) 476 { 477 /* 478 * Register [re]initialization 479 */ 480 if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF) 481 ahci_pwrite(ap, AHCI_PREG_SNTF, -1); 482 ap->ap_probe = ATA_PROBE_NEED_HARD_RESET; 483 ap->ap_pmcount = 0; 484 485 /* 486 * Flush the TFD and SERR and make sure the port is stopped before 487 * enabling its interrupt. We no longer cycle the port start as 488 * the port should not be started unless a device is present. 489 * 490 * XXX should we enable FIS reception? (FRE)? 491 */ 492 ahci_flush_tfd(ap); 493 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 494 ahci_port_stop(ap, 0); 495 ahci_port_interrupt_enable(ap); 496 return (0); 497 } 498 499 /* 500 * Enable or re-enable interrupts on a port. 501 * 502 * This routine is called from the port initialization code or from the 503 * helper thread as the real interrupt may be forced to turn off certain 504 * interrupt sources. 505 */ 506 void 507 ahci_port_interrupt_enable(struct ahci_port *ap) 508 { 509 ahci_pwrite(ap, AHCI_PREG_IE, ap->ap_intmask); 510 } 511 512 /* 513 * Manage the agressive link power management capability. 514 */ 515 void 516 ahci_port_link_pwr_mgmt(struct ahci_port *ap, int link_pwr_mgmt) 517 { 518 u_int32_t cmd, sctl; 519 520 if (link_pwr_mgmt == ap->link_pwr_mgmt) 521 return; 522 523 if ((ap->ap_sc->sc_cap & AHCI_REG_CAP_SALP) == 0) { 524 kprintf("%s: link power management not supported.\n", 525 PORTNAME(ap)); 526 return; 527 } 528 529 ahci_os_lock_port(ap); 530 531 if (link_pwr_mgmt == AHCI_LINK_PWR_MGMT_AGGR && 532 (ap->ap_sc->sc_cap & AHCI_REG_CAP_SSC)) { 533 kprintf("%s: enabling aggressive link power management.\n", 534 PORTNAME(ap)); 535 536 ap->ap_intmask &= ~AHCI_PREG_IE_PRCE; 537 ahci_port_interrupt_enable(ap); 538 539 sctl = ahci_pread(ap, AHCI_PREG_SCTL); 540 sctl &= ~(AHCI_PREG_SCTL_IPM_DISABLED); 541 ahci_pwrite(ap, AHCI_PREG_SCTL, sctl); 542 543 cmd = ahci_pread(ap, AHCI_PREG_CMD); 544 cmd |= AHCI_PREG_CMD_ASP; 545 cmd |= AHCI_PREG_CMD_ALPE; 546 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 547 548 ap->link_pwr_mgmt = link_pwr_mgmt; 549 550 } else if (link_pwr_mgmt == AHCI_LINK_PWR_MGMT_MEDIUM && 551 (ap->ap_sc->sc_cap & AHCI_REG_CAP_PSC)) { 552 kprintf("%s: enabling medium link power management.\n", 553 PORTNAME(ap)); 554 555 ap->ap_intmask &= ~AHCI_PREG_IE_PRCE; 556 ahci_port_interrupt_enable(ap); 557 558 sctl = ahci_pread(ap, AHCI_PREG_SCTL); 559 sctl &= ~(AHCI_PREG_SCTL_IPM_DISABLED); 560 ahci_pwrite(ap, AHCI_PREG_SCTL, sctl); 561 562 cmd = ahci_pread(ap, AHCI_PREG_CMD); 563 cmd &= ~AHCI_PREG_CMD_ASP; 564 cmd |= AHCI_PREG_CMD_ALPE; 565 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 566 567 ap->link_pwr_mgmt = link_pwr_mgmt; 568 569 } else if (link_pwr_mgmt == AHCI_LINK_PWR_MGMT_NONE) { 570 kprintf("%s: disabling link power management.\n", 571 PORTNAME(ap)); 572 573 cmd = ahci_pread(ap, AHCI_PREG_CMD); 574 cmd |= AHCI_PREG_CMD_ASP; 575 cmd &= ~(AHCI_PREG_CMD_ALPE | AHCI_PREG_CMD_ASP); 576 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 577 578 sctl = ahci_pread(ap, AHCI_PREG_SCTL); 579 sctl |= AHCI_PREG_SCTL_IPM_DISABLED; 580 ahci_pwrite(ap, AHCI_PREG_SCTL, sctl); 581 582 /* let the drive come back to avoid PRCS interrupts later */ 583 ahci_os_unlock_port(ap); 584 ahci_os_sleep(1000); 585 ahci_os_lock_port(ap); 586 587 ahci_pwrite(ap, AHCI_PREG_SERR, AHCI_PREG_SERR_DIAG_N); 588 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_PRCS); 589 590 ap->ap_intmask |= AHCI_PREG_IE_PRCE; 591 ahci_port_interrupt_enable(ap); 592 593 ap->link_pwr_mgmt = link_pwr_mgmt; 594 } else { 595 kprintf("%s: unsupported link power management state %d.\n", 596 PORTNAME(ap), link_pwr_mgmt); 597 } 598 599 ahci_os_unlock_port(ap); 600 } 601 602 603 /* 604 * Run the port / target state machine from a main context. 605 * 606 * The state machine for the port is always run. 607 * 608 * If atx is non-NULL run the state machine for a particular target. 609 * If atx is NULL run the state machine for all targets. 610 */ 611 void 612 ahci_port_state_machine(struct ahci_port *ap, int initial) 613 { 614 struct ata_port *at; 615 u_int32_t data; 616 int target; 617 int didsleep; 618 int loop; 619 620 /* 621 * State machine for port. Note that CAM is not yet associated 622 * during the initial parallel probe and the port's probe state 623 * will not get past ATA_PROBE_NEED_IDENT. 624 */ 625 { 626 if (initial == 0 && ap->ap_probe <= ATA_PROBE_NEED_HARD_RESET) { 627 kprintf("%s: Waiting 10 seconds on insertion\n", 628 PORTNAME(ap)); 629 ahci_os_sleep(10000); 630 initial = 1; 631 } 632 if (ap->ap_probe == ATA_PROBE_NEED_INIT) 633 ahci_port_init(ap); 634 if (ap->ap_probe == ATA_PROBE_NEED_HARD_RESET) 635 ahci_port_reset(ap, NULL, 1); 636 if (ap->ap_probe == ATA_PROBE_NEED_SOFT_RESET) 637 ahci_port_reset(ap, NULL, 0); 638 if (ap->ap_probe == ATA_PROBE_NEED_IDENT) 639 ahci_cam_probe(ap, NULL); 640 } 641 if (ap->ap_type != ATA_PORT_T_PM) { 642 if (ap->ap_probe == ATA_PROBE_FAILED) { 643 ahci_cam_changed(ap, NULL, 0); 644 } else if (ap->ap_probe >= ATA_PROBE_NEED_IDENT) { 645 ahci_cam_changed(ap, NULL, 1); 646 } 647 return; 648 } 649 650 /* 651 * Port Multiplier state machine. 652 * 653 * Get a mask of changed targets and combine with any runnable 654 * states already present. 655 */ 656 for (loop = 0; ;++loop) { 657 if (ahci_pm_read(ap, 15, SATA_PMREG_EINFO, &data)) { 658 kprintf("%s: PM unable to read hot-plug bitmap\n", 659 PORTNAME(ap)); 660 break; 661 } 662 663 /* 664 * Do at least one loop, then stop if no more state changes 665 * have occured. The PM might not generate a new 666 * notification until we clear the entire bitmap. 667 */ 668 if (loop && data == 0) 669 break; 670 671 /* 672 * New devices showing up in the bitmap require some spin-up 673 * time before we start probing them. Reset didsleep. The 674 * first new device we detect will sleep before probing. 675 * 676 * This only applies to devices whos change bit is set in 677 * the data, and does not apply to the initial boot-time 678 * probe. 679 */ 680 didsleep = 0; 681 682 for (target = 0; target < ap->ap_pmcount; ++target) { 683 at = ap->ap_ata[target]; 684 685 /* 686 * Check the target state for targets behind the PM 687 * which have changed state. This will adjust 688 * at_probe and set ATA_PORT_F_RESCAN 689 * 690 * We want to wait at least 10 seconds before probing 691 * a newly inserted device. If the check status 692 * indicates a device is present and in need of a 693 * hard reset, we make sure we have slept before 694 * continuing. 695 * 696 * We also need to wait at least 1 second for the 697 * PHY state to change after insertion, if we 698 * haven't already waited the 10 seconds. 699 * 700 * NOTE: When pm_check_good finds a good port it 701 * typically starts us in probe state 702 * NEED_HARD_RESET rather than INIT. 703 */ 704 if (data & (1 << target)) { 705 if (initial == 0 && didsleep == 0) 706 ahci_os_sleep(1000); 707 ahci_pm_check_good(ap, target); 708 if (initial == 0 && didsleep == 0 && 709 at->at_probe <= ATA_PROBE_NEED_HARD_RESET 710 ) { 711 didsleep = 1; 712 kprintf("%s: Waiting 10 seconds on insertion\n", PORTNAME(ap)); 713 ahci_os_sleep(10000); 714 } 715 } 716 717 /* 718 * Report hot-plug events before the probe state 719 * really gets hot. Only actual events are reported 720 * here to reduce spew. 721 */ 722 if (data & (1 << target)) { 723 kprintf("%s: HOTPLUG (PM) - ", ATANAME(ap, at)); 724 switch(at->at_probe) { 725 case ATA_PROBE_NEED_INIT: 726 case ATA_PROBE_NEED_HARD_RESET: 727 kprintf("Device inserted\n"); 728 break; 729 case ATA_PROBE_FAILED: 730 kprintf("Device removed\n"); 731 break; 732 default: 733 kprintf("Device probe in progress\n"); 734 break; 735 } 736 } 737 738 /* 739 * Run through the state machine as necessary if 740 * the port is not marked failed. 741 * 742 * The state machine may stop at NEED_IDENT if 743 * CAM is not yet attached. 744 * 745 * Acquire exclusive access to the port while we 746 * are doing this. This prevents command-completion 747 * from queueing commands for non-polled targets 748 * inbetween our probe steps. We need to do this 749 * because the reset probes can generate severe PHY 750 * and protocol errors and soft-brick the port. 751 */ 752 if (at->at_probe != ATA_PROBE_FAILED && 753 at->at_probe != ATA_PROBE_GOOD) { 754 ahci_beg_exclusive_access(ap, at); 755 if (at->at_probe == ATA_PROBE_NEED_INIT) 756 ahci_pm_port_init(ap, at); 757 if (at->at_probe == ATA_PROBE_NEED_HARD_RESET) 758 ahci_port_reset(ap, at, 1); 759 if (at->at_probe == ATA_PROBE_NEED_SOFT_RESET) 760 ahci_port_reset(ap, at, 0); 761 if (at->at_probe == ATA_PROBE_NEED_IDENT) 762 ahci_cam_probe(ap, at); 763 ahci_end_exclusive_access(ap, at); 764 } 765 766 /* 767 * Add or remove from CAM 768 */ 769 if (at->at_features & ATA_PORT_F_RESCAN) { 770 at->at_features &= ~ATA_PORT_F_RESCAN; 771 if (at->at_probe == ATA_PROBE_FAILED) { 772 ahci_cam_changed(ap, at, 0); 773 } else if (at->at_probe >= ATA_PROBE_NEED_IDENT) { 774 ahci_cam_changed(ap, at, 1); 775 } 776 } 777 data &= ~(1 << target); 778 } 779 if (data) { 780 kprintf("%s: WARNING (PM): extra bits set in " 781 "EINFO: %08x\n", PORTNAME(ap), data); 782 while (target < AHCI_MAX_PMPORTS) { 783 ahci_pm_check_good(ap, target); 784 ++target; 785 } 786 } 787 } 788 } 789 790 791 /* 792 * De-initialize and detach a port. 793 */ 794 void 795 ahci_port_free(struct ahci_softc *sc, u_int port) 796 { 797 struct ahci_port *ap = sc->sc_ports[port]; 798 struct ahci_ccb *ccb; 799 int i; 800 801 /* 802 * Ensure port is disabled and its interrupts are all flushed. 803 */ 804 if (ap->ap_sc) { 805 ahci_port_stop(ap, 1); 806 ahci_os_stop_port(ap); 807 ahci_pwrite(ap, AHCI_PREG_CMD, 0); 808 ahci_pwrite(ap, AHCI_PREG_IE, 0); 809 ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS)); 810 ahci_write(sc, AHCI_REG_IS, 1 << port); 811 } 812 813 if (ap->ap_ccbs) { 814 while ((ccb = ahci_get_ccb(ap)) != NULL) { 815 if (ccb->ccb_dmamap) { 816 bus_dmamap_destroy(sc->sc_tag_data, 817 ccb->ccb_dmamap); 818 ccb->ccb_dmamap = NULL; 819 } 820 } 821 if ((ccb = ap->ap_err_ccb) != NULL) { 822 if (ccb->ccb_dmamap) { 823 bus_dmamap_destroy(sc->sc_tag_data, 824 ccb->ccb_dmamap); 825 ccb->ccb_dmamap = NULL; 826 } 827 ap->ap_err_ccb = NULL; 828 } 829 kfree(ap->ap_ccbs, M_DEVBUF); 830 ap->ap_ccbs = NULL; 831 } 832 833 if (ap->ap_dmamem_cmd_list) { 834 ahci_dmamem_free(sc, ap->ap_dmamem_cmd_list); 835 ap->ap_dmamem_cmd_list = NULL; 836 } 837 if (ap->ap_dmamem_rfis) { 838 ahci_dmamem_free(sc, ap->ap_dmamem_rfis); 839 ap->ap_dmamem_rfis = NULL; 840 } 841 if (ap->ap_dmamem_cmd_table) { 842 ahci_dmamem_free(sc, ap->ap_dmamem_cmd_table); 843 ap->ap_dmamem_cmd_table = NULL; 844 } 845 if (ap->ap_ata) { 846 for (i = 0; i < AHCI_MAX_PMPORTS; ++i) { 847 if (ap->ap_ata[i]) { 848 kfree(ap->ap_ata[i], M_DEVBUF); 849 ap->ap_ata[i] = NULL; 850 } 851 } 852 } 853 if (ap->ap_err_scratch) { 854 kfree(ap->ap_err_scratch, M_DEVBUF); 855 ap->ap_err_scratch = NULL; 856 } 857 858 /* bus_space(9) says we dont free the subregions handle */ 859 860 kfree(ap, M_DEVBUF); 861 sc->sc_ports[port] = NULL; 862 } 863 864 /* 865 * Start high-level command processing on the port 866 */ 867 int 868 ahci_port_start(struct ahci_port *ap) 869 { 870 u_int32_t r, s, is, tfd; 871 872 /* 873 * FRE must be turned on before ST. Wait for FR to go active 874 * before turning on ST. The spec doesn't seem to think this 875 * is necessary but waiting here avoids an on-off race in the 876 * ahci_port_stop() code. 877 */ 878 r = ahci_pread(ap, AHCI_PREG_CMD); 879 if ((r & AHCI_PREG_CMD_FRE) == 0) { 880 r |= AHCI_PREG_CMD_FRE; 881 ahci_pwrite(ap, AHCI_PREG_CMD, r); 882 } 883 if ((ap->ap_sc->sc_flags & AHCI_F_IGN_FR) == 0) { 884 if (ahci_pwait_set(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_FR)) { 885 kprintf("%s: Cannot start FIS reception\n", 886 PORTNAME(ap)); 887 return (2); 888 } 889 } else { 890 ahci_os_sleep(10); 891 } 892 893 /* 894 * Turn on ST, wait for CR to come up. 895 */ 896 r |= AHCI_PREG_CMD_ST; 897 ahci_pwrite(ap, AHCI_PREG_CMD, r); 898 if (ahci_pwait_set_to(ap, 2000, AHCI_PREG_CMD, AHCI_PREG_CMD_CR)) { 899 s = ahci_pread(ap, AHCI_PREG_SERR); 900 is = ahci_pread(ap, AHCI_PREG_IS); 901 tfd = ahci_pread(ap, AHCI_PREG_TFD); 902 kprintf("%s: Cannot start command DMA\n" 903 "NCMP=%b NSERR=%b\n" 904 "NEWIS=%b\n" 905 "NEWTFD=%b\n", 906 PORTNAME(ap), 907 r, AHCI_PFMT_CMD, s, AHCI_PFMT_SERR, 908 is, AHCI_PFMT_IS, 909 tfd, AHCI_PFMT_TFD_STS); 910 return (1); 911 } 912 913 #ifdef AHCI_COALESCE 914 /* 915 * (Re-)enable coalescing on the port. 916 */ 917 if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) { 918 ap->ap_sc->sc_ccc_ports_cur |= (1 << ap->ap_num); 919 ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS, 920 ap->ap_sc->sc_ccc_ports_cur); 921 } 922 #endif 923 924 return (0); 925 } 926 927 /* 928 * Stop high-level command processing on a port 929 * 930 * WARNING! If the port is stopped while CR is still active our saved 931 * CI/SACT will race any commands completed by the command 932 * processor prior to being able to stop. Thus we never call 933 * this function unless we intend to dispose of any remaining 934 * active commands. In particular, this complicates the timeout 935 * code. 936 */ 937 int 938 ahci_port_stop(struct ahci_port *ap, int stop_fis_rx) 939 { 940 u_int32_t r; 941 942 #ifdef AHCI_COALESCE 943 /* 944 * Disable coalescing on the port while it is stopped. 945 */ 946 if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) { 947 ap->ap_sc->sc_ccc_ports_cur &= ~(1 << ap->ap_num); 948 ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS, 949 ap->ap_sc->sc_ccc_ports_cur); 950 } 951 #endif 952 953 /* 954 * Turn off ST, then wait for CR to go off. 955 */ 956 r = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 957 r &= ~AHCI_PREG_CMD_ST; 958 ahci_pwrite(ap, AHCI_PREG_CMD, r); 959 960 if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CR)) { 961 kprintf("%s: Port bricked, unable to stop (ST)\n", 962 PORTNAME(ap)); 963 return (1); 964 } 965 966 #if 0 967 /* 968 * Turn off FRE, then wait for FR to go off. FRE cannot 969 * be turned off until CR transitions to 0. 970 */ 971 if ((r & AHCI_PREG_CMD_FR) == 0) { 972 kprintf("%s: FR stopped, clear FRE for next start\n", 973 PORTNAME(ap)); 974 stop_fis_rx = 2; 975 } 976 #endif 977 if (stop_fis_rx) { 978 r &= ~AHCI_PREG_CMD_FRE; 979 ahci_pwrite(ap, AHCI_PREG_CMD, r); 980 if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_FR)) { 981 kprintf("%s: Port bricked, unable to stop (FRE)\n", 982 PORTNAME(ap)); 983 return (2); 984 } 985 } 986 987 return (0); 988 } 989 990 /* 991 * AHCI command list override -> forcibly clear TFD.STS.{BSY,DRQ} 992 */ 993 int 994 ahci_port_clo(struct ahci_port *ap) 995 { 996 struct ahci_softc *sc = ap->ap_sc; 997 u_int32_t cmd; 998 999 /* Only attempt CLO if supported by controller */ 1000 if ((ahci_read(sc, AHCI_REG_CAP) & AHCI_REG_CAP_SCLO) == 0) 1001 return (1); 1002 1003 /* Issue CLO */ 1004 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 1005 ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_CLO); 1006 1007 /* Wait for completion */ 1008 if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CLO)) { 1009 kprintf("%s: CLO did not complete\n", PORTNAME(ap)); 1010 return (1); 1011 } 1012 1013 return (0); 1014 } 1015 1016 /* 1017 * Reset a port. 1018 * 1019 * If hard is 0 perform a softreset of the port. 1020 * If hard is 1 perform a hard reset of the port. 1021 * 1022 * If at is non-NULL an indirect port via a port-multiplier is being 1023 * reset, otherwise a direct port is being reset. 1024 * 1025 * NOTE: Indirect ports can only be soft-reset. 1026 */ 1027 int 1028 ahci_port_reset(struct ahci_port *ap, struct ata_port *at, int hard) 1029 { 1030 int rc; 1031 1032 if (hard) { 1033 if (at) 1034 rc = ahci_pm_hardreset(ap, at->at_target, hard); 1035 else 1036 rc = ahci_port_hardreset(ap, hard); 1037 } else { 1038 if (at) 1039 rc = ahci_pm_softreset(ap, at->at_target); 1040 else 1041 rc = ahci_port_softreset(ap); 1042 } 1043 return(rc); 1044 } 1045 1046 /* 1047 * AHCI soft reset, Section 10.4.1 1048 * 1049 * (at) will be NULL when soft-resetting a directly-attached device, and 1050 * non-NULL when soft-resetting a device through a port multiplier. 1051 * 1052 * This function keeps port communications intact and attempts to generate 1053 * a reset to the connected device using device commands. 1054 */ 1055 int 1056 ahci_port_softreset(struct ahci_port *ap) 1057 { 1058 struct ahci_ccb *ccb = NULL; 1059 struct ahci_cmd_hdr *cmd_slot; 1060 u_int8_t *fis; 1061 int error; 1062 1063 error = EIO; 1064 1065 if (bootverbose) { 1066 kprintf("%s: START SOFTRESET %b\n", PORTNAME(ap), 1067 ahci_pread(ap, AHCI_PREG_CMD), AHCI_PFMT_CMD); 1068 } 1069 1070 DPRINTF(AHCI_D_VERBOSE, "%s: soft reset\n", PORTNAME(ap)); 1071 1072 crit_enter(); 1073 ap->ap_flags |= AP_F_IN_RESET; 1074 ap->ap_state = AP_S_NORMAL; 1075 1076 /* 1077 * Remember port state in cmd (main to restore start/stop) 1078 * 1079 * Idle port. 1080 */ 1081 if (ahci_port_stop(ap, 0)) { 1082 kprintf("%s: failed to stop port, cannot softreset\n", 1083 PORTNAME(ap)); 1084 goto err; 1085 } 1086 1087 /* 1088 * Request CLO if device appears hung. 1089 */ 1090 if (ahci_pread(ap, AHCI_PREG_TFD) & 1091 (AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) { 1092 ahci_port_clo(ap); 1093 } 1094 1095 /* 1096 * This is an attempt to clear errors so a new signature will 1097 * be latched. It isn't working properly. XXX 1098 */ 1099 ahci_flush_tfd(ap); 1100 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 1101 1102 /* Restart port */ 1103 if (ahci_port_start(ap)) { 1104 kprintf("%s: failed to start port, cannot softreset\n", 1105 PORTNAME(ap)); 1106 goto err; 1107 } 1108 1109 /* Check whether CLO worked */ 1110 if (ahci_pwait_clr(ap, AHCI_PREG_TFD, 1111 AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) { 1112 kprintf("%s: CLO %s, need port reset\n", 1113 PORTNAME(ap), 1114 (ahci_read(ap->ap_sc, AHCI_REG_CAP) & AHCI_REG_CAP_SCLO) 1115 ? "failed" : "unsupported"); 1116 error = EBUSY; 1117 goto err; 1118 } 1119 1120 /* 1121 * Prep first D2H command with SRST feature & clear busy/reset flags 1122 * 1123 * It is unclear which other fields in the FIS are used. Just zero 1124 * everything. 1125 * 1126 * NOTE! This CCB is used for both the first and second commands. 1127 * The second command must use CCB slot 1 to properly load 1128 * the signature. 1129 */ 1130 ccb = ahci_get_err_ccb(ap); 1131 ccb->ccb_xa.complete = ahci_dummy_done; 1132 ccb->ccb_xa.flags = ATA_F_POLL | ATA_F_EXCLUSIVE; 1133 KKASSERT(ccb->ccb_slot == 1); 1134 ccb->ccb_xa.at = NULL; 1135 cmd_slot = ccb->ccb_cmd_hdr; 1136 1137 fis = ccb->ccb_cmd_table->cfis; 1138 bzero(fis, sizeof(ccb->ccb_cmd_table->cfis)); 1139 fis[0] = ATA_FIS_TYPE_H2D; 1140 fis[15] = ATA_FIS_CONTROL_SRST|ATA_FIS_CONTROL_4BIT; 1141 1142 cmd_slot->prdtl = 0; 1143 cmd_slot->flags = htole16(5); /* FIS length: 5 DWORDS */ 1144 cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_C); /* Clear busy on OK */ 1145 cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_R); /* Reset */ 1146 1147 ccb->ccb_xa.state = ATA_S_PENDING; 1148 1149 if (ahci_poll(ccb, 1000, ahci_quick_timeout) != ATA_S_COMPLETE) { 1150 kprintf("%s: First FIS failed\n", PORTNAME(ap)); 1151 goto err; 1152 } 1153 1154 /* 1155 * WARNING! TIME SENSITIVE SPACE! WARNING! 1156 * 1157 * The two FISes are supposed to be back to back. Don't issue other 1158 * commands or even delay if we can help it. 1159 */ 1160 1161 /* 1162 * Prep second D2H command to read status and complete reset sequence 1163 * AHCI 10.4.1 and "Serial ATA Revision 2.6". I can't find the ATA 1164 * Rev 2.6 and it is unclear how the second FIS should be set up 1165 * from the AHCI document. 1166 * 1167 * Give the device 3ms before sending the second FIS. 1168 * 1169 * It is unclear which other fields in the FIS are used. Just zero 1170 * everything. 1171 */ 1172 ccb->ccb_xa.flags = ATA_F_POLL | ATA_F_AUTOSENSE | ATA_F_EXCLUSIVE; 1173 1174 bzero(fis, sizeof(ccb->ccb_cmd_table->cfis)); 1175 fis[0] = ATA_FIS_TYPE_H2D; 1176 fis[15] = ATA_FIS_CONTROL_4BIT; 1177 1178 cmd_slot->prdtl = 0; 1179 cmd_slot->flags = htole16(5); /* FIS length: 5 DWORDS */ 1180 1181 ccb->ccb_xa.state = ATA_S_PENDING; 1182 if (ahci_poll(ccb, 1000, ahci_quick_timeout) != ATA_S_COMPLETE) { 1183 kprintf("%s: Second FIS failed\n", PORTNAME(ap)); 1184 goto err; 1185 } 1186 1187 if (ahci_pwait_clr(ap, AHCI_PREG_TFD, 1188 AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) { 1189 kprintf("%s: device didn't come ready after reset, TFD: 0x%b\n", 1190 PORTNAME(ap), 1191 ahci_pread(ap, AHCI_PREG_TFD), AHCI_PFMT_TFD_STS); 1192 error = EBUSY; 1193 goto err; 1194 } 1195 ahci_os_sleep(10); 1196 1197 /* 1198 * If the softreset is trying to clear a BSY condition after a 1199 * normal portreset we assign the port type. 1200 * 1201 * If the softreset is being run first as part of the ccb error 1202 * processing code then report if the device signature changed 1203 * unexpectedly. 1204 */ 1205 if (ap->ap_type == ATA_PORT_T_NONE) { 1206 ap->ap_type = ahci_port_signature_detect(ap, NULL); 1207 } else { 1208 if (ahci_port_signature_detect(ap, NULL) != ap->ap_type) { 1209 kprintf("%s: device signature unexpectedly " 1210 "changed\n", PORTNAME(ap)); 1211 error = EBUSY; /* XXX */ 1212 } 1213 } 1214 error = 0; 1215 1216 ahci_os_sleep(3); 1217 err: 1218 if (ccb != NULL) { 1219 ahci_put_err_ccb(ccb); 1220 1221 /* 1222 * If the target is busy use CLO to clear the busy 1223 * condition. The BSY should be cleared on the next 1224 * start. 1225 */ 1226 if (ahci_pread(ap, AHCI_PREG_TFD) & 1227 (AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) { 1228 ahci_port_clo(ap); 1229 } 1230 } 1231 1232 /* 1233 * If we failed to softreset make the port quiescent, otherwise 1234 * make sure the port's start/stop state matches what it was on 1235 * entry. 1236 * 1237 * Don't kill the port if the softreset is on a port multiplier 1238 * target, that would kill all the targets! 1239 */ 1240 if (error) { 1241 ahci_port_hardstop(ap); 1242 /* ap_probe set to failed */ 1243 } else { 1244 ap->ap_probe = ATA_PROBE_NEED_IDENT; 1245 ap->ap_pmcount = 1; 1246 ahci_port_start(ap); 1247 } 1248 ap->ap_flags &= ~AP_F_IN_RESET; 1249 crit_exit(); 1250 1251 if (bootverbose) 1252 kprintf("%s: END SOFTRESET\n", PORTNAME(ap)); 1253 1254 return (error); 1255 } 1256 1257 /* 1258 * AHCI port reset, Section 10.4.2 1259 * 1260 * This function does a hard reset of the port. Note that the device 1261 * connected to the port could still end-up hung. 1262 */ 1263 int 1264 ahci_port_hardreset(struct ahci_port *ap, int hard) 1265 { 1266 u_int32_t cmd, r; 1267 u_int32_t data; 1268 int error; 1269 int loop; 1270 1271 if (bootverbose) 1272 kprintf("%s: START HARDRESET\n", PORTNAME(ap)); 1273 ap->ap_flags |= AP_F_IN_RESET; 1274 1275 /* 1276 * Idle the port, 1277 */ 1278 ahci_port_stop(ap, 0); 1279 ap->ap_state = AP_S_NORMAL; 1280 1281 /* 1282 * The port may have been quiescent with its SUD bit cleared, so 1283 * set the SUD (spin up device). 1284 */ 1285 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; 1286 cmd |= AHCI_PREG_CMD_SUD; 1287 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 1288 1289 /* 1290 * Perform device detection. 1291 * 1292 * NOTE! AHCi_PREG_SCTL_DET_DISABLE seems to be highly unreliable 1293 * on multiple chipsets and can brick the chipset or even 1294 * the whole PC. Never use it. 1295 */ 1296 ap->ap_type = ATA_PORT_T_NONE; 1297 1298 r = AHCI_PREG_SCTL_IPM_DISABLED; 1299 ahci_pwrite(ap, AHCI_PREG_SCTL, r); 1300 ahci_os_sleep(10); 1301 1302 /* 1303 * Start transmitting COMRESET. COMRESET must be sent for at 1304 * least 1ms. 1305 */ 1306 r = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_INIT; 1307 if (AhciForceGen1 & (1 << ap->ap_num)) 1308 r |= AHCI_PREG_SCTL_SPD_GEN1; 1309 else 1310 r |= AHCI_PREG_SCTL_SPD_ANY; 1311 ahci_pwrite(ap, AHCI_PREG_SCTL, r); 1312 1313 /* 1314 * Through trial and error it seems to take around 100ms 1315 * for the detect logic to settle down. If this is too 1316 * short the softreset code will fail. 1317 */ 1318 ahci_os_sleep(100); 1319 1320 /* 1321 * Only SERR_DIAG_X needs to be cleared for TFD updates, but 1322 * since we are hard-resetting the port we might as well clear 1323 * the whole enchillada 1324 */ 1325 ahci_flush_tfd(ap); 1326 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 1327 r &= ~AHCI_PREG_SCTL_DET_INIT; 1328 r |= AHCI_PREG_SCTL_DET_NONE; 1329 ahci_pwrite(ap, AHCI_PREG_SCTL, r); 1330 1331 /* 1332 * Try to determine if there is a device on the port. 1333 * 1334 * Give the device 3/10 second to at least be detected. 1335 * If we fail clear PRCS (phy detect) since we may cycled 1336 * the phy and probably caused another PRCS interrupt. 1337 */ 1338 loop = 300; 1339 while (loop > 0) { 1340 r = ahci_pread(ap, AHCI_PREG_SSTS); 1341 if (r & AHCI_PREG_SSTS_DET) 1342 break; 1343 loop -= ahci_os_softsleep(); 1344 } 1345 if (loop == 0) { 1346 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_PRCS); 1347 if (bootverbose) { 1348 kprintf("%s: Port appears to be unplugged\n", 1349 PORTNAME(ap)); 1350 } 1351 error = ENODEV; 1352 goto done; 1353 } 1354 1355 /* 1356 * There is something on the port. Give the device 3 seconds 1357 * to fully negotiate. 1358 */ 1359 if (ahci_pwait_eq(ap, 3000, AHCI_PREG_SSTS, 1360 AHCI_PREG_SSTS_DET, AHCI_PREG_SSTS_DET_DEV)) { 1361 if (bootverbose) { 1362 kprintf("%s: Device may be powered down\n", 1363 PORTNAME(ap)); 1364 } 1365 error = ENODEV; 1366 goto pmdetect; 1367 } 1368 1369 /* 1370 * We got something that definitely looks like a device. Give 1371 * the device time to send us its first D2H FIS. Waiting for 1372 * BSY to clear accomplishes this. 1373 * 1374 * NOTE that a port multiplier may or may not clear BSY here, 1375 * depending on what is sitting in target 0 behind it. 1376 */ 1377 ahci_flush_tfd(ap); 1378 1379 if (ahci_pwait_clr_to(ap, 3000, AHCI_PREG_TFD, 1380 AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) { 1381 error = EBUSY; 1382 } else { 1383 error = 0; 1384 } 1385 1386 pmdetect: 1387 /* 1388 * Do the PM port probe regardless of how things turned out on 1389 * the BSY check. 1390 */ 1391 if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SPM) 1392 error = ahci_pm_port_probe(ap, error); 1393 1394 done: 1395 /* 1396 * Finish up. 1397 */ 1398 switch(error) { 1399 case 0: 1400 /* 1401 * All good, make sure the port is running and set the 1402 * probe state. Ignore the signature junk (it's unreliable) 1403 * until we get to the softreset code. 1404 */ 1405 if (ahci_port_start(ap)) { 1406 kprintf("%s: failed to start command DMA on port, " 1407 "disabling\n", PORTNAME(ap)); 1408 error = EBUSY; 1409 goto done; 1410 } 1411 if (ap->ap_type == ATA_PORT_T_PM) 1412 ap->ap_probe = ATA_PROBE_GOOD; 1413 else 1414 ap->ap_probe = ATA_PROBE_NEED_SOFT_RESET; 1415 break; 1416 case ENODEV: 1417 /* 1418 * Normal device probe failure 1419 */ 1420 data = ahci_pread(ap, AHCI_PREG_SSTS); 1421 1422 switch(data & AHCI_PREG_SSTS_DET) { 1423 case AHCI_PREG_SSTS_DET_DEV_NE: 1424 kprintf("%s: Device not communicating\n", 1425 PORTNAME(ap)); 1426 break; 1427 case AHCI_PREG_SSTS_DET_PHYOFFLINE: 1428 kprintf("%s: PHY offline\n", 1429 PORTNAME(ap)); 1430 break; 1431 default: 1432 kprintf("%s: No device detected\n", 1433 PORTNAME(ap)); 1434 break; 1435 } 1436 ahci_port_hardstop(ap); 1437 break; 1438 default: 1439 /* 1440 * Abnormal probe (EBUSY) 1441 */ 1442 kprintf("%s: Device on port is bricked\n", 1443 PORTNAME(ap)); 1444 ahci_port_hardstop(ap); 1445 #if 0 1446 rc = ahci_port_reset(ap, atx, 0); 1447 if (rc) { 1448 kprintf("%s: Unable unbrick device\n", 1449 PORTNAME(ap)); 1450 } else { 1451 kprintf("%s: Successfully unbricked\n", 1452 PORTNAME(ap)); 1453 } 1454 #endif 1455 break; 1456 } 1457 1458 /* 1459 * Clean up 1460 */ 1461 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 1462 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_PCS | AHCI_PREG_IS_PRCS); 1463 1464 ap->ap_flags &= ~AP_F_IN_RESET; 1465 1466 if (bootverbose) 1467 kprintf("%s: END HARDRESET %d\n", PORTNAME(ap), error); 1468 return (error); 1469 } 1470 1471 /* 1472 * Hard-stop on hot-swap device removal. See 10.10.1 1473 * 1474 * Place the port in a mode that will allow it to detect hot-swap insertions. 1475 * This is a bit imprecise because just setting-up SCTL to DET_INIT doesn't 1476 * seem to do the job. 1477 * 1478 * FIS reception is left enabled but command processing is disabled. 1479 * Cycling FIS reception (FRE) can brick ports. 1480 */ 1481 void 1482 ahci_port_hardstop(struct ahci_port *ap) 1483 { 1484 struct ahci_ccb *ccb; 1485 struct ata_port *at; 1486 u_int32_t r; 1487 u_int32_t cmd; 1488 int slot; 1489 int i; 1490 1491 /* 1492 * Stop the port. We can't modify things like SUD if the port 1493 * is running. 1494 */ 1495 ap->ap_state = AP_S_FATAL_ERROR; 1496 ap->ap_probe = ATA_PROBE_FAILED; 1497 ap->ap_type = ATA_PORT_T_NONE; 1498 ahci_port_stop(ap, 0); 1499 cmd = ahci_pread(ap, AHCI_PREG_CMD); 1500 1501 /* 1502 * Clean up AT sub-ports on SATA port. 1503 */ 1504 for (i = 0; ap->ap_ata && i < AHCI_MAX_PMPORTS; ++i) { 1505 at = ap->ap_ata[i]; 1506 at->at_type = ATA_PORT_T_NONE; 1507 at->at_probe = ATA_PROBE_FAILED; 1508 } 1509 1510 /* 1511 * Turn off port-multiplier control bit 1512 */ 1513 if (cmd & AHCI_PREG_CMD_PMA) { 1514 cmd &= ~AHCI_PREG_CMD_PMA; 1515 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 1516 } 1517 1518 /* 1519 * Make sure FRE is active. There isn't anything we can do if it 1520 * fails so just ignore errors. 1521 */ 1522 if ((cmd & AHCI_PREG_CMD_FRE) == 0) { 1523 cmd |= AHCI_PREG_CMD_FRE; 1524 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 1525 if ((ap->ap_sc->sc_flags & AHCI_F_IGN_FR) == 0) 1526 ahci_pwait_set(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_FR); 1527 } 1528 1529 /* 1530 * 10.10.3 DET must be set to 0 before setting SUD to 0. 1531 * 10.10.1 place us in the Listen state. 1532 * 1533 * Deactivating SUD only applies if the controller supports SUD. 1534 */ 1535 ahci_pwrite(ap, AHCI_PREG_SCTL, AHCI_PREG_SCTL_IPM_DISABLED); 1536 ahci_os_sleep(1); 1537 if (cmd & AHCI_PREG_CMD_SUD) { 1538 cmd &= ~AHCI_PREG_CMD_SUD; 1539 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 1540 } 1541 ahci_os_sleep(1); 1542 1543 /* 1544 * Transition su to the spin-up state. HVA shall send COMRESET and 1545 * begin initialization sequence (whatever that means). 1546 * 1547 * This only applies if the controller supports SUD. 1548 * NEVER use AHCI_PREG_DET_DISABLE. 1549 */ 1550 cmd |= AHCI_PREG_CMD_SUD; 1551 ahci_pwrite(ap, AHCI_PREG_CMD, cmd); 1552 ahci_os_sleep(1); 1553 1554 /* 1555 * Transition us to the Reset state. Theoretically we send a 1556 * continuous stream of COMRESETs in this state. 1557 */ 1558 r = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_INIT; 1559 if (AhciForceGen1 & (1 << ap->ap_num)) { 1560 kprintf("%s: Force 1.5Gbits\n", PORTNAME(ap)); 1561 r |= AHCI_PREG_SCTL_SPD_GEN1; 1562 } else { 1563 r |= AHCI_PREG_SCTL_SPD_ANY; 1564 } 1565 ahci_pwrite(ap, AHCI_PREG_SCTL, r); 1566 ahci_os_sleep(1); 1567 1568 /* 1569 * Flush SERR_DIAG_X so the TFD can update. 1570 */ 1571 ahci_flush_tfd(ap); 1572 1573 /* 1574 * Clean out pending ccbs 1575 */ 1576 while (ap->ap_active) { 1577 slot = ffs(ap->ap_active) - 1; 1578 ap->ap_active &= ~(1 << slot); 1579 ap->ap_expired &= ~(1 << slot); 1580 --ap->ap_active_cnt; 1581 ccb = &ap->ap_ccbs[slot]; 1582 if (ccb->ccb_xa.flags & ATA_F_TIMEOUT_RUNNING) { 1583 callout_stop(&ccb->ccb_timeout); 1584 ccb->ccb_xa.flags &= ~ATA_F_TIMEOUT_RUNNING; 1585 } 1586 ccb->ccb_xa.flags &= ~(ATA_F_TIMEOUT_DESIRED | 1587 ATA_F_TIMEOUT_EXPIRED); 1588 ccb->ccb_xa.state = ATA_S_TIMEOUT; 1589 ccb->ccb_done(ccb); 1590 ccb->ccb_xa.complete(&ccb->ccb_xa); 1591 } 1592 while (ap->ap_sactive) { 1593 slot = ffs(ap->ap_sactive) - 1; 1594 ap->ap_sactive &= ~(1 << slot); 1595 ap->ap_expired &= ~(1 << slot); 1596 ccb = &ap->ap_ccbs[slot]; 1597 if (ccb->ccb_xa.flags & ATA_F_TIMEOUT_RUNNING) { 1598 callout_stop(&ccb->ccb_timeout); 1599 ccb->ccb_xa.flags &= ~ATA_F_TIMEOUT_RUNNING; 1600 } 1601 ccb->ccb_xa.flags &= ~(ATA_F_TIMEOUT_DESIRED | 1602 ATA_F_TIMEOUT_EXPIRED); 1603 ccb->ccb_xa.state = ATA_S_TIMEOUT; 1604 ccb->ccb_done(ccb); 1605 ccb->ccb_xa.complete(&ccb->ccb_xa); 1606 } 1607 KKASSERT(ap->ap_active_cnt == 0); 1608 1609 while ((ccb = TAILQ_FIRST(&ap->ap_ccb_pending)) != NULL) { 1610 TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry); 1611 ccb->ccb_xa.state = ATA_S_TIMEOUT; 1612 ccb->ccb_xa.flags &= ~ATA_F_TIMEOUT_DESIRED; 1613 ccb->ccb_done(ccb); 1614 ccb->ccb_xa.complete(&ccb->ccb_xa); 1615 } 1616 1617 /* 1618 * Leave us in COMRESET (both SUD and INIT active), the HBA should 1619 * hopefully send us a DIAG_X-related interrupt if it receives 1620 * a COMINIT, and if not that then at least a Phy transition 1621 * interrupt. 1622 * 1623 * If we transition INIT from 1->0 to begin the initalization 1624 * sequence it is unclear if that sequence will remain active 1625 * until the next device insertion. 1626 * 1627 * If we go back to the listen state it is unclear if the 1628 * device will actually send us a COMINIT, since we aren't 1629 * sending any COMRESET's 1630 */ 1631 /* NOP */ 1632 } 1633 1634 /* 1635 * We can't loop on the X bit, a continuous COMINIT received will make 1636 * it loop forever. Just assume one event has built up and clear X 1637 * so the task file descriptor can update. 1638 */ 1639 void 1640 ahci_flush_tfd(struct ahci_port *ap) 1641 { 1642 u_int32_t r; 1643 1644 r = ahci_pread(ap, AHCI_PREG_SERR); 1645 if (r & AHCI_PREG_SERR_DIAG_X) 1646 ahci_pwrite(ap, AHCI_PREG_SERR, AHCI_PREG_SERR_DIAG_X); 1647 } 1648 1649 /* 1650 * Figure out what type of device is connected to the port, ATAPI or 1651 * DISK. 1652 */ 1653 int 1654 ahci_port_signature_detect(struct ahci_port *ap, struct ata_port *at) 1655 { 1656 u_int32_t sig; 1657 1658 sig = ahci_pread(ap, AHCI_PREG_SIG); 1659 if (bootverbose) 1660 kprintf("%s: sig %08x\n", ATANAME(ap, at), sig); 1661 if ((sig & 0xffff0000) == (SATA_SIGNATURE_ATAPI & 0xffff0000)) { 1662 return(ATA_PORT_T_ATAPI); 1663 } else if ((sig & 0xffff0000) == 1664 (SATA_SIGNATURE_PORT_MULTIPLIER & 0xffff0000)) { 1665 return(ATA_PORT_T_PM); 1666 } else { 1667 return(ATA_PORT_T_DISK); 1668 } 1669 } 1670 1671 /* 1672 * Load the DMA descriptor table for a CCB's buffer. 1673 */ 1674 int 1675 ahci_load_prdt(struct ahci_ccb *ccb) 1676 { 1677 struct ahci_port *ap = ccb->ccb_port; 1678 struct ahci_softc *sc = ap->ap_sc; 1679 struct ata_xfer *xa = &ccb->ccb_xa; 1680 struct ahci_prdt *prdt = ccb->ccb_cmd_table->prdt; 1681 bus_dmamap_t dmap = ccb->ccb_dmamap; 1682 struct ahci_cmd_hdr *cmd_slot = ccb->ccb_cmd_hdr; 1683 int error; 1684 1685 if (xa->datalen == 0) { 1686 ccb->ccb_cmd_hdr->prdtl = 0; 1687 return (0); 1688 } 1689 1690 error = bus_dmamap_load(sc->sc_tag_data, dmap, 1691 xa->data, xa->datalen, 1692 ahci_load_prdt_callback, 1693 &prdt, 1694 ((xa->flags & ATA_F_NOWAIT) ? 1695 BUS_DMA_NOWAIT : BUS_DMA_WAITOK)); 1696 if (error != 0) { 1697 kprintf("%s: error %d loading dmamap\n", PORTNAME(ap), error); 1698 return (1); 1699 } 1700 #if 0 1701 if (xa->flags & ATA_F_PIO) 1702 prdt->flags |= htole32(AHCI_PRDT_FLAG_INTR); 1703 #endif 1704 1705 cmd_slot->prdtl = htole16(prdt - ccb->ccb_cmd_table->prdt + 1); 1706 1707 if (xa->flags & ATA_F_READ) 1708 bus_dmamap_sync(sc->sc_tag_data, dmap, BUS_DMASYNC_PREREAD); 1709 if (xa->flags & ATA_F_WRITE) 1710 bus_dmamap_sync(sc->sc_tag_data, dmap, BUS_DMASYNC_PREWRITE); 1711 1712 return (0); 1713 } 1714 1715 /* 1716 * Callback from BUSDMA system to load the segment list. The passed segment 1717 * list is a temporary structure. 1718 */ 1719 static 1720 void 1721 ahci_load_prdt_callback(void *info, bus_dma_segment_t *segs, int nsegs, 1722 int error) 1723 { 1724 struct ahci_prdt *prd = *(void **)info; 1725 u_int64_t addr; 1726 1727 KKASSERT(nsegs <= AHCI_MAX_PRDT); 1728 1729 while (nsegs) { 1730 addr = segs->ds_addr; 1731 prd->dba_hi = htole32((u_int32_t)(addr >> 32)); 1732 prd->dba_lo = htole32((u_int32_t)addr); 1733 prd->flags = htole32(segs->ds_len - 1); 1734 --nsegs; 1735 if (nsegs) 1736 ++prd; 1737 ++segs; 1738 } 1739 *(void **)info = prd; /* return last valid segment */ 1740 } 1741 1742 void 1743 ahci_unload_prdt(struct ahci_ccb *ccb) 1744 { 1745 struct ahci_port *ap = ccb->ccb_port; 1746 struct ahci_softc *sc = ap->ap_sc; 1747 struct ata_xfer *xa = &ccb->ccb_xa; 1748 bus_dmamap_t dmap = ccb->ccb_dmamap; 1749 1750 if (xa->datalen != 0) { 1751 if (xa->flags & ATA_F_READ) { 1752 bus_dmamap_sync(sc->sc_tag_data, dmap, 1753 BUS_DMASYNC_POSTREAD); 1754 } 1755 if (xa->flags & ATA_F_WRITE) { 1756 bus_dmamap_sync(sc->sc_tag_data, dmap, 1757 BUS_DMASYNC_POSTWRITE); 1758 } 1759 bus_dmamap_unload(sc->sc_tag_data, dmap); 1760 1761 /* 1762 * prdbc is only updated by hardware for non-NCQ commands. 1763 */ 1764 if (ccb->ccb_xa.flags & ATA_F_NCQ) { 1765 xa->resid = 0; 1766 } else { 1767 if (ccb->ccb_cmd_hdr->prdbc == 0 && 1768 ccb->ccb_xa.state == ATA_S_COMPLETE) { 1769 kprintf("%s: WARNING! Unload prdbc resid " 1770 "was zero! tag=%d\n", 1771 ATANAME(ap, xa->at), ccb->ccb_slot); 1772 } 1773 xa->resid = xa->datalen - 1774 le32toh(ccb->ccb_cmd_hdr->prdbc); 1775 } 1776 } 1777 } 1778 1779 /* 1780 * Start a command and poll for completion. 1781 * 1782 * timeout is in ms and only counts once the command gets on-chip. 1783 * 1784 * Returns ATA_S_* state, compare against ATA_S_COMPLETE to determine 1785 * that no error occured. 1786 * 1787 * NOTE: If the caller specifies a NULL timeout function the caller is 1788 * responsible for clearing hardware state on failure, but we will 1789 * deal with removing the ccb from any pending queue. 1790 * 1791 * NOTE: NCQ should never be used with this function. 1792 * 1793 * NOTE: If the port is in a failed state and stopped we do not try 1794 * to activate the ccb. 1795 */ 1796 int 1797 ahci_poll(struct ahci_ccb *ccb, int timeout, 1798 void (*timeout_fn)(struct ahci_ccb *)) 1799 { 1800 struct ahci_port *ap = ccb->ccb_port; 1801 1802 if (ccb->ccb_port->ap_state == AP_S_FATAL_ERROR) { 1803 ccb->ccb_xa.state = ATA_S_ERROR; 1804 return(ccb->ccb_xa.state); 1805 } 1806 crit_enter(); 1807 #if 0 1808 kprintf("%s: Start command %02x tag=%d\n", 1809 ATANAME(ccb->ccb_port, ccb->ccb_xa.at), 1810 ccb->ccb_xa.fis->command, ccb->ccb_slot); 1811 #endif 1812 ahci_start(ccb); 1813 1814 do { 1815 ahci_port_intr(ap, 1); 1816 switch(ccb->ccb_xa.state) { 1817 case ATA_S_ONCHIP: 1818 timeout -= ahci_os_softsleep(); 1819 break; 1820 case ATA_S_PENDING: 1821 ahci_os_softsleep(); 1822 ahci_check_active_timeouts(ap); 1823 break; 1824 default: 1825 crit_exit(); 1826 return (ccb->ccb_xa.state); 1827 } 1828 } while (timeout > 0); 1829 1830 kprintf("%s: Poll timeout slot %d CMD: %b TFD: 0x%b SERR: %b\n", 1831 ATANAME(ap, ccb->ccb_xa.at), ccb->ccb_slot, 1832 ahci_pread(ap, AHCI_PREG_CMD), AHCI_PFMT_CMD, 1833 ahci_pread(ap, AHCI_PREG_TFD), AHCI_PFMT_TFD_STS, 1834 ahci_pread(ap, AHCI_PREG_SERR), AHCI_PFMT_SERR); 1835 1836 timeout_fn(ccb); 1837 1838 crit_exit(); 1839 1840 return(ccb->ccb_xa.state); 1841 } 1842 1843 /* 1844 * When polling we have to check if the currently active CCB(s) 1845 * have timed out as the callout will be deadlocked while we 1846 * hold the port lock. 1847 */ 1848 void 1849 ahci_check_active_timeouts(struct ahci_port *ap) 1850 { 1851 struct ahci_ccb *ccb; 1852 u_int32_t mask; 1853 int tag; 1854 1855 mask = ap->ap_active | ap->ap_sactive; 1856 while (mask) { 1857 tag = ffs(mask) - 1; 1858 mask &= ~(1 << tag); 1859 ccb = &ap->ap_ccbs[tag]; 1860 if (ccb->ccb_xa.flags & ATA_F_TIMEOUT_EXPIRED) { 1861 ahci_ata_cmd_timeout(ccb); 1862 } 1863 } 1864 } 1865 1866 static 1867 __inline 1868 void 1869 ahci_start_timeout(struct ahci_ccb *ccb) 1870 { 1871 if (ccb->ccb_xa.flags & ATA_F_TIMEOUT_DESIRED) { 1872 ccb->ccb_xa.flags |= ATA_F_TIMEOUT_RUNNING; 1873 callout_reset(&ccb->ccb_timeout, 1874 (ccb->ccb_xa.timeout * hz + 999) / 1000, 1875 ahci_ata_cmd_timeout_unserialized, ccb); 1876 } 1877 } 1878 1879 void 1880 ahci_start(struct ahci_ccb *ccb) 1881 { 1882 struct ahci_port *ap = ccb->ccb_port; 1883 struct ahci_softc *sc = ap->ap_sc; 1884 1885 KKASSERT(ccb->ccb_xa.state == ATA_S_PENDING); 1886 1887 /* Zero transferred byte count before transfer */ 1888 ccb->ccb_cmd_hdr->prdbc = 0; 1889 1890 /* Sync command list entry and corresponding command table entry */ 1891 bus_dmamap_sync(sc->sc_tag_cmdh, 1892 AHCI_DMA_MAP(ap->ap_dmamem_cmd_list), 1893 BUS_DMASYNC_PREWRITE); 1894 bus_dmamap_sync(sc->sc_tag_cmdt, 1895 AHCI_DMA_MAP(ap->ap_dmamem_cmd_table), 1896 BUS_DMASYNC_PREWRITE); 1897 1898 /* Prepare RFIS area for write by controller */ 1899 bus_dmamap_sync(sc->sc_tag_rfis, 1900 AHCI_DMA_MAP(ap->ap_dmamem_rfis), 1901 BUS_DMASYNC_PREREAD); 1902 1903 /* 1904 * There's no point trying to optimize this, it only shaves a few 1905 * nanoseconds so just queue the command and call our generic issue. 1906 */ 1907 ahci_issue_pending_commands(ap, ccb); 1908 } 1909 1910 /* 1911 * While holding the port lock acquire exclusive access to the port. 1912 * 1913 * This is used when running the state machine to initialize and identify 1914 * targets over a port multiplier. Setting exclusive access prevents 1915 * ahci_port_intr() from activating any requests sitting on the pending 1916 * queue. 1917 */ 1918 void 1919 ahci_beg_exclusive_access(struct ahci_port *ap, struct ata_port *at) 1920 { 1921 KKASSERT((ap->ap_flags & AP_F_EXCLUSIVE_ACCESS) == 0); 1922 ap->ap_flags |= AP_F_EXCLUSIVE_ACCESS; 1923 while (ap->ap_active || ap->ap_sactive) { 1924 ahci_port_intr(ap, 1); 1925 ahci_os_softsleep(); 1926 } 1927 } 1928 1929 void 1930 ahci_end_exclusive_access(struct ahci_port *ap, struct ata_port *at) 1931 { 1932 KKASSERT((ap->ap_flags & AP_F_EXCLUSIVE_ACCESS) != 0); 1933 ap->ap_flags &= ~AP_F_EXCLUSIVE_ACCESS; 1934 ahci_issue_pending_commands(ap, NULL); 1935 } 1936 1937 #if 0 1938 1939 static void 1940 fubar(struct ahci_ccb *ccb) 1941 { 1942 struct ahci_port *ap = ccb->ccb_port; 1943 struct ahci_cmd_hdr *cmd; 1944 struct ahci_cmd_table *tab; 1945 struct ahci_prdt *prdt; 1946 int i; 1947 1948 kprintf("%s: ISSUE %02x\n", 1949 ATANAME(ap, ccb->ccb_xa.at), 1950 ccb->ccb_xa.fis->command); 1951 cmd = ccb->ccb_cmd_hdr; 1952 tab = ccb->ccb_cmd_table; 1953 prdt = ccb->ccb_cmd_table->prdt; 1954 kprintf("cmd flags=%04x prdtl=%d prdbc=%d ctba=%08x%08x\n", 1955 cmd->flags, cmd->prdtl, cmd->prdbc, 1956 cmd->ctba_hi, cmd->ctba_lo); 1957 for (i = 0; i < cmd->prdtl; ++i) { 1958 kprintf("\t%d dba=%08x%08x res=%08x flags=%08x\n", 1959 i, prdt->dba_hi, prdt->dba_lo, prdt->reserved, 1960 prdt->flags); 1961 } 1962 kprintf("tab\n"); 1963 } 1964 1965 #endif 1966 1967 /* 1968 * If ccb is not NULL enqueue and/or issue it. 1969 * 1970 * If ccb is NULL issue whatever we can from the queue. However, nothing 1971 * new is issued if the exclusive access flag is set or expired ccb's are 1972 * present. 1973 * 1974 * If existing commands are still active (ap_active/ap_sactive) we can only 1975 * issue matching new commands. 1976 */ 1977 void 1978 ahci_issue_pending_commands(struct ahci_port *ap, struct ahci_ccb *ccb) 1979 { 1980 u_int32_t mask; 1981 int limit; 1982 1983 /* 1984 * Enqueue the ccb. 1985 * 1986 * If just running the queue and in exclusive access mode we 1987 * just return. Also in this case if there are any expired ccb's 1988 * we want to clear the queue so the port can be safely stopped. 1989 */ 1990 if (ccb) { 1991 TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry); 1992 } else if ((ap->ap_flags & AP_F_EXCLUSIVE_ACCESS) || ap->ap_expired) { 1993 return; 1994 } 1995 1996 /* 1997 * Pull the next ccb off the queue and run it if possible. 1998 */ 1999 if ((ccb = TAILQ_FIRST(&ap->ap_ccb_pending)) == NULL) 2000 return; 2001 2002 /* 2003 * Handle exclusivity requirements. 2004 * 2005 * ATA_F_EXCLUSIVE is used when we want to be the only command 2006 * running. 2007 * 2008 * ATA_F_AUTOSENSE is used when we want the D2H rfis loaded 2009 * back into the ccb on a normal (non-errored) command completion. 2010 * For example, for PM requests to target 15. Because the AHCI 2011 * spec does not stop the command processor and has only one rfis 2012 * area (for non-FBSS anyway), AUTOSENSE currently implies EXCLUSIVE. 2013 * Otherwise multiple completions can destroy the rfis data before 2014 * we have a chance to copy it. 2015 */ 2016 if (ap->ap_active & ~ap->ap_expired) { 2017 /* 2018 * There may be multiple ccb's already running, 2019 * if any are running and ap_run_flags sets 2020 * one of these flags then we know only one is 2021 * running. 2022 * 2023 * XXX Current AUTOSENSE code forces exclusivity 2024 * to simplify the code. 2025 */ 2026 if (ap->ap_run_flags & 2027 (ATA_F_EXCLUSIVE | ATA_F_AUTOSENSE)) { 2028 return; 2029 } 2030 2031 if (ccb->ccb_xa.flags & 2032 (ATA_F_EXCLUSIVE | ATA_F_AUTOSENSE)) { 2033 return; 2034 } 2035 } 2036 2037 2038 if (ccb->ccb_xa.flags & ATA_F_NCQ) { 2039 /* 2040 * The next command is a NCQ command and can be issued as 2041 * long as currently active commands are not standard. 2042 */ 2043 if (ap->ap_active) { 2044 KKASSERT(ap->ap_active_cnt > 0); 2045 return; 2046 } 2047 KKASSERT(ap->ap_active_cnt == 0); 2048 2049 mask = 0; 2050 do { 2051 TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry); 2052 mask |= 1 << ccb->ccb_slot; 2053 ccb->ccb_xa.state = ATA_S_ONCHIP; 2054 ahci_start_timeout(ccb); 2055 ap->ap_run_flags = ccb->ccb_xa.flags; 2056 ccb = TAILQ_FIRST(&ap->ap_ccb_pending); 2057 } while (ccb && (ccb->ccb_xa.flags & ATA_F_NCQ) && 2058 (ap->ap_run_flags & 2059 (ATA_F_EXCLUSIVE | ATA_F_AUTOSENSE)) == 0); 2060 2061 ap->ap_sactive |= mask; 2062 ahci_pwrite(ap, AHCI_PREG_SACT, mask); 2063 ahci_pwrite(ap, AHCI_PREG_CI, mask); 2064 } else { 2065 /* 2066 * The next command is a standard command and can be issued 2067 * as long as currently active commands are not NCQ. 2068 * 2069 * We limit ourself to 1 command if we have a port multiplier, 2070 * (at least without FBSS support), otherwise timeouts on 2071 * one port can race completions on other ports (see 2072 * ahci_ata_cmd_timeout() for more information). 2073 * 2074 * If not on a port multiplier generally allow up to 4 2075 * standard commands to be enqueued. Remember that the 2076 * command processor will still process them sequentially. 2077 */ 2078 if (ap->ap_sactive) 2079 return; 2080 if (ap->ap_type == ATA_PORT_T_PM) 2081 limit = 1; 2082 else if (ap->ap_sc->sc_ncmds > 4) 2083 limit = 4; 2084 else 2085 limit = 2; 2086 2087 while (ap->ap_active_cnt < limit && ccb && 2088 (ccb->ccb_xa.flags & ATA_F_NCQ) == 0) { 2089 TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry); 2090 #if 0 2091 fubar(ccb); 2092 #endif 2093 ap->ap_active |= 1 << ccb->ccb_slot; 2094 ap->ap_active_cnt++; 2095 ap->ap_run_flags = ccb->ccb_xa.flags; 2096 ccb->ccb_xa.state = ATA_S_ONCHIP; 2097 ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot); 2098 ahci_start_timeout(ccb); 2099 ccb = TAILQ_FIRST(&ap->ap_ccb_pending); 2100 if (ccb && (ccb->ccb_xa.flags & 2101 (ATA_F_EXCLUSIVE | ATA_F_AUTOSENSE))) { 2102 break; 2103 } 2104 } 2105 } 2106 } 2107 2108 void 2109 ahci_intr(void *arg) 2110 { 2111 struct ahci_softc *sc = arg; 2112 struct ahci_port *ap; 2113 u_int32_t is; 2114 u_int32_t ack; 2115 int port; 2116 2117 /* 2118 * Check if the master enable is up, and whether any interrupts are 2119 * pending. 2120 */ 2121 if ((sc->sc_flags & AHCI_F_INT_GOOD) == 0) 2122 return; 2123 is = ahci_read(sc, AHCI_REG_IS); 2124 if (is == 0 || is == 0xffffffff) { 2125 return; 2126 } 2127 is &= sc->sc_portmask; 2128 2129 #ifdef AHCI_COALESCE 2130 /* Check coalescing interrupt first */ 2131 if (is & sc->sc_ccc_mask) { 2132 DPRINTF(AHCI_D_INTR, "%s: command coalescing interrupt\n", 2133 DEVNAME(sc)); 2134 is &= ~sc->sc_ccc_mask; 2135 is |= sc->sc_ccc_ports_cur; 2136 } 2137 #endif 2138 2139 /* 2140 * Process interrupts for each port in a non-blocking fashion. 2141 * 2142 * The global IS bit is forced on if any unmasked port interrupts 2143 * are pending, even if we clear. 2144 */ 2145 for (ack = 0; is; is &= ~(1 << port)) { 2146 port = ffs(is) - 1; 2147 ack |= 1 << port; 2148 2149 ap = sc->sc_ports[port]; 2150 if (ap == NULL) 2151 continue; 2152 2153 if (ahci_os_lock_port_nb(ap) == 0) { 2154 ahci_port_intr(ap, 0); 2155 ahci_os_unlock_port(ap); 2156 } else { 2157 ahci_pwrite(ap, AHCI_PREG_IE, 0); 2158 ahci_os_signal_port_thread(ap, AP_SIGF_PORTINT); 2159 } 2160 } 2161 ahci_write(sc, AHCI_REG_IS, ack); 2162 } 2163 2164 /* 2165 * Core called from helper thread. 2166 */ 2167 void 2168 ahci_port_thread_core(struct ahci_port *ap, int mask) 2169 { 2170 /* 2171 * Process any expired timedouts. 2172 */ 2173 ahci_os_lock_port(ap); 2174 if (mask & AP_SIGF_TIMEOUT) { 2175 ahci_check_active_timeouts(ap); 2176 } 2177 2178 /* 2179 * Process port interrupts which require a higher level of 2180 * intervention. 2181 */ 2182 if (mask & AP_SIGF_PORTINT) { 2183 ahci_port_intr(ap, 1); 2184 ahci_port_interrupt_enable(ap); 2185 ahci_os_unlock_port(ap); 2186 } else if (ap->ap_probe != ATA_PROBE_FAILED) { 2187 ahci_port_intr(ap, 1); 2188 ahci_port_interrupt_enable(ap); 2189 ahci_os_unlock_port(ap); 2190 } else { 2191 ahci_os_unlock_port(ap); 2192 } 2193 } 2194 2195 /* 2196 * Core per-port interrupt handler. 2197 * 2198 * If blockable is 0 we cannot call ahci_os_sleep() at all and we can only 2199 * deal with normal command completions which do not require blocking. 2200 */ 2201 void 2202 ahci_port_intr(struct ahci_port *ap, int blockable) 2203 { 2204 struct ahci_softc *sc = ap->ap_sc; 2205 u_int32_t is, ci_saved, ci_masked; 2206 int slot; 2207 struct ahci_ccb *ccb = NULL; 2208 struct ata_port *ccb_at = NULL; 2209 volatile u_int32_t *active; 2210 const u_int32_t blockable_mask = AHCI_PREG_IS_TFES | 2211 AHCI_PREG_IS_IFS | 2212 AHCI_PREG_IS_PCS | 2213 AHCI_PREG_IS_PRCS | 2214 AHCI_PREG_IS_HBFS | 2215 AHCI_PREG_IS_OFS | 2216 AHCI_PREG_IS_UFS; 2217 2218 enum { NEED_NOTHING, NEED_RESTART, NEED_HOTPLUG_INSERT, 2219 NEED_HOTPLUG_REMOVE } need = NEED_NOTHING; 2220 2221 /* 2222 * All basic command completions are always processed. 2223 */ 2224 is = ahci_pread(ap, AHCI_PREG_IS); 2225 if (is & AHCI_PREG_IS_DPS) 2226 ahci_pwrite(ap, AHCI_PREG_IS, is & AHCI_PREG_IS_DPS); 2227 2228 /* 2229 * If we can't block then we can't handle these here. Disable 2230 * the interrupts in question so we don't live-lock, the helper 2231 * thread will re-enable them. 2232 * 2233 * If the port is in a completely failed state we do not want 2234 * to drop through to failed-command-processing if blockable is 0, 2235 * just let the thread deal with it all. 2236 * 2237 * Otherwise we fall through and still handle DHRS and any commands 2238 * which completed normally. Even if we are errored we haven't 2239 * stopped the port yet so CI/SACT are still good. 2240 */ 2241 if (blockable == 0) { 2242 if (ap->ap_state == AP_S_FATAL_ERROR) { 2243 ahci_pwrite(ap, AHCI_PREG_IE, 0); 2244 ahci_os_signal_port_thread(ap, AP_SIGF_PORTINT); 2245 return; 2246 } 2247 if (is & blockable_mask) { 2248 ahci_pwrite(ap, AHCI_PREG_IE, 0); 2249 ahci_os_signal_port_thread(ap, AP_SIGF_PORTINT); 2250 return; 2251 } 2252 } 2253 2254 /* 2255 * Either NCQ or non-NCQ commands will be active, never both. 2256 */ 2257 if (ap->ap_sactive) { 2258 KKASSERT(ap->ap_active == 0); 2259 KKASSERT(ap->ap_active_cnt == 0); 2260 ci_saved = ahci_pread(ap, AHCI_PREG_SACT); 2261 active = &ap->ap_sactive; 2262 } else { 2263 ci_saved = ahci_pread(ap, AHCI_PREG_CI); 2264 active = &ap->ap_active; 2265 } 2266 KKASSERT(!(ap->ap_sactive && ap->ap_active)); 2267 #if 0 2268 kprintf("CHECK act=%08x/%08x sact=%08x/%08x\n", 2269 ap->ap_active, ahci_pread(ap, AHCI_PREG_CI), 2270 ap->ap_sactive, ahci_pread(ap, AHCI_PREG_SACT)); 2271 #endif 2272 2273 if (is & AHCI_PREG_IS_TFES) { 2274 /* 2275 * Command failed (blockable). 2276 * 2277 * See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2. 2278 * 2279 * This stops command processing. 2280 */ 2281 u_int32_t tfd, serr; 2282 int err_slot; 2283 2284 process_error: 2285 tfd = ahci_pread(ap, AHCI_PREG_TFD); 2286 serr = ahci_pread(ap, AHCI_PREG_SERR); 2287 2288 /* 2289 * Load the error slot and restart command processing. 2290 * CLO if we need to. The error slot may not be valid. 2291 * MUST BE DONE BEFORE CLEARING ST! 2292 * 2293 * Cycle ST. 2294 * 2295 * It is unclear but we may have to clear SERR to reenable 2296 * error processing. 2297 */ 2298 err_slot = AHCI_PREG_CMD_CCS(ahci_pread(ap, AHCI_PREG_CMD)); 2299 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_TFES | 2300 AHCI_PREG_IS_PSS | 2301 AHCI_PREG_IS_DHRS | 2302 AHCI_PREG_IS_SDBS); 2303 is &= ~(AHCI_PREG_IS_TFES | AHCI_PREG_IS_PSS | 2304 AHCI_PREG_IS_DHRS | AHCI_PREG_IS_SDBS); 2305 ahci_pwrite(ap, AHCI_PREG_SERR, serr); 2306 ahci_port_stop(ap, 0); 2307 ahci_os_hardsleep(10); 2308 if (tfd & (AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) { 2309 kprintf("%s: Issuing CLO\n", PORTNAME(ap)); 2310 ahci_port_clo(ap); 2311 } 2312 ahci_port_start(ap); 2313 need = NEED_RESTART; 2314 2315 /* 2316 * ATAPI errors are fairly common from probing, just 2317 * report disk errors or if bootverbose is on. 2318 */ 2319 if (bootverbose || ap->ap_type != ATA_PORT_T_ATAPI) { 2320 kprintf("%s: TFES slot %d ci_saved = %08x\n", 2321 PORTNAME(ap), err_slot, ci_saved); 2322 } 2323 2324 /* 2325 * If we got an error on an error CCB just complete it 2326 * with an error. ci_saved has the mask to restart 2327 * (the err_ccb will be removed from it by finish_error). 2328 */ 2329 if (ap->ap_flags & AP_F_ERR_CCB_RESERVED) { 2330 err_slot = ap->ap_err_ccb->ccb_slot; 2331 goto finish_error; 2332 } 2333 2334 /* 2335 * If NCQ commands were active get the error slot from 2336 * the log page. NCQ is not supported for PM's so this 2337 * is a direct-attached target. 2338 * 2339 * Otherwise if no commands were active we have a problem. 2340 * 2341 * Otherwise if the error slot is bad we have a problem. 2342 * 2343 * Otherwise process the error for the slot. 2344 */ 2345 if (ap->ap_sactive) { 2346 err_slot = ahci_port_read_ncq_error(ap, 0); 2347 } else if (ap->ap_active == 0) { 2348 kprintf("%s: TFES with no commands pending\n", 2349 PORTNAME(ap)); 2350 err_slot = -1; 2351 } else if (err_slot < 0 || err_slot >= ap->ap_sc->sc_ncmds) { 2352 kprintf("%s: bad error slot %d\n", 2353 PORTNAME(ap), err_slot); 2354 err_slot = -1; 2355 } else { 2356 ccb = &ap->ap_ccbs[err_slot]; 2357 2358 /* 2359 * Validate the errored ccb. Note that ccb_at can 2360 * be NULL for direct-attached ccb's. 2361 * 2362 * Copy received taskfile data from the RFIS. 2363 */ 2364 if (ccb->ccb_xa.state == ATA_S_ONCHIP) { 2365 ccb_at = ccb->ccb_xa.at; 2366 memcpy(&ccb->ccb_xa.rfis, ap->ap_rfis->rfis, 2367 sizeof(struct ata_fis_d2h)); 2368 if (bootverbose) { 2369 kprintf("%s: Copying rfis slot %d\n", 2370 ATANAME(ap, ccb_at), err_slot); 2371 } 2372 } else { 2373 kprintf("%s: Cannot copy rfis, CCB slot " 2374 "%d is not on-chip (state=%d)\n", 2375 ATANAME(ap, ccb->ccb_xa.at), 2376 err_slot, ccb->ccb_xa.state); 2377 err_slot = -1; 2378 } 2379 } 2380 2381 /* 2382 * If we could not determine the errored slot then 2383 * reset the port. 2384 */ 2385 if (err_slot < 0) { 2386 kprintf("%s: TFES: Unable to determine errored slot\n", 2387 PORTNAME(ap)); 2388 if (ap->ap_flags & AP_F_IN_RESET) 2389 goto fatal; 2390 goto failall; 2391 } 2392 2393 /* 2394 * Finish error on slot. We will restart ci_saved 2395 * commands except the errored slot which we generate 2396 * a failure for. 2397 */ 2398 finish_error: 2399 ccb = &ap->ap_ccbs[err_slot]; 2400 ci_saved &= ~(1 << err_slot); 2401 KKASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP); 2402 ccb->ccb_xa.state = ATA_S_ERROR; 2403 } else if (is & AHCI_PREG_IS_DHRS) { 2404 /* 2405 * Command posted D2H register FIS to the rfis (non-blocking). 2406 * 2407 * A normal completion with an error may set DHRS instead 2408 * of TFES. The CCS bits are only valid if ERR was set. 2409 * If ERR is set command processing was probably stopped. 2410 * 2411 * If ERR was not set we can only copy-back data for 2412 * exclusive-mode commands because otherwise we won't know 2413 * which tag the rfis belonged to. 2414 * 2415 * err_slot must be read from the CCS before any other port 2416 * action, such as stopping the port. 2417 * 2418 * WARNING! This is not well documented in the AHCI spec. 2419 * It can be found in the state machine tables 2420 * but not in the explanations. 2421 */ 2422 u_int32_t tfd; 2423 u_int32_t cmd; 2424 int err_slot; 2425 2426 tfd = ahci_pread(ap, AHCI_PREG_TFD); 2427 cmd = ahci_pread(ap, AHCI_PREG_CMD); 2428 2429 if ((tfd & AHCI_PREG_TFD_STS_ERR) && 2430 (cmd & AHCI_PREG_CMD_CR) == 0) { 2431 err_slot = AHCI_PREG_CMD_CCS( 2432 ahci_pread(ap, AHCI_PREG_CMD)); 2433 ccb = &ap->ap_ccbs[err_slot]; 2434 kprintf("%s: DHRS tfd=%b err_slot=%d cmd=%02x\n", 2435 PORTNAME(ap), 2436 tfd, AHCI_PFMT_TFD_STS, 2437 err_slot, ccb->ccb_xa.fis->command); 2438 goto process_error; 2439 } 2440 /* 2441 * NO ELSE... copy back is in the normal command completion 2442 * code and only if no error occured and ATA_F_AUTOSENSE 2443 * was set. 2444 */ 2445 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_DHRS); 2446 } 2447 2448 /* 2449 * Device notification to us (non-blocking) 2450 * 2451 * NOTE! On some parts notification bits can cause an IPMS 2452 * interrupt instead of a SDBS interrupt. 2453 * 2454 * NOTE! On some parts (e.g. VBOX, probably intel ICHx), 2455 * SDBS notifies us of the completion of a NCQ command 2456 * and DBS does not. 2457 */ 2458 if (is & (AHCI_PREG_IS_SDBS | AHCI_PREG_IS_IPMS)) { 2459 u_int32_t data; 2460 2461 ahci_pwrite(ap, AHCI_PREG_IS, 2462 AHCI_PREG_IS_SDBS | AHCI_PREG_IS_IPMS); 2463 if (sc->sc_cap & AHCI_REG_CAP_SSNTF) { 2464 data = ahci_pread(ap, AHCI_PREG_SNTF); 2465 if (data) { 2466 ahci_pwrite(ap, AHCI_PREG_IS, 2467 AHCI_PREG_IS_SDBS); 2468 kprintf("%s: NOTIFY %08x\n", 2469 PORTNAME(ap), data); 2470 ahci_pwrite(ap, AHCI_PREG_SERR, 2471 AHCI_PREG_SERR_DIAG_N); 2472 ahci_pwrite(ap, AHCI_PREG_SNTF, data); 2473 ahci_cam_changed(ap, NULL, -1); 2474 } 2475 } 2476 is &= ~(AHCI_PREG_IS_SDBS | AHCI_PREG_IS_IPMS); 2477 } 2478 2479 /* 2480 * Spurious IFS errors (blockable). 2481 * 2482 * Spurious IFS errors can occur while we are doing a reset 2483 * sequence through a PM. Try to recover if we are being asked 2484 * to ignore IFS errors during these periods. 2485 */ 2486 if ((is & AHCI_PREG_IS_IFS) && (ap->ap_flags & AP_F_IGNORE_IFS)) { 2487 u_int32_t serr = ahci_pread(ap, AHCI_PREG_SERR); 2488 if ((ap->ap_flags & AP_F_IFS_IGNORED) == 0) { 2489 kprintf("%s: Ignoring IFS (XXX) (IS: %b, SERR: %b)\n", 2490 PORTNAME(ap), 2491 is, AHCI_PFMT_IS, 2492 serr, AHCI_PFMT_SERR); 2493 ap->ap_flags |= AP_F_IFS_IGNORED; 2494 } 2495 ap->ap_flags |= AP_F_IFS_OCCURED; 2496 ahci_pwrite(ap, AHCI_PREG_SERR, -1); 2497 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS); 2498 is &= ~AHCI_PREG_IS_IFS; 2499 ahci_port_stop(ap, 0); 2500 ahci_port_start(ap); 2501 kprintf("%s: Spurious IFS error\n", PORTNAME(ap)); 2502 goto failall; 2503 /* need = NEED_RESTART; */ 2504 } 2505 2506 /* 2507 * Port change (hot-plug) (blockable). 2508 * 2509 * A PCS interrupt will occur on hot-plug once communication is 2510 * established. 2511 * 2512 * A PRCS interrupt will occur on hot-unplug (and possibly also 2513 * on hot-plug). 2514 * 2515 * XXX We can then check the CPS (Cold Presence State) bit, if 2516 * supported, to determine if a device is plugged in or not and do 2517 * the right thing. 2518 * 2519 * WARNING: A PCS interrupt is cleared by clearing DIAG_X, and 2520 * can also occur if an unsolicited COMINIT is received. 2521 * If this occurs command processing is automatically 2522 * stopped (CR goes inactive) and the port must be stopped 2523 * and restarted. 2524 */ 2525 2526 /* ignore AHCI_PREG_IS_PRCS when link power management is on */ 2527 if (ap->link_pwr_mgmt != AHCI_LINK_PWR_MGMT_NONE) { 2528 is &= ~AHCI_PREG_IS_PRCS; 2529 } 2530 2531 if (is & (AHCI_PREG_IS_PCS | AHCI_PREG_IS_PRCS)) { 2532 kprintf("%s: Transient Errors: %b\n", 2533 PORTNAME(ap), is, AHCI_PFMT_IS); 2534 ahci_pwrite(ap, AHCI_PREG_SERR, 2535 (AHCI_PREG_SERR_DIAG_N | AHCI_PREG_SERR_DIAG_X)); 2536 ahci_pwrite(ap, AHCI_PREG_IS, 2537 is & (AHCI_PREG_IS_PCS | AHCI_PREG_IS_PRCS)); 2538 is &= ~(AHCI_PREG_IS_PCS | AHCI_PREG_IS_PRCS); 2539 ahci_port_stop(ap, 0); 2540 2541 switch (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) { 2542 case AHCI_PREG_SSTS_DET_DEV: 2543 if (ap->ap_probe == ATA_PROBE_FAILED) { 2544 need = NEED_HOTPLUG_INSERT; 2545 goto fatal; 2546 } 2547 need = NEED_RESTART; 2548 break; 2549 default: 2550 if (ap->ap_probe != ATA_PROBE_FAILED) { 2551 need = NEED_HOTPLUG_REMOVE; 2552 goto fatal; 2553 } 2554 need = NEED_RESTART; 2555 break; 2556 } 2557 } 2558 2559 /* 2560 * Check for remaining errors - they are fatal. (blockable) 2561 */ 2562 if (is & (AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS | AHCI_PREG_IS_IFS | 2563 AHCI_PREG_IS_OFS | AHCI_PREG_IS_UFS)) { 2564 u_int32_t serr; 2565 2566 ahci_pwrite(ap, AHCI_PREG_IS, 2567 is & (AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS | 2568 AHCI_PREG_IS_IFS | AHCI_PREG_IS_OFS | 2569 AHCI_PREG_IS_UFS)); 2570 serr = ahci_pread(ap, AHCI_PREG_SERR); 2571 kprintf("%s: Unrecoverable errors (IS: %b, SERR: %b), " 2572 "disabling port.\n", 2573 PORTNAME(ap), 2574 is, AHCI_PFMT_IS, 2575 serr, AHCI_PFMT_SERR 2576 ); 2577 is &= ~(AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS | 2578 AHCI_PREG_IS_IFS | AHCI_PREG_IS_OFS | 2579 AHCI_PREG_IS_UFS); 2580 /* XXX try recovery first */ 2581 goto fatal; 2582 } 2583 2584 /* 2585 * Fail all outstanding commands if we know the port won't recover. 2586 * 2587 * We may have a ccb_at if the failed command is known and was 2588 * being sent to a device over a port multiplier (PM). In this 2589 * case if the port itself has not completely failed we fail just 2590 * the commands related to that target. 2591 * 2592 * ci_saved contains the mask of active commands as of when the 2593 * error occured, prior to any port stops. 2594 */ 2595 if (ap->ap_state == AP_S_FATAL_ERROR) { 2596 fatal: 2597 ap->ap_state = AP_S_FATAL_ERROR; 2598 ahci_port_stop(ap, 0); 2599 failall: 2600 kprintf("%s: Failing all commands\n", PORTNAME(ap)); 2601 2602 /* 2603 * Error all the active slots not already errored. If 2604 * running across a PM try to error out just the slots 2605 * related to the target. 2606 */ 2607 ci_masked = ci_saved & *active & ~ap->ap_expired; 2608 while (ci_masked) { 2609 slot = ffs(ci_masked) - 1; 2610 ccb = &ap->ap_ccbs[slot]; 2611 if (ccb_at == ccb->ccb_xa.at || 2612 ap->ap_state == AP_S_FATAL_ERROR) { 2613 ccb->ccb_xa.state = ATA_S_TIMEOUT; 2614 ap->ap_expired |= 1 << slot; 2615 ci_saved &= ~(1 << slot); 2616 } 2617 ci_masked &= ~(1 << slot); 2618 } 2619 2620 /* 2621 * Clear bits in ci_saved (cause completions to be run) 2622 * for all slots which are not active. 2623 */ 2624 ci_saved &= ~*active; 2625 2626 /* 2627 * Don't restart the port if our problems were deemed fatal. 2628 * 2629 * Also acknowlege all fatal interrupt sources to prevent 2630 * a livelock. 2631 */ 2632 if (ap->ap_state == AP_S_FATAL_ERROR) { 2633 if (need == NEED_RESTART) 2634 need = NEED_NOTHING; 2635 ahci_pwrite(ap, AHCI_PREG_IS, 2636 AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS | 2637 AHCI_PREG_IS_IFS | AHCI_PREG_IS_OFS | 2638 AHCI_PREG_IS_UFS); 2639 } 2640 } 2641 2642 /* 2643 * CCB completion (non blocking). 2644 * 2645 * CCB completion is detected by noticing its slot's bit in CI has 2646 * changed to zero some time after we activated it. 2647 * If we are polling, we may only be interested in particular slot(s). 2648 * 2649 * Any active bits not saved are completed within the restrictions 2650 * imposed by the caller. 2651 */ 2652 ci_masked = ~ci_saved & *active; 2653 while (ci_masked) { 2654 slot = ffs(ci_masked) - 1; 2655 ccb = &ap->ap_ccbs[slot]; 2656 ci_masked &= ~(1 << slot); 2657 2658 DPRINTF(AHCI_D_INTR, "%s: slot %d is complete%s\n", 2659 PORTNAME(ap), slot, ccb->ccb_xa.state == ATA_S_ERROR ? 2660 " (error)" : ""); 2661 2662 bus_dmamap_sync(sc->sc_tag_cmdh, 2663 AHCI_DMA_MAP(ap->ap_dmamem_cmd_list), 2664 BUS_DMASYNC_POSTWRITE); 2665 2666 bus_dmamap_sync(sc->sc_tag_cmdt, 2667 AHCI_DMA_MAP(ap->ap_dmamem_cmd_table), 2668 BUS_DMASYNC_POSTWRITE); 2669 2670 bus_dmamap_sync(sc->sc_tag_rfis, 2671 AHCI_DMA_MAP(ap->ap_dmamem_rfis), 2672 BUS_DMASYNC_POSTREAD); 2673 2674 *active &= ~(1 << ccb->ccb_slot); 2675 if (active == &ap->ap_active) { 2676 KKASSERT(ap->ap_active_cnt > 0); 2677 --ap->ap_active_cnt; 2678 } 2679 2680 /* 2681 * Complete the ccb. If the ccb was marked expired it 2682 * was probably already removed from the command processor, 2683 * so don't take the clear ci_saved bit as meaning the 2684 * command actually succeeded, it didn't. 2685 */ 2686 if (ap->ap_expired & (1 << ccb->ccb_slot)) { 2687 ap->ap_expired &= ~(1 << ccb->ccb_slot); 2688 ccb->ccb_xa.state = ATA_S_TIMEOUT; 2689 ccb->ccb_done(ccb); 2690 ccb->ccb_xa.complete(&ccb->ccb_xa); 2691 } else { 2692 if (ccb->ccb_xa.state == ATA_S_ONCHIP) { 2693 ccb->ccb_xa.state = ATA_S_COMPLETE; 2694 if (ccb->ccb_xa.flags & ATA_F_AUTOSENSE) { 2695 memcpy(&ccb->ccb_xa.rfis, 2696 ap->ap_rfis->rfis, 2697 sizeof(struct ata_fis_d2h)); 2698 if (ccb->ccb_xa.state == ATA_S_TIMEOUT) 2699 ccb->ccb_xa.state = ATA_S_ERROR; 2700 } 2701 } 2702 ccb->ccb_done(ccb); 2703 } 2704 } 2705 ahci_issue_pending_commands(ap, NULL); 2706 2707 /* 2708 * Cleanup. Will not be set if non-blocking. 2709 */ 2710 switch(need) { 2711 case NEED_RESTART: 2712 /* 2713 * A recoverable error occured and we can restart outstanding 2714 * commands on the port. 2715 */ 2716 ci_saved &= ~ap->ap_expired; 2717 if (ci_saved) { 2718 kprintf("%s: Restart %08x\n", PORTNAME(ap), ci_saved); 2719 ahci_issue_saved_commands(ap, ci_saved); 2720 } 2721 break; 2722 case NEED_HOTPLUG_INSERT: 2723 /* 2724 * A hot-plug insertion event has occured and all 2725 * outstanding commands have already been revoked. 2726 * 2727 * Don't recurse if this occurs while we are 2728 * resetting the port. 2729 */ 2730 if ((ap->ap_flags & AP_F_IN_RESET) == 0) { 2731 kprintf("%s: HOTPLUG - Device inserted\n", 2732 PORTNAME(ap)); 2733 ap->ap_probe = ATA_PROBE_NEED_INIT; 2734 ahci_cam_changed(ap, NULL, -1); 2735 } 2736 break; 2737 case NEED_HOTPLUG_REMOVE: 2738 /* 2739 * A hot-plug removal event has occured and all 2740 * outstanding commands have already been revoked. 2741 * 2742 * Don't recurse if this occurs while we are 2743 * resetting the port. 2744 */ 2745 if ((ap->ap_flags & AP_F_IN_RESET) == 0) { 2746 kprintf("%s: HOTPLUG - Device removed\n", 2747 PORTNAME(ap)); 2748 ahci_port_hardstop(ap); 2749 /* ap_probe set to failed */ 2750 ahci_cam_changed(ap, NULL, -1); 2751 } 2752 break; 2753 default: 2754 break; 2755 } 2756 } 2757 2758 struct ahci_ccb * 2759 ahci_get_ccb(struct ahci_port *ap) 2760 { 2761 struct ahci_ccb *ccb; 2762 2763 lockmgr(&ap->ap_ccb_lock, LK_EXCLUSIVE); 2764 ccb = TAILQ_FIRST(&ap->ap_ccb_free); 2765 if (ccb != NULL) { 2766 KKASSERT(ccb->ccb_xa.state == ATA_S_PUT); 2767 TAILQ_REMOVE(&ap->ap_ccb_free, ccb, ccb_entry); 2768 ccb->ccb_xa.state = ATA_S_SETUP; 2769 ccb->ccb_xa.at = NULL; 2770 } 2771 lockmgr(&ap->ap_ccb_lock, LK_RELEASE); 2772 2773 return (ccb); 2774 } 2775 2776 void 2777 ahci_put_ccb(struct ahci_ccb *ccb) 2778 { 2779 struct ahci_port *ap = ccb->ccb_port; 2780 2781 ccb->ccb_xa.state = ATA_S_PUT; 2782 lockmgr(&ap->ap_ccb_lock, LK_EXCLUSIVE); 2783 TAILQ_INSERT_TAIL(&ap->ap_ccb_free, ccb, ccb_entry); 2784 lockmgr(&ap->ap_ccb_lock, LK_RELEASE); 2785 } 2786 2787 struct ahci_ccb * 2788 ahci_get_err_ccb(struct ahci_port *ap) 2789 { 2790 struct ahci_ccb *err_ccb; 2791 u_int32_t sact; 2792 u_int32_t ci; 2793 2794 /* No commands may be active on the chip. */ 2795 2796 if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SNCQ) { 2797 sact = ahci_pread(ap, AHCI_PREG_SACT); 2798 if (sact != 0) { 2799 kprintf("%s: ahci_get_err_ccb but SACT %08x != 0?\n", 2800 PORTNAME(ap), sact); 2801 } 2802 } 2803 ci = ahci_pread(ap, AHCI_PREG_CI); 2804 if (ci) { 2805 kprintf("%s: ahci_get_err_ccb: ci not 0 (%08x)\n", 2806 ap->ap_name, ci); 2807 } 2808 KKASSERT(ci == 0); 2809 KKASSERT((ap->ap_flags & AP_F_ERR_CCB_RESERVED) == 0); 2810 ap->ap_flags |= AP_F_ERR_CCB_RESERVED; 2811 2812 /* Save outstanding command state. */ 2813 ap->ap_err_saved_active = ap->ap_active; 2814 ap->ap_err_saved_active_cnt = ap->ap_active_cnt; 2815 ap->ap_err_saved_sactive = ap->ap_sactive; 2816 2817 /* 2818 * Pretend we have no commands outstanding, so that completions won't 2819 * run prematurely. 2820 */ 2821 ap->ap_active = ap->ap_active_cnt = ap->ap_sactive = 0; 2822 2823 /* 2824 * Grab a CCB to use for error recovery. This should never fail, as 2825 * we ask atascsi to reserve one for us at init time. 2826 */ 2827 err_ccb = ap->ap_err_ccb; 2828 KKASSERT(err_ccb != NULL); 2829 err_ccb->ccb_xa.flags = 0; 2830 err_ccb->ccb_done = ahci_empty_done; 2831 2832 return err_ccb; 2833 } 2834 2835 void 2836 ahci_put_err_ccb(struct ahci_ccb *ccb) 2837 { 2838 struct ahci_port *ap = ccb->ccb_port; 2839 u_int32_t sact; 2840 u_int32_t ci; 2841 2842 KKASSERT((ap->ap_flags & AP_F_ERR_CCB_RESERVED) != 0); 2843 2844 /* 2845 * No commands may be active on the chip 2846 */ 2847 if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SNCQ) { 2848 sact = ahci_pread(ap, AHCI_PREG_SACT); 2849 if (sact) { 2850 panic("ahci_port_err_ccb(%d) but SACT %08x != 0\n", 2851 ccb->ccb_slot, sact); 2852 } 2853 } 2854 ci = ahci_pread(ap, AHCI_PREG_CI); 2855 if (ci) { 2856 panic("ahci_put_err_ccb(%d) but CI %08x != 0 " 2857 "(act=%08x sact=%08x)\n", 2858 ccb->ccb_slot, ci, 2859 ap->ap_active, ap->ap_sactive); 2860 } 2861 2862 KKASSERT(ccb == ap->ap_err_ccb); 2863 2864 /* Restore outstanding command state */ 2865 ap->ap_sactive = ap->ap_err_saved_sactive; 2866 ap->ap_active_cnt = ap->ap_err_saved_active_cnt; 2867 ap->ap_active = ap->ap_err_saved_active; 2868 2869 ap->ap_flags &= ~AP_F_ERR_CCB_RESERVED; 2870 } 2871 2872 /* 2873 * Read log page to get NCQ error. 2874 * 2875 * NOTE: NCQ not currently supported on port multipliers. XXX 2876 */ 2877 int 2878 ahci_port_read_ncq_error(struct ahci_port *ap, int target) 2879 { 2880 struct ata_log_page_10h *log; 2881 struct ahci_ccb *ccb; 2882 struct ahci_cmd_hdr *cmd_slot; 2883 struct ata_fis_h2d *fis; 2884 int err_slot; 2885 2886 if (bootverbose) { 2887 kprintf("%s: READ LOG PAGE target %d\n", PORTNAME(ap), 2888 target); 2889 } 2890 2891 /* 2892 * Prep error CCB for READ LOG EXT, page 10h, 1 sector. 2893 * 2894 * Getting err_ccb clears active/sactive/active_cnt, putting 2895 * it back restores the fields. 2896 */ 2897 ccb = ahci_get_err_ccb(ap); 2898 ccb->ccb_xa.flags = ATA_F_READ | ATA_F_POLL; 2899 ccb->ccb_xa.data = ap->ap_err_scratch; 2900 ccb->ccb_xa.datalen = 512; 2901 ccb->ccb_xa.complete = ahci_dummy_done; 2902 ccb->ccb_xa.at = ap->ap_ata[target]; 2903 2904 fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis; 2905 bzero(fis, sizeof(*fis)); 2906 fis->type = ATA_FIS_TYPE_H2D; 2907 fis->flags = ATA_H2D_FLAGS_CMD | target; 2908 fis->command = ATA_C_READ_LOG_EXT; 2909 fis->lba_low = 0x10; /* queued error log page (10h) */ 2910 fis->sector_count = 1; /* number of sectors (1) */ 2911 fis->sector_count_exp = 0; 2912 fis->lba_mid = 0; /* starting offset */ 2913 fis->lba_mid_exp = 0; 2914 fis->device = 0; 2915 2916 cmd_slot = ccb->ccb_cmd_hdr; 2917 cmd_slot->flags = htole16(5); /* FIS length: 5 DWORDS */ 2918 2919 if (ahci_load_prdt(ccb) != 0) { 2920 err_slot = -1; 2921 goto err; 2922 } 2923 2924 ccb->ccb_xa.state = ATA_S_PENDING; 2925 if (ahci_poll(ccb, 1000, ahci_quick_timeout) != ATA_S_COMPLETE) { 2926 err_slot = -1; 2927 ahci_unload_prdt(ccb); 2928 goto err; 2929 } 2930 ahci_unload_prdt(ccb); 2931 2932 /* 2933 * Success, extract failed register set and tags from the scratch 2934 * space. 2935 */ 2936 log = (struct ata_log_page_10h *)ap->ap_err_scratch; 2937 if (log->err_regs.type & ATA_LOG_10H_TYPE_NOTQUEUED) { 2938 /* Not queued bit was set - wasn't an NCQ error? */ 2939 kprintf("%s: read NCQ error page, but not an NCQ error?\n", 2940 PORTNAME(ap)); 2941 err_slot = -1; 2942 } else { 2943 /* Copy back the log record as a D2H register FIS. */ 2944 err_slot = log->err_regs.type & ATA_LOG_10H_TYPE_TAG_MASK; 2945 2946 ccb = &ap->ap_ccbs[err_slot]; 2947 if (ccb->ccb_xa.state == ATA_S_ONCHIP) { 2948 kprintf("%s: read NCQ error page slot=%d\n", 2949 ATANAME(ap, ccb->ccb_xa.at), 2950 err_slot); 2951 memcpy(&ccb->ccb_xa.rfis, &log->err_regs, 2952 sizeof(struct ata_fis_d2h)); 2953 ccb->ccb_xa.rfis.type = ATA_FIS_TYPE_D2H; 2954 ccb->ccb_xa.rfis.flags = 0; 2955 } else { 2956 kprintf("%s: read NCQ error page slot=%d, " 2957 "slot does not match any cmds\n", 2958 ATANAME(ccb->ccb_port, ccb->ccb_xa.at), 2959 err_slot); 2960 err_slot = -1; 2961 } 2962 } 2963 err: 2964 ahci_put_err_ccb(ccb); 2965 kprintf("%s: DONE log page target %d err_slot=%d\n", 2966 PORTNAME(ap), target, err_slot); 2967 return (err_slot); 2968 } 2969 2970 /* 2971 * Allocate memory for various structures DMAd by hardware. The maximum 2972 * number of segments for these tags is 1 so the DMA memory will have a 2973 * single physical base address. 2974 */ 2975 struct ahci_dmamem * 2976 ahci_dmamem_alloc(struct ahci_softc *sc, bus_dma_tag_t tag) 2977 { 2978 struct ahci_dmamem *adm; 2979 int error; 2980 2981 adm = kmalloc(sizeof(*adm), M_DEVBUF, M_INTWAIT | M_ZERO); 2982 2983 error = bus_dmamem_alloc(tag, (void **)&adm->adm_kva, 2984 BUS_DMA_ZERO, &adm->adm_map); 2985 if (error == 0) { 2986 adm->adm_tag = tag; 2987 error = bus_dmamap_load(tag, adm->adm_map, 2988 adm->adm_kva, 2989 bus_dma_tag_getmaxsize(tag), 2990 ahci_dmamem_saveseg, &adm->adm_busaddr, 2991 0); 2992 } 2993 if (error) { 2994 if (adm->adm_map) { 2995 bus_dmamap_destroy(tag, adm->adm_map); 2996 adm->adm_map = NULL; 2997 adm->adm_tag = NULL; 2998 adm->adm_kva = NULL; 2999 } 3000 kfree(adm, M_DEVBUF); 3001 adm = NULL; 3002 } 3003 return (adm); 3004 } 3005 3006 static 3007 void 3008 ahci_dmamem_saveseg(void *info, bus_dma_segment_t *segs, int nsegs, int error) 3009 { 3010 KKASSERT(error == 0); 3011 KKASSERT(nsegs == 1); 3012 *(bus_addr_t *)info = segs->ds_addr; 3013 } 3014 3015 3016 void 3017 ahci_dmamem_free(struct ahci_softc *sc, struct ahci_dmamem *adm) 3018 { 3019 if (adm->adm_map) { 3020 bus_dmamap_unload(adm->adm_tag, adm->adm_map); 3021 bus_dmamap_destroy(adm->adm_tag, adm->adm_map); 3022 adm->adm_map = NULL; 3023 adm->adm_tag = NULL; 3024 adm->adm_kva = NULL; 3025 } 3026 kfree(adm, M_DEVBUF); 3027 } 3028 3029 u_int32_t 3030 ahci_read(struct ahci_softc *sc, bus_size_t r) 3031 { 3032 bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4, 3033 BUS_SPACE_BARRIER_READ); 3034 return (bus_space_read_4(sc->sc_iot, sc->sc_ioh, r)); 3035 } 3036 3037 void 3038 ahci_write(struct ahci_softc *sc, bus_size_t r, u_int32_t v) 3039 { 3040 bus_space_write_4(sc->sc_iot, sc->sc_ioh, r, v); 3041 bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4, 3042 BUS_SPACE_BARRIER_WRITE); 3043 } 3044 3045 u_int32_t 3046 ahci_pread(struct ahci_port *ap, bus_size_t r) 3047 { 3048 bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4, 3049 BUS_SPACE_BARRIER_READ); 3050 return (bus_space_read_4(ap->ap_sc->sc_iot, ap->ap_ioh, r)); 3051 } 3052 3053 void 3054 ahci_pwrite(struct ahci_port *ap, bus_size_t r, u_int32_t v) 3055 { 3056 bus_space_write_4(ap->ap_sc->sc_iot, ap->ap_ioh, r, v); 3057 bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4, 3058 BUS_SPACE_BARRIER_WRITE); 3059 } 3060 3061 /* 3062 * Wait up to (timeout) milliseconds for the masked port register to 3063 * match the target. 3064 * 3065 * Timeout is in milliseconds. 3066 */ 3067 int 3068 ahci_pwait_eq(struct ahci_port *ap, int timeout, 3069 bus_size_t r, u_int32_t mask, u_int32_t target) 3070 { 3071 int t; 3072 3073 /* 3074 * Loop hard up to 100uS 3075 */ 3076 for (t = 0; t < 100; ++t) { 3077 if ((ahci_pread(ap, r) & mask) == target) 3078 return (0); 3079 ahci_os_hardsleep(1); /* us */ 3080 } 3081 3082 do { 3083 timeout -= ahci_os_softsleep(); 3084 if ((ahci_pread(ap, r) & mask) == target) 3085 return (0); 3086 } while (timeout > 0); 3087 return (1); 3088 } 3089 3090 int 3091 ahci_wait_ne(struct ahci_softc *sc, bus_size_t r, u_int32_t mask, 3092 u_int32_t target) 3093 { 3094 int t; 3095 3096 /* 3097 * Loop hard up to 100uS 3098 */ 3099 for (t = 0; t < 100; ++t) { 3100 if ((ahci_read(sc, r) & mask) != target) 3101 return (0); 3102 ahci_os_hardsleep(1); /* us */ 3103 } 3104 3105 /* 3106 * And one millisecond the slow way 3107 */ 3108 t = 1000; 3109 do { 3110 t -= ahci_os_softsleep(); 3111 if ((ahci_read(sc, r) & mask) != target) 3112 return (0); 3113 } while (t > 0); 3114 3115 return (1); 3116 } 3117 3118 3119 /* 3120 * Acquire an ata transfer. 3121 * 3122 * Pass a NULL at for direct-attached transfers, and a non-NULL at for 3123 * targets that go through the port multiplier. 3124 */ 3125 struct ata_xfer * 3126 ahci_ata_get_xfer(struct ahci_port *ap, struct ata_port *at) 3127 { 3128 struct ahci_ccb *ccb; 3129 3130 ccb = ahci_get_ccb(ap); 3131 if (ccb == NULL) { 3132 DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer: NULL ccb\n", 3133 PORTNAME(ap)); 3134 return (NULL); 3135 } 3136 3137 DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer got slot %d\n", 3138 PORTNAME(ap), ccb->ccb_slot); 3139 3140 bzero(ccb->ccb_xa.fis, sizeof(*ccb->ccb_xa.fis)); 3141 ccb->ccb_xa.at = at; 3142 ccb->ccb_xa.fis->type = ATA_FIS_TYPE_H2D; 3143 3144 return (&ccb->ccb_xa); 3145 } 3146 3147 void 3148 ahci_ata_put_xfer(struct ata_xfer *xa) 3149 { 3150 struct ahci_ccb *ccb = (struct ahci_ccb *)xa; 3151 3152 DPRINTF(AHCI_D_XFER, "ahci_ata_put_xfer slot %d\n", ccb->ccb_slot); 3153 3154 ahci_put_ccb(ccb); 3155 } 3156 3157 int 3158 ahci_ata_cmd(struct ata_xfer *xa) 3159 { 3160 struct ahci_ccb *ccb = (struct ahci_ccb *)xa; 3161 struct ahci_cmd_hdr *cmd_slot; 3162 3163 KKASSERT(xa->state == ATA_S_SETUP); 3164 3165 if (ccb->ccb_port->ap_state == AP_S_FATAL_ERROR) 3166 goto failcmd; 3167 ccb->ccb_done = ahci_ata_cmd_done; 3168 3169 cmd_slot = ccb->ccb_cmd_hdr; 3170 cmd_slot->flags = htole16(5); /* FIS length (in DWORDs) */ 3171 if (ccb->ccb_xa.at) { 3172 cmd_slot->flags |= htole16(ccb->ccb_xa.at->at_target << 3173 AHCI_CMD_LIST_FLAG_PMP_SHIFT); 3174 } 3175 3176 if (xa->flags & ATA_F_WRITE) 3177 cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_W); 3178 3179 if (xa->flags & ATA_F_PACKET) 3180 cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_A); 3181 3182 if (ahci_load_prdt(ccb) != 0) 3183 goto failcmd; 3184 3185 xa->state = ATA_S_PENDING; 3186 3187 if (xa->flags & ATA_F_POLL) 3188 return (ahci_poll(ccb, xa->timeout, ahci_ata_cmd_timeout)); 3189 3190 crit_enter(); 3191 KKASSERT((xa->flags & ATA_F_TIMEOUT_EXPIRED) == 0); 3192 xa->flags |= ATA_F_TIMEOUT_DESIRED; 3193 ahci_start(ccb); 3194 crit_exit(); 3195 return (xa->state); 3196 3197 failcmd: 3198 crit_enter(); 3199 xa->state = ATA_S_ERROR; 3200 xa->complete(xa); 3201 crit_exit(); 3202 return (ATA_S_ERROR); 3203 } 3204 3205 void 3206 ahci_ata_cmd_done(struct ahci_ccb *ccb) 3207 { 3208 struct ata_xfer *xa = &ccb->ccb_xa; 3209 3210 /* 3211 * NOTE: callout does not lock port and may race us modifying 3212 * the flags, so make sure its stopped. 3213 */ 3214 if (xa->flags & ATA_F_TIMEOUT_RUNNING) { 3215 callout_stop(&ccb->ccb_timeout); 3216 xa->flags &= ~ATA_F_TIMEOUT_RUNNING; 3217 } 3218 xa->flags &= ~(ATA_F_TIMEOUT_DESIRED | ATA_F_TIMEOUT_EXPIRED); 3219 3220 KKASSERT(xa->state != ATA_S_ONCHIP); 3221 ahci_unload_prdt(ccb); 3222 3223 if (xa->state != ATA_S_TIMEOUT) 3224 xa->complete(xa); 3225 } 3226 3227 /* 3228 * Timeout from callout, MPSAFE - nothing can mess with the CCB's flags 3229 * while the callout is runing. 3230 * 3231 * We can't safely get the port lock here or delay, we could block 3232 * the callout thread. 3233 */ 3234 static void 3235 ahci_ata_cmd_timeout_unserialized(void *arg) 3236 { 3237 struct ahci_ccb *ccb = arg; 3238 struct ahci_port *ap = ccb->ccb_port; 3239 3240 ccb->ccb_xa.flags &= ~ATA_F_TIMEOUT_RUNNING; 3241 ccb->ccb_xa.flags |= ATA_F_TIMEOUT_EXPIRED; 3242 ahci_os_signal_port_thread(ap, AP_SIGF_TIMEOUT); 3243 } 3244 3245 /* 3246 * Timeout code, typically called when the port command processor is running. 3247 * 3248 * We have to be very very careful here. We cannot stop the port unless 3249 * CR is already clear or the only active commands remaining are timed-out 3250 * ones. Otherwise stopping the port will race the command processor and 3251 * we can lose events. While we can theoretically just restart everything 3252 * that could result in a double-issue which will not work for ATAPI commands. 3253 */ 3254 void 3255 ahci_ata_cmd_timeout(struct ahci_ccb *ccb) 3256 { 3257 struct ata_xfer *xa = &ccb->ccb_xa; 3258 struct ahci_port *ap = ccb->ccb_port; 3259 struct ata_port *at; 3260 int ci_saved; 3261 int slot; 3262 3263 at = ccb->ccb_xa.at; 3264 3265 kprintf("%s: CMD TIMEOUT state=%d slot=%d\n" 3266 "\tcmd-reg 0x%b\n" 3267 "\tsactive=%08x active=%08x expired=%08x\n" 3268 "\t sact=%08x ci=%08x\n" 3269 "\t STS=%b\n", 3270 ATANAME(ap, at), 3271 ccb->ccb_xa.state, ccb->ccb_slot, 3272 ahci_pread(ap, AHCI_PREG_CMD), AHCI_PFMT_CMD, 3273 ap->ap_sactive, ap->ap_active, ap->ap_expired, 3274 ahci_pread(ap, AHCI_PREG_SACT), 3275 ahci_pread(ap, AHCI_PREG_CI), 3276 ahci_pread(ap, AHCI_PREG_TFD), AHCI_PFMT_TFD_STS 3277 ); 3278 3279 3280 /* 3281 * NOTE: Timeout will not be running if the command was polled. 3282 * If we got here at least one of these flags should be set. 3283 */ 3284 KKASSERT(xa->flags & (ATA_F_POLL | ATA_F_TIMEOUT_DESIRED | 3285 ATA_F_TIMEOUT_RUNNING)); 3286 xa->flags &= ~(ATA_F_TIMEOUT_RUNNING | ATA_F_TIMEOUT_EXPIRED); 3287 3288 if (ccb->ccb_xa.state == ATA_S_PENDING) { 3289 TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry); 3290 ccb->ccb_xa.state = ATA_S_TIMEOUT; 3291 ccb->ccb_done(ccb); 3292 xa->complete(xa); 3293 ahci_issue_pending_commands(ap, NULL); 3294 return; 3295 } 3296 if (ccb->ccb_xa.state != ATA_S_ONCHIP) { 3297 kprintf("%s: Unexpected state during timeout: %d\n", 3298 ATANAME(ap, at), ccb->ccb_xa.state); 3299 return; 3300 } 3301 3302 /* 3303 * Ok, we can only get this command off the chip if CR is inactive 3304 * or if the only commands running on the chip are all expired. 3305 * Otherwise we have to wait until the port is in a safe state. 3306 * 3307 * Do not set state here, it will cause polls to return when the 3308 * ccb is not yet off the chip. 3309 */ 3310 ap->ap_expired |= 1 << ccb->ccb_slot; 3311 3312 if ((ahci_pread(ap, AHCI_PREG_CMD) & AHCI_PREG_CMD_CR) && 3313 (ap->ap_active | ap->ap_sactive) != ap->ap_expired) { 3314 /* 3315 * If using FBSS or NCQ we can't safely stop the port 3316 * right now. 3317 */ 3318 kprintf("%s: Deferred timeout until its safe, slot %d\n", 3319 ATANAME(ap, at), ccb->ccb_slot); 3320 return; 3321 } 3322 3323 /* 3324 * We can safely stop the port and process all expired ccb's, 3325 * which will include our current ccb. 3326 */ 3327 ci_saved = (ap->ap_sactive) ? ahci_pread(ap, AHCI_PREG_SACT) : 3328 ahci_pread(ap, AHCI_PREG_CI); 3329 ahci_port_stop(ap, 0); 3330 3331 while (ap->ap_expired) { 3332 slot = ffs(ap->ap_expired) - 1; 3333 ap->ap_expired &= ~(1 << slot); 3334 ci_saved &= ~(1 << slot); 3335 ccb = &ap->ap_ccbs[slot]; 3336 ccb->ccb_xa.state = ATA_S_TIMEOUT; 3337 if (ccb->ccb_xa.flags & ATA_F_NCQ) { 3338 KKASSERT(ap->ap_sactive & (1 << slot)); 3339 ap->ap_sactive &= ~(1 << slot); 3340 } else { 3341 KKASSERT(ap->ap_active & (1 << slot)); 3342 ap->ap_active &= ~(1 << slot); 3343 --ap->ap_active_cnt; 3344 } 3345 ccb->ccb_done(ccb); 3346 ccb->ccb_xa.complete(&ccb->ccb_xa); 3347 } 3348 /* ccb invalid now */ 3349 3350 /* 3351 * We can safely CLO the port to clear any BSY/DRQ, a case which 3352 * can occur with port multipliers. This will unbrick the port 3353 * and allow commands to other targets behind the PM continue. 3354 * (FBSS). 3355 * 3356 * Finally, once the port has been restarted we can issue any 3357 * previously saved pending commands, and run the port interrupt 3358 * code to handle any completions which may have occured when 3359 * we saved CI. 3360 */ 3361 if (ahci_pread(ap, AHCI_PREG_TFD) & 3362 (AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) { 3363 kprintf("%s: Warning, issuing CLO after timeout\n", 3364 ATANAME(ap, at)); 3365 ahci_port_clo(ap); 3366 } 3367 ahci_port_start(ap); 3368 ahci_issue_saved_commands(ap, ci_saved & ~ap->ap_expired); 3369 ahci_issue_pending_commands(ap, NULL); 3370 ahci_port_intr(ap, 0); 3371 } 3372 3373 /* 3374 * Issue a previously saved set of commands 3375 */ 3376 void 3377 ahci_issue_saved_commands(struct ahci_port *ap, u_int32_t ci_saved) 3378 { 3379 if (ci_saved) { 3380 KKASSERT(!((ap->ap_active & ci_saved) && 3381 (ap->ap_sactive & ci_saved))); 3382 KKASSERT((ci_saved & ap->ap_expired) == 0); 3383 if (ap->ap_sactive & ci_saved) 3384 ahci_pwrite(ap, AHCI_PREG_SACT, ci_saved); 3385 ahci_pwrite(ap, AHCI_PREG_CI, ci_saved); 3386 } 3387 } 3388 3389 /* 3390 * Used by the softreset, pmprobe, and read_ncq_error only, in very 3391 * specialized, controlled circumstances. 3392 * 3393 * Only one command may be pending. 3394 */ 3395 void 3396 ahci_quick_timeout(struct ahci_ccb *ccb) 3397 { 3398 struct ahci_port *ap = ccb->ccb_port; 3399 3400 switch (ccb->ccb_xa.state) { 3401 case ATA_S_PENDING: 3402 TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry); 3403 ccb->ccb_xa.state = ATA_S_TIMEOUT; 3404 break; 3405 case ATA_S_ONCHIP: 3406 KKASSERT(ap->ap_active == (1 << ccb->ccb_slot) && 3407 ap->ap_sactive == 0); 3408 ahci_port_stop(ap, 0); 3409 ahci_port_start(ap); 3410 3411 ccb->ccb_xa.state = ATA_S_TIMEOUT; 3412 ap->ap_active &= ~(1 << ccb->ccb_slot); 3413 KKASSERT(ap->ap_active_cnt > 0); 3414 --ap->ap_active_cnt; 3415 break; 3416 default: 3417 panic("%s: ahci_quick_timeout: ccb in bad state %d", 3418 ATANAME(ap, ccb->ccb_xa.at), ccb->ccb_xa.state); 3419 } 3420 } 3421 3422 static void 3423 ahci_dummy_done(struct ata_xfer *xa) 3424 { 3425 } 3426 3427 static void 3428 ahci_empty_done(struct ahci_ccb *ccb) 3429 { 3430 } 3431