xref: /dragonfly/sys/dev/disk/ahci/ahci_cam.c (revision 0dace59e)
1 /*
2  * (MPSAFE)
3  *
4  * Copyright (c) 2009 The DragonFly Project.  All rights reserved.
5  *
6  * This code is derived from software contributed to The DragonFly Project
7  * by Matthew Dillon <dillon@backplane.com>
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  *
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in
17  *    the documentation and/or other materials provided with the
18  *    distribution.
19  * 3. Neither the name of The DragonFly Project nor the names of its
20  *    contributors may be used to endorse or promote products derived
21  *    from this software without specific, prior written permission.
22  *
23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
24  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
25  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
26  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
27  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
28  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
29  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
30  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
31  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
32  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
33  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34  * SUCH DAMAGE.
35  *
36  *
37  * Copyright (c) 2007 David Gwynne <dlg@openbsd.org>
38  *
39  * Permission to use, copy, modify, and distribute this software for any
40  * purpose with or without fee is hereby granted, provided that the above
41  * copyright notice and this permission notice appear in all copies.
42  *
43  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
44  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
45  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
46  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
47  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
48  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
49  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
50  *
51  * $OpenBSD: atascsi.c,v 1.64 2009/02/16 21:19:06 miod Exp $
52  */
53 /*
54  * Implement each SATA port as its own SCSI bus on CAM.  This way we can
55  * implement future port multiplier features as individual devices on the
56  * bus.
57  *
58  * Much of the cdb<->xa conversion code was taken from OpenBSD, the rest
59  * was written natively for DragonFly.
60  *
61  * NOTE-1: I was temporarily unlocking the port while making the CCB
62  *	   callback, to reduce the chance of a deadlock and to improve
63  *	   performance by allowing new commands to be queued.
64  *
65  *	   However, this also creates an opening where another AHCI
66  *	   interrupt can come in and execute the ahci_port_intr()
67  *	   function, creating a huge mess in the sequencing of the
68  *	   chipset.
69  *
70  *	   So for now we don't do this. XXX
71  */
72 
73 #include "ahci.h"
74 
75 static void ahci_xpt_action(struct cam_sim *sim, union ccb *ccb);
76 static void ahci_xpt_poll(struct cam_sim *sim);
77 static void ahci_xpt_scsi_disk_io(struct ahci_port *ap,
78 			struct ata_port *at, union ccb *ccb);
79 static void ahci_xpt_scsi_atapi_io(struct ahci_port *ap,
80 			struct ata_port *at, union ccb *ccb);
81 static void ahci_xpt_page_inquiry(struct ahci_port *ap,
82 			struct ata_port *at, union ccb *ccb);
83 
84 static void ahci_ata_complete_disk_rw(struct ata_xfer *xa);
85 static void ahci_ata_complete_disk_synchronize_cache(struct ata_xfer *xa);
86 static void ahci_atapi_complete_cmd(struct ata_xfer *xa);
87 static void ahci_ata_dummy_sense(struct scsi_sense_data *sense_data);
88 static void ahci_ata_atapi_sense(struct ata_fis_d2h *rfis,
89 		     struct scsi_sense_data *sense_data);
90 
91 static int ahci_cam_probe_disk(struct ahci_port *ap, struct ata_port *at);
92 static int ahci_cam_probe_atapi(struct ahci_port *ap, struct ata_port *at);
93 static int ahci_set_xfer(struct ahci_port *ap, struct ata_port *atx);
94 static void ahci_ata_dummy_done(struct ata_xfer *xa);
95 static void ata_fix_identify(struct ata_identify *id);
96 static void ahci_cam_rescan(struct ahci_port *ap);
97 static void ahci_strip_string(const char **basep, int *lenp);
98 
99 int
100 ahci_cam_attach(struct ahci_port *ap)
101 {
102 	struct cam_devq *devq;
103 	struct cam_sim *sim;
104 	int error;
105 	int unit;
106 
107 	/*
108 	 * We want at least one ccb to be available for error processing
109 	 * so don't let CAM use more then ncmds - 1.
110 	 */
111 	unit = device_get_unit(ap->ap_sc->sc_dev);
112 	if (ap->ap_sc->sc_ncmds > 1)
113 		devq = cam_simq_alloc(ap->ap_sc->sc_ncmds - 1);
114 	else
115 		devq = cam_simq_alloc(ap->ap_sc->sc_ncmds);
116 	if (devq == NULL) {
117 		return (ENOMEM);
118 	}
119 
120 	/*
121 	 * Give the devq enough room to run with 32 max_dev_transactions,
122 	 * but set the overall max tags to 1 until NCQ is negotiated.
123 	 */
124 	sim = cam_sim_alloc(ahci_xpt_action, ahci_xpt_poll, "ahci",
125 			   (void *)ap, unit, &ap->ap_sim_lock,
126 			   32, 1, devq);
127 	cam_simq_release(devq);
128 	if (sim == NULL) {
129 		return (ENOMEM);
130 	}
131 	ap->ap_sim = sim;
132 	ahci_os_unlock_port(ap);
133 	lockmgr(&ap->ap_sim_lock, LK_EXCLUSIVE);
134 	error = xpt_bus_register(ap->ap_sim, ap->ap_num);
135 	lockmgr(&ap->ap_sim_lock, LK_RELEASE);
136 	ahci_os_lock_port(ap);
137 	if (error != CAM_SUCCESS) {
138 		ahci_cam_detach(ap);
139 		return (EINVAL);
140 	}
141 	ap->ap_flags |= AP_F_BUS_REGISTERED;
142 
143 	if (ap->ap_probe == ATA_PROBE_NEED_IDENT)
144 		error = ahci_cam_probe(ap, NULL);
145 	else
146 		error = 0;
147 	if (error) {
148 		ahci_cam_detach(ap);
149 		return (EIO);
150 	}
151 	ap->ap_flags |= AP_F_CAM_ATTACHED;
152 
153 	return(0);
154 }
155 
156 /*
157  * The state of the port has changed.
158  *
159  * If atx is NULL the physical port has changed state.
160  * If atx is non-NULL a particular target behind a PM has changed state.
161  *
162  * If found is -1 the target state must be queued to a non-interrupt context.
163  * (only works with at == NULL).
164  *
165  * If found is 0 the target was removed.
166  * If found is 1 the target was inserted.
167  */
168 void
169 ahci_cam_changed(struct ahci_port *ap, struct ata_port *atx, int found)
170 {
171 	struct cam_path *tmppath;
172 	int status;
173 	int target;
174 
175 	target = atx ? atx->at_target : CAM_TARGET_WILDCARD;
176 
177 	if (ap->ap_sim == NULL)
178 		return;
179 	if (found == CAM_TARGET_WILDCARD) {
180 		status = xpt_create_path(&tmppath, NULL,
181 					 cam_sim_path(ap->ap_sim),
182 					 target, CAM_LUN_WILDCARD);
183 		if (status != CAM_REQ_CMP)
184 			return;
185 		ahci_cam_rescan(ap);
186 	} else {
187 		status = xpt_create_path(&tmppath, NULL,
188 					 cam_sim_path(ap->ap_sim),
189 					 target,
190 					 CAM_LUN_WILDCARD);
191 		if (status != CAM_REQ_CMP)
192 			return;
193 #if 0
194 		/*
195 		 * This confuses CAM
196 		 */
197 		if (found)
198 			xpt_async(AC_FOUND_DEVICE, tmppath, NULL);
199 		else
200 			xpt_async(AC_LOST_DEVICE, tmppath, NULL);
201 #endif
202 	}
203 	xpt_free_path(tmppath);
204 }
205 
206 void
207 ahci_cam_detach(struct ahci_port *ap)
208 {
209 	int error __debugvar;
210 
211 	if ((ap->ap_flags & AP_F_CAM_ATTACHED) == 0)
212 		return;
213 	lockmgr(&ap->ap_sim_lock, LK_EXCLUSIVE);
214 	if (ap->ap_sim) {
215 		xpt_freeze_simq(ap->ap_sim, 1);
216 	}
217 	if (ap->ap_flags & AP_F_BUS_REGISTERED) {
218 		error = xpt_bus_deregister(cam_sim_path(ap->ap_sim));
219 		KKASSERT(error == CAM_REQ_CMP);
220 		ap->ap_flags &= ~AP_F_BUS_REGISTERED;
221 	}
222 	if (ap->ap_sim) {
223 		cam_sim_free(ap->ap_sim);
224 		ap->ap_sim = NULL;
225 	}
226 	lockmgr(&ap->ap_sim_lock, LK_RELEASE);
227 	ap->ap_flags &= ~AP_F_CAM_ATTACHED;
228 }
229 
230 /*
231  * Once the AHCI port has been attached we need to probe for a device or
232  * devices on the port and setup various options.
233  *
234  * If at is NULL we are probing the direct-attached device on the port,
235  * which may or may not be a port multiplier.
236  */
237 int
238 ahci_cam_probe(struct ahci_port *ap, struct ata_port *atx)
239 {
240 	struct ata_port	*at;
241 	struct ata_xfer	*xa;
242 	u_int64_t	capacity;
243 	u_int64_t	capacity_bytes;
244 	int		model_len;
245 	int		firmware_len;
246 	int		serial_len;
247 	int		error;
248 	int		devncqdepth;
249 	int		i;
250 	const char	*model_id;
251 	const char	*firmware_id;
252 	const char	*serial_id;
253 	const char	*wcstr;
254 	const char	*rastr;
255 	const char	*scstr;
256 	const char	*type;
257 
258 	error = EIO;
259 
260 	/*
261 	 * Delayed CAM attachment for initial probe, sim may be NULL
262 	 */
263 	if (ap->ap_sim == NULL)
264 		return(0);
265 
266 	/*
267 	 * A NULL atx indicates a probe of the directly connected device.
268 	 * A non-NULL atx indicates a device connected via a port multiplier.
269 	 * We need to preserve atx for calls to ahci_ata_get_xfer().
270 	 *
271 	 * at is always non-NULL.  For directly connected devices we supply
272 	 * an (at) pointing to target 0.
273 	 */
274 	if (atx == NULL) {
275 		at = ap->ap_ata[0];	/* direct attached - device 0 */
276 		if (ap->ap_type == ATA_PORT_T_PM) {
277 			kprintf("%s: Found Port Multiplier\n",
278 				ATANAME(ap, atx));
279 			return (0);
280 		}
281 		at->at_type = ap->ap_type;
282 	} else {
283 		at = atx;
284 		if (atx->at_type == ATA_PORT_T_PM) {
285 			kprintf("%s: Bogus device, reducing port count to %d\n",
286 				ATANAME(ap, atx), atx->at_target);
287 			if (ap->ap_pmcount > atx->at_target)
288 				ap->ap_pmcount = atx->at_target;
289 			goto err;
290 		}
291 	}
292 	if (ap->ap_type == ATA_PORT_T_NONE)
293 		goto err;
294 	if (at->at_type == ATA_PORT_T_NONE)
295 		goto err;
296 
297 	/*
298 	 * Issue identify, saving the result
299 	 */
300 	xa = ahci_ata_get_xfer(ap, atx);
301 	xa->complete = ahci_ata_dummy_done;
302 	xa->data = &at->at_identify;
303 	xa->datalen = sizeof(at->at_identify);
304 	xa->flags = ATA_F_READ | ATA_F_PIO | ATA_F_POLL;
305 	xa->fis->flags = ATA_H2D_FLAGS_CMD | at->at_target;
306 
307 	switch(at->at_type) {
308 	case ATA_PORT_T_DISK:
309 		xa->fis->command = ATA_C_IDENTIFY;
310 		type = "DISK";
311 		break;
312 	case ATA_PORT_T_ATAPI:
313 		xa->fis->command = ATA_C_ATAPI_IDENTIFY;
314 		xa->flags |= ATA_F_AUTOSENSE;
315 		type = "ATAPI";
316 		break;
317 	default:
318 		xa->fis->command = ATA_C_ATAPI_IDENTIFY;
319 		type = "UNKNOWN(ATAPI?)";
320 		break;
321 	}
322 	xa->fis->features = 0;
323 	xa->fis->device = 0;
324 	xa->timeout = 1000;
325 
326 	if (ahci_ata_cmd(xa) != ATA_S_COMPLETE) {
327 		kprintf("%s: Detected %s device but unable to IDENTIFY\n",
328 			ATANAME(ap, atx), type);
329 		ahci_ata_put_xfer(xa);
330 		goto err;
331 	}
332 	ahci_ata_put_xfer(xa);
333 
334 	ata_fix_identify(&at->at_identify);
335 
336 	/*
337 	 * Read capacity using SATA probe info.
338 	 */
339 	if (le16toh(at->at_identify.cmdset83) & 0x0400) {
340 		/* LBA48 feature set supported */
341 		capacity = 0;
342 		for (i = 3; i >= 0; --i) {
343 			capacity <<= 16;
344 			capacity +=
345 			    le16toh(at->at_identify.addrsecxt[i]);
346 		}
347 	} else {
348 		capacity = le16toh(at->at_identify.addrsec[1]);
349 		capacity <<= 16;
350 		capacity += le16toh(at->at_identify.addrsec[0]);
351 	}
352 	if (capacity == 0)
353 		capacity = 1024 * 1024 / 512;
354 	at->at_capacity = capacity;
355 	if (atx == NULL)
356 		ap->ap_probe = ATA_PROBE_GOOD;
357 
358 	capacity_bytes = capacity * 512;
359 
360 	/*
361 	 * Negotiate NCQ, throw away any ata_xfer's beyond the negotiated
362 	 * number of slots and limit the number of CAM ccb's to one less
363 	 * so we always have a slot available for recovery.
364 	 *
365 	 * NCQ is not used if ap_ncqdepth is 1 or the host controller does
366 	 * not support it, and in that case the driver can handle extra
367 	 * ccb's.
368 	 *
369 	 * NCQ is currently used only with direct-attached disks.  It is
370 	 * not used with port multipliers or direct-attached ATAPI devices.
371 	 *
372 	 * Remember at least one extra CCB needs to be reserved for the
373 	 * error ccb.
374 	 */
375 	if ((ap->ap_sc->sc_cap & AHCI_REG_CAP_SNCQ) &&
376 	    ap->ap_type == ATA_PORT_T_DISK &&
377 	    (le16toh(at->at_identify.satacap) & (1 << 8))) {
378 		at->at_ncqdepth = (le16toh(at->at_identify.qdepth) & 0x1F) + 1;
379 		devncqdepth = at->at_ncqdepth;
380 		if (at->at_ncqdepth > ap->ap_sc->sc_ncmds)
381 			at->at_ncqdepth = ap->ap_sc->sc_ncmds;
382 		if (at->at_ncqdepth > 1) {
383 			for (i = 0; i < ap->ap_sc->sc_ncmds; ++i) {
384 				xa = ahci_ata_get_xfer(ap, atx);
385 				if (xa->tag < at->at_ncqdepth) {
386 					xa->state = ATA_S_COMPLETE;
387 					ahci_ata_put_xfer(xa);
388 				}
389 			}
390 			if (at->at_ncqdepth >= ap->ap_sc->sc_ncmds) {
391 				cam_sim_set_max_tags(ap->ap_sim,
392 						     at->at_ncqdepth - 1);
393 			}
394 		}
395 	} else {
396 		devncqdepth = 0;
397 	}
398 
399 	model_len = sizeof(at->at_identify.model);
400 	model_id = at->at_identify.model;
401 	ahci_strip_string(&model_id, &model_len);
402 
403 	firmware_len = sizeof(at->at_identify.firmware);
404 	firmware_id = at->at_identify.firmware;
405 	ahci_strip_string(&firmware_id, &firmware_len);
406 
407 	serial_len = sizeof(at->at_identify.serial);
408 	serial_id = at->at_identify.serial;
409 	ahci_strip_string(&serial_id, &serial_len);
410 
411 	/*
412 	 * Generate informatiive strings.
413 	 *
414 	 * NOTE: We do not automatically set write caching, lookahead,
415 	 *	 or the security state for ATAPI devices.
416 	 */
417 	if (at->at_identify.cmdset82 & ATA_IDENTIFY_WRITECACHE) {
418 		if (at->at_identify.features85 & ATA_IDENTIFY_WRITECACHE)
419 			wcstr = "enabled";
420 		else if (at->at_type == ATA_PORT_T_ATAPI)
421 			wcstr = "disabled";
422 		else
423 			wcstr = "enabling";
424 	} else {
425 		    wcstr = "notsupp";
426 	}
427 
428 	if (at->at_identify.cmdset82 & ATA_IDENTIFY_LOOKAHEAD) {
429 		if (at->at_identify.features85 & ATA_IDENTIFY_LOOKAHEAD)
430 			rastr = "enabled";
431 		else if (at->at_type == ATA_PORT_T_ATAPI)
432 			rastr = "disabled";
433 		else
434 			rastr = "enabling";
435 	} else {
436 		    rastr = "notsupp";
437 	}
438 
439 	if (at->at_identify.cmdset82 & ATA_IDENTIFY_SECURITY) {
440 		if (at->at_identify.securestatus & ATA_SECURE_FROZEN)
441 			scstr = "frozen";
442 		else if (at->at_type == ATA_PORT_T_ATAPI)
443 			scstr = "unfrozen";
444 		else if (AhciNoFeatures & (1 << ap->ap_num))
445 			scstr = "<disabled>";
446 		else
447 			scstr = "freezing";
448 	} else {
449 		    scstr = "notsupp";
450 	}
451 
452 	kprintf("%s: Found %s \"%*.*s %*.*s\" serial=\"%*.*s\"\n"
453 		"%s: tags=%d/%d satacap=%04x satafea=%04x NCQ=%s "
454 		"capacity=%lld.%02dMB\n",
455 
456 		ATANAME(ap, atx),
457 		type,
458 		model_len, model_len, model_id,
459 		firmware_len, firmware_len, firmware_id,
460 		serial_len, serial_len, serial_id,
461 
462 		ATANAME(ap, atx),
463 		devncqdepth, ap->ap_sc->sc_ncmds,
464 		at->at_identify.satacap,
465 		at->at_identify.satafsup,
466 		(at->at_ncqdepth > 1 ? "YES" : "NO"),
467 		(long long)capacity_bytes / (1024 * 1024),
468 		(int)(capacity_bytes % (1024 * 1024)) * 100 / (1024 * 1024)
469 	);
470 	kprintf("%s: f85=%04x f86=%04x f87=%04x WC=%s RA=%s SEC=%s\n",
471 		ATANAME(ap, atx),
472 		at->at_identify.features85,
473 		at->at_identify.features86,
474 		at->at_identify.features87,
475 		wcstr,
476 		rastr,
477 		scstr
478 	);
479 
480 	/*
481 	 * Additional type-specific probing
482 	 */
483 	switch(at->at_type) {
484 	case ATA_PORT_T_DISK:
485 		error = ahci_cam_probe_disk(ap, atx);
486 		break;
487 	case ATA_PORT_T_ATAPI:
488 		error = ahci_cam_probe_atapi(ap, atx);
489 		break;
490 	default:
491 		error = EIO;
492 		break;
493 	}
494 err:
495 	if (error) {
496 		at->at_probe = ATA_PROBE_FAILED;
497 		if (atx == NULL)
498 			ap->ap_probe = at->at_probe;
499 	} else {
500 		at->at_probe = ATA_PROBE_GOOD;
501 		if (atx == NULL)
502 			ap->ap_probe = at->at_probe;
503 	}
504 	return (error);
505 }
506 
507 /*
508  * DISK-specific probe after initial ident
509  */
510 static int
511 ahci_cam_probe_disk(struct ahci_port *ap, struct ata_port *atx)
512 {
513 	struct ata_port *at;
514 	struct ata_xfer	*xa;
515 
516 	at = atx ? atx : ap->ap_ata[0];
517 
518 	/*
519 	 * Set dummy xfer mode
520 	 */
521 	ahci_set_xfer(ap, atx);
522 
523 	/*
524 	 * Enable write cache if supported
525 	 *
526 	 * NOTE: "WD My Book" external disk devices have a very poor
527 	 *	 daughter board between the the ESATA and the HD.  Sending
528 	 *	 any ATA_C_SET_FEATURES commands will break the hardware port
529 	 *	 with a fatal protocol error.  However, this device also
530 	 *	 indicates that WRITECACHE is already on and READAHEAD is
531 	 *	 not supported so we avoid the issue.
532 	 */
533 	if ((at->at_identify.cmdset82 & ATA_IDENTIFY_WRITECACHE) &&
534 	    (at->at_identify.features85 & ATA_IDENTIFY_WRITECACHE) == 0) {
535 		xa = ahci_ata_get_xfer(ap, atx);
536 		xa->complete = ahci_ata_dummy_done;
537 		xa->fis->command = ATA_C_SET_FEATURES;
538 		xa->fis->features = ATA_SF_WRITECACHE_EN;
539 		/* xa->fis->features = ATA_SF_LOOKAHEAD_EN; */
540 		xa->fis->flags = ATA_H2D_FLAGS_CMD | at->at_target;
541 		xa->fis->device = 0;
542 		xa->flags = ATA_F_PIO | ATA_F_POLL;
543 		xa->timeout = 1000;
544 		xa->datalen = 0;
545 		if (ahci_ata_cmd(xa) == ATA_S_COMPLETE)
546 			at->at_features |= ATA_PORT_F_WCACHE;
547 		else
548 			kprintf("%s: Unable to enable write-caching\n",
549 				ATANAME(ap, atx));
550 		ahci_ata_put_xfer(xa);
551 	}
552 
553 	/*
554 	 * Enable readahead if supported
555 	 */
556 	if ((at->at_identify.cmdset82 & ATA_IDENTIFY_LOOKAHEAD) &&
557 	    (at->at_identify.features85 & ATA_IDENTIFY_LOOKAHEAD) == 0) {
558 		xa = ahci_ata_get_xfer(ap, atx);
559 		xa->complete = ahci_ata_dummy_done;
560 		xa->fis->command = ATA_C_SET_FEATURES;
561 		xa->fis->features = ATA_SF_LOOKAHEAD_EN;
562 		xa->fis->flags = ATA_H2D_FLAGS_CMD | at->at_target;
563 		xa->fis->device = 0;
564 		xa->flags = ATA_F_PIO | ATA_F_POLL;
565 		xa->timeout = 1000;
566 		xa->datalen = 0;
567 		if (ahci_ata_cmd(xa) == ATA_S_COMPLETE)
568 			at->at_features |= ATA_PORT_F_RAHEAD;
569 		else
570 			kprintf("%s: Unable to enable read-ahead\n",
571 				ATANAME(ap, atx));
572 		ahci_ata_put_xfer(xa);
573 	}
574 
575 	/*
576 	 * FREEZE LOCK the device so malicious users can't lock it on us.
577 	 * As there is no harm in issuing this to devices that don't
578 	 * support the security feature set we just send it, and don't bother
579 	 * checking if the device sends a command abort to tell us it doesn't
580 	 * support it
581 	 */
582 	if ((at->at_identify.cmdset82 & ATA_IDENTIFY_SECURITY) &&
583 	    (at->at_identify.securestatus & ATA_SECURE_FROZEN) == 0 &&
584 	    (AhciNoFeatures & (1 << ap->ap_num)) == 0) {
585 		xa = ahci_ata_get_xfer(ap, atx);
586 		xa->complete = ahci_ata_dummy_done;
587 		xa->fis->command = ATA_C_SEC_FREEZE_LOCK;
588 		xa->fis->flags = ATA_H2D_FLAGS_CMD | at->at_target;
589 		xa->flags = ATA_F_PIO | ATA_F_POLL;
590 		xa->timeout = 1000;
591 		xa->datalen = 0;
592 		if (ahci_ata_cmd(xa) == ATA_S_COMPLETE)
593 			at->at_features |= ATA_PORT_F_FRZLCK;
594 		else
595 			kprintf("%s: Unable to set security freeze\n",
596 				ATANAME(ap, atx));
597 		ahci_ata_put_xfer(xa);
598 	}
599 
600 	return (0);
601 }
602 
603 /*
604  * ATAPI-specific probe after initial ident
605  */
606 static int
607 ahci_cam_probe_atapi(struct ahci_port *ap, struct ata_port *atx)
608 {
609 	ahci_set_xfer(ap, atx);
610 	return(0);
611 }
612 
613 /*
614  * Setting the transfer mode is irrelevant for the SATA transport
615  * but some (atapi) devices seem to need it anyway.  In addition
616  * if we are running through a SATA->PATA converter for some reason
617  * beyond my comprehension we might have to set the mode.
618  *
619  * We only support DMA modes for SATA attached devices, so don't bother
620  * with legacy modes.
621  */
622 static int
623 ahci_set_xfer(struct ahci_port *ap, struct ata_port *atx)
624 {
625 	struct ata_port *at;
626 	struct ata_xfer	*xa;
627 	u_int16_t mode;
628 	u_int16_t mask;
629 
630 	at = atx ? atx : ap->ap_ata[0];
631 
632 	/*
633 	 * Figure out the supported UDMA mode.  Ignore other legacy modes.
634 	 */
635 	mask = le16toh(at->at_identify.ultradma);
636 	if ((mask & 0xFF) == 0 || mask == 0xFFFF)
637 		return(0);
638 	mask &= 0xFF;
639 	mode = 0x4F;
640 	while ((mask & 0x8000) == 0) {
641 		mask <<= 1;
642 		--mode;
643 	}
644 
645 	/*
646 	 * SATA atapi devices often still report a dma mode, even though
647 	 * it is irrelevant for SATA transport.  It is also possible that
648 	 * we are running through a SATA->PATA converter and seeing the
649 	 * PATA dma mode.
650 	 *
651 	 * In this case the device may require a (dummy) SETXFER to be
652 	 * sent before it will work properly.
653 	 */
654 	xa = ahci_ata_get_xfer(ap, atx);
655 	xa->complete = ahci_ata_dummy_done;
656 	xa->fis->command = ATA_C_SET_FEATURES;
657 	xa->fis->features = ATA_SF_SETXFER;
658 	xa->fis->flags = ATA_H2D_FLAGS_CMD | at->at_target;
659 	xa->fis->sector_count = mode;
660 	xa->flags = ATA_F_PIO | ATA_F_POLL;
661 	xa->timeout = 1000;
662 	xa->datalen = 0;
663 	if (ahci_ata_cmd(xa) != ATA_S_COMPLETE) {
664 		kprintf("%s: Unable to set dummy xfer mode \n",
665 			ATANAME(ap, atx));
666 	} else if (bootverbose) {
667 		kprintf("%s: Set dummy xfer mode to %02x\n",
668 			ATANAME(ap, atx), mode);
669 	}
670 	ahci_ata_put_xfer(xa);
671 	return(0);
672 }
673 
674 /*
675  * Fix byte ordering so buffers can be accessed as
676  * strings.
677  */
678 static void
679 ata_fix_identify(struct ata_identify *id)
680 {
681 	u_int16_t	*swap;
682 	int		i;
683 
684 	swap = (u_int16_t *)id->serial;
685 	for (i = 0; i < sizeof(id->serial) / sizeof(u_int16_t); i++)
686 		swap[i] = bswap16(swap[i]);
687 
688 	swap = (u_int16_t *)id->firmware;
689 	for (i = 0; i < sizeof(id->firmware) / sizeof(u_int16_t); i++)
690 		swap[i] = bswap16(swap[i]);
691 
692 	swap = (u_int16_t *)id->model;
693 	for (i = 0; i < sizeof(id->model) / sizeof(u_int16_t); i++)
694 		swap[i] = bswap16(swap[i]);
695 }
696 
697 /*
698  * Dummy done callback for xa.
699  */
700 static void
701 ahci_ata_dummy_done(struct ata_xfer *xa)
702 {
703 }
704 
705 /*
706  * Use an engineering request to initiate a target scan for devices
707  * behind a port multiplier.
708  *
709  * An asynchronous bus scan is used to avoid reentrancy issues.
710  */
711 static void
712 ahci_cam_rescan_callback(struct cam_periph *periph, union ccb *ccb)
713 {
714 	struct ahci_port *ap = ccb->ccb_h.sim_priv.entries[0].ptr;
715 
716 	if (ccb->ccb_h.func_code == XPT_SCAN_BUS) {
717 		ap->ap_flags &= ~AP_F_SCAN_RUNNING;
718 		if (ap->ap_flags & AP_F_SCAN_REQUESTED) {
719 			ap->ap_flags &= ~AP_F_SCAN_REQUESTED;
720 			ahci_cam_rescan(ap);
721 		}
722 		ap->ap_flags |= AP_F_SCAN_COMPLETED;
723 		wakeup(&ap->ap_flags);
724 	}
725 	xpt_free_ccb(ccb);
726 }
727 
728 static void
729 ahci_cam_rescan(struct ahci_port *ap)
730 {
731 	struct cam_path *path;
732 	union ccb *ccb;
733 	int status;
734 	int i;
735 
736 	if (ap->ap_flags & AP_F_SCAN_RUNNING) {
737 		ap->ap_flags |= AP_F_SCAN_REQUESTED;
738 		return;
739 	}
740 	ap->ap_flags |= AP_F_SCAN_RUNNING;
741 	for (i = 0; i < AHCI_MAX_PMPORTS; ++i) {
742 		ap->ap_ata[i]->at_features |= ATA_PORT_F_RESCAN;
743 	}
744 
745 	status = xpt_create_path(&path, xpt_periph, cam_sim_path(ap->ap_sim),
746 				 CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
747 	if (status != CAM_REQ_CMP)
748 		return;
749 
750 	ccb = xpt_alloc_ccb();
751 	xpt_setup_ccb(&ccb->ccb_h, path, 5);	/* 5 = low priority */
752 	ccb->ccb_h.func_code = XPT_ENG_EXEC;
753 	ccb->ccb_h.cbfcnp = ahci_cam_rescan_callback;
754 	ccb->ccb_h.sim_priv.entries[0].ptr = ap;
755 	ccb->crcn.flags = CAM_FLAG_NONE;
756 	xpt_action_async(ccb);
757 }
758 
759 static void
760 ahci_xpt_rescan(struct ahci_port *ap)
761 {
762 	struct cam_path *path;
763 	union ccb *ccb;
764 	int status;
765 
766 	status = xpt_create_path(&path, xpt_periph, cam_sim_path(ap->ap_sim),
767 				 CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
768 	if (status != CAM_REQ_CMP)
769 		return;
770 
771 	ccb = xpt_alloc_ccb();
772 	xpt_setup_ccb(&ccb->ccb_h, path, 5);	/* 5 = low priority */
773 	ccb->ccb_h.func_code = XPT_SCAN_BUS;
774 	ccb->ccb_h.cbfcnp = ahci_cam_rescan_callback;
775 	ccb->ccb_h.sim_priv.entries[0].ptr = ap;
776 	ccb->crcn.flags = CAM_FLAG_NONE;
777 	xpt_action_async(ccb);
778 }
779 
780 /*
781  * Action function - dispatch command
782  */
783 static
784 void
785 ahci_xpt_action(struct cam_sim *sim, union ccb *ccb)
786 {
787 	struct ahci_port *ap;
788 	struct ata_port	 *at, *atx;
789 	struct ccb_hdr *ccbh;
790 
791 	/* XXX lock */
792 	ap = cam_sim_softc(sim);
793 	atx = NULL;
794 	KKASSERT(ap != NULL);
795 	ccbh = &ccb->ccb_h;
796 
797 	/*
798 	 * Early failure checks.  These checks do not apply to XPT_PATH_INQ,
799 	 * otherwise the bus rescan will not remove the dead devices when
800 	 * unplugging a PM.
801 	 *
802 	 * For non-wildcards we have one target (0) and one lun (0),
803 	 * unless we have a port multiplier.
804 	 *
805 	 * A wildcard target indicates only the general bus is being
806 	 * probed.
807 	 *
808 	 * Calculate at and atx.  at is always non-NULL.  atx is only
809 	 * NULL for direct-attached devices.  It will be non-NULL for
810 	 * devices behind a port multiplier.
811 	 *
812 	 * XXX What do we do with a LUN wildcard?
813 	 */
814 	if (ccbh->target_id != CAM_TARGET_WILDCARD &&
815 	    ccbh->func_code != XPT_PATH_INQ) {
816 		if (ap->ap_type == ATA_PORT_T_NONE) {
817 			ccbh->status = CAM_DEV_NOT_THERE;
818 			xpt_done(ccb);
819 			return;
820 		}
821 		if (ccbh->target_id < 0 || ccbh->target_id >= ap->ap_pmcount) {
822 			ccbh->status = CAM_DEV_NOT_THERE;
823 			xpt_done(ccb);
824 			return;
825 		}
826 		at = ap->ap_ata[ccbh->target_id];
827 		if (ap->ap_type == ATA_PORT_T_PM)
828 			atx = at;
829 
830 		if (ccbh->target_lun != CAM_LUN_WILDCARD && ccbh->target_lun) {
831 			ccbh->status = CAM_DEV_NOT_THERE;
832 			xpt_done(ccb);
833 			return;
834 		}
835 	} else {
836 		at = ap->ap_ata[0];
837 	}
838 
839 	/*
840 	 * Switch on the meta XPT command
841 	 */
842 	switch(ccbh->func_code) {
843 	case XPT_ENG_EXEC:
844 		/*
845 		 * This routine is called after a port multiplier has been
846 		 * probed.
847 		 */
848 		ccbh->status = CAM_REQ_CMP;
849 		ahci_os_lock_port(ap);
850 		ahci_port_state_machine(ap, 0);
851 		ahci_os_unlock_port(ap);
852 		xpt_done(ccb);
853 		ahci_xpt_rescan(ap);
854 		break;
855 	case XPT_PATH_INQ:
856 		/*
857 		 * This command always succeeds, otherwise the bus scan
858 		 * will not detach dead devices.
859 		 */
860 		ccb->cpi.version_num = 1;
861 		ccb->cpi.hba_inquiry = 0;
862 		ccb->cpi.target_sprt = 0;
863 		ccb->cpi.hba_misc = PIM_SEQSCAN;
864 		ccb->cpi.hba_eng_cnt = 0;
865 		bzero(ccb->cpi.vuhba_flags, sizeof(ccb->cpi.vuhba_flags));
866 		ccb->cpi.max_target = AHCI_MAX_PMPORTS - 1;
867 		ccb->cpi.max_lun = 0;
868 		ccb->cpi.async_flags = 0;
869 		ccb->cpi.hpath_id = 0;
870 		ccb->cpi.initiator_id = AHCI_MAX_PMPORTS - 1;
871 		ccb->cpi.unit_number = cam_sim_unit(sim);
872 		ccb->cpi.bus_id = cam_sim_bus(sim);
873 		ccb->cpi.base_transfer_speed = 150000;
874 		ccb->cpi.transport = XPORT_SATA;
875 		ccb->cpi.transport_version = 1;
876 		ccb->cpi.protocol = PROTO_SCSI;
877 		ccb->cpi.protocol_version = SCSI_REV_2;
878 		ccb->cpi.maxio = AHCI_MAXPHYS;
879 
880 		ccbh->status = CAM_REQ_CMP;
881 		if (ccbh->target_id == CAM_TARGET_WILDCARD) {
882 			ahci_os_lock_port(ap);
883 			ahci_port_state_machine(ap, 0);
884 			ahci_os_unlock_port(ap);
885 		} else {
886 			switch(ahci_pread(ap, AHCI_PREG_SSTS) &
887 			       AHCI_PREG_SSTS_SPD) {
888 			case AHCI_PREG_SSTS_SPD_GEN1:
889 				ccb->cpi.base_transfer_speed = 150000;
890 				break;
891 			case AHCI_PREG_SSTS_SPD_GEN2:
892 				ccb->cpi.base_transfer_speed = 300000;
893 				break;
894 			case AHCI_PREG_SSTS_SPD_GEN3:
895 				ccb->cpi.base_transfer_speed = 600000;
896 				break;
897 			default:
898 				/* unknown */
899 				ccb->cpi.base_transfer_speed = 1000;
900 				break;
901 			}
902 #if 0
903 			if (ap->ap_type == ATA_PORT_T_NONE)
904 				ccbh->status = CAM_DEV_NOT_THERE;
905 #endif
906 		}
907 		xpt_done(ccb);
908 		break;
909 	case XPT_RESET_DEV:
910 		ahci_os_lock_port(ap);
911 		if (ap->ap_type == ATA_PORT_T_NONE) {
912 			ccbh->status = CAM_DEV_NOT_THERE;
913 		} else {
914 			ahci_port_reset(ap, atx, 0);
915 			ccbh->status = CAM_REQ_CMP;
916 		}
917 		ahci_os_unlock_port(ap);
918 		xpt_done(ccb);
919 		break;
920 	case XPT_RESET_BUS:
921 		ahci_os_lock_port(ap);
922 		ahci_port_reset(ap, NULL, 1);
923 		ahci_os_unlock_port(ap);
924 		ccbh->status = CAM_REQ_CMP;
925 		xpt_done(ccb);
926 		break;
927 	case XPT_SET_TRAN_SETTINGS:
928 		ccbh->status = CAM_FUNC_NOTAVAIL;
929 		xpt_done(ccb);
930 		break;
931 	case XPT_GET_TRAN_SETTINGS:
932 		ccb->cts.protocol = PROTO_SCSI;
933 		ccb->cts.protocol_version = SCSI_REV_2;
934 		ccb->cts.transport = XPORT_SATA;
935 		ccb->cts.transport_version = XPORT_VERSION_UNSPECIFIED;
936 		ccb->cts.proto_specific.valid = 0;
937 		ccb->cts.xport_specific.valid = 0;
938 		ccbh->status = CAM_REQ_CMP;
939 		xpt_done(ccb);
940 		break;
941 	case XPT_CALC_GEOMETRY:
942 		cam_calc_geometry(&ccb->ccg, 1);
943 		xpt_done(ccb);
944 		break;
945 	case XPT_SCSI_IO:
946 		/*
947 		 * Our parallel startup code might have only probed through
948 		 * to the IDENT, so do the last step if necessary.
949 		 */
950 		if (at->at_probe == ATA_PROBE_NEED_IDENT)
951 			ahci_cam_probe(ap, atx);
952 		if (at->at_probe != ATA_PROBE_GOOD) {
953 			ccbh->status = CAM_DEV_NOT_THERE;
954 			xpt_done(ccb);
955 			break;
956 		}
957 		switch(at->at_type) {
958 		case ATA_PORT_T_DISK:
959 			ahci_xpt_scsi_disk_io(ap, atx, ccb);
960 			break;
961 		case ATA_PORT_T_ATAPI:
962 			ahci_xpt_scsi_atapi_io(ap, atx, ccb);
963 			break;
964 		default:
965 			ccbh->status = CAM_REQ_INVALID;
966 			xpt_done(ccb);
967 			break;
968 		}
969 		break;
970 	case XPT_TRIM:
971 	{
972 		scsi_cdb_t cdb;
973 		struct ccb_scsiio *csio;
974 		csio = &ccb->csio;
975 		cdb = (void *)((ccbh->flags & CAM_CDB_POINTER) ?
976 		    csio->cdb_io.cdb_ptr : csio->cdb_io.cdb_bytes);
977 		cdb->generic.opcode = TRIM;
978 		ahci_xpt_scsi_disk_io(ap, atx, ccb);
979 		break;
980 	}
981 	default:
982 		ccbh->status = CAM_REQ_INVALID;
983 		xpt_done(ccb);
984 		break;
985 	}
986 }
987 
988 /*
989  * Poll function.
990  *
991  * Generally this function gets called heavily when interrupts might be
992  * non-operational, during a halt/reboot or panic.
993  */
994 static
995 void
996 ahci_xpt_poll(struct cam_sim *sim)
997 {
998 	struct ahci_port *ap;
999 
1000 	ap = cam_sim_softc(sim);
1001 	crit_enter();
1002 	ahci_os_lock_port(ap);
1003 	ahci_port_intr(ap, 1);
1004 	ahci_os_unlock_port(ap);
1005 	crit_exit();
1006 }
1007 
1008 /*
1009  * Convert the SCSI command in ccb to an ata_xfer command in xa
1010  * for ATA_PORT_T_DISK operations.  Set the completion function
1011  * to convert the response back, then dispatch to the OpenBSD AHCI
1012  * layer.
1013  *
1014  * AHCI DISK commands only support a limited command set, and we
1015  * fake additional commands to make it play nice with the CAM subsystem.
1016  */
1017 static
1018 void
1019 ahci_xpt_scsi_disk_io(struct ahci_port *ap, struct ata_port *atx,
1020 		      union ccb *ccb)
1021 {
1022 	struct ccb_hdr *ccbh;
1023 	struct ccb_scsiio *csio;
1024 	struct ata_xfer *xa;
1025 	struct ata_port	*at;
1026 	struct ata_fis_h2d *fis;
1027 	struct ata_pass_12 *atp12;
1028 	struct ata_pass_16 *atp16;
1029 	scsi_cdb_t cdb;
1030 	union scsi_data *rdata;
1031 	int rdata_len;
1032 	u_int64_t capacity;
1033 	u_int64_t lba;
1034 	u_int32_t count;
1035 
1036 	ccbh = &ccb->csio.ccb_h;
1037 	csio = &ccb->csio;
1038 	at = atx ? atx : ap->ap_ata[0];
1039 
1040 	/*
1041 	 * XXX not passing NULL at for direct attach!
1042 	 */
1043 	xa = ahci_ata_get_xfer(ap, atx);
1044 	rdata = (void *)csio->data_ptr;
1045 	rdata_len = csio->dxfer_len;
1046 
1047 	/*
1048 	 * Build the FIS or process the csio to completion.
1049 	 */
1050 	cdb = (void *)((ccbh->flags & CAM_CDB_POINTER) ?
1051 			csio->cdb_io.cdb_ptr : csio->cdb_io.cdb_bytes);
1052 
1053 	switch(cdb->generic.opcode) {
1054 	case REQUEST_SENSE:
1055 		/*
1056 		 * Auto-sense everything, so explicit sense requests
1057 		 * return no-sense.
1058 		 */
1059 		ccbh->status = CAM_SCSI_STATUS_ERROR;
1060 		break;
1061 	case INQUIRY:
1062 		/*
1063 		 * Inquiry supported features
1064 		 *
1065 		 * [opcode, byte2, page_code, length, control]
1066 		 */
1067 		if (cdb->inquiry.byte2 & SI_EVPD) {
1068 			ahci_xpt_page_inquiry(ap, at, ccb);
1069 		} else {
1070 			bzero(rdata, rdata_len);
1071 			if (rdata_len < SHORT_INQUIRY_LENGTH) {
1072 				ccbh->status = CAM_CCB_LEN_ERR;
1073 				break;
1074 			}
1075 			if (rdata_len > sizeof(rdata->inquiry_data))
1076 				rdata_len = sizeof(rdata->inquiry_data);
1077 			rdata->inquiry_data.device = T_DIRECT;
1078 			rdata->inquiry_data.version = SCSI_REV_SPC2;
1079 			rdata->inquiry_data.response_format = 2;
1080 			rdata->inquiry_data.additional_length = 32;
1081 			bcopy("SATA    ", rdata->inquiry_data.vendor, 8);
1082 			bcopy(at->at_identify.model,
1083 			      rdata->inquiry_data.product,
1084 			      sizeof(rdata->inquiry_data.product));
1085 			bcopy(at->at_identify.firmware,
1086 			      rdata->inquiry_data.revision,
1087 			      sizeof(rdata->inquiry_data.revision));
1088 			ccbh->status = CAM_REQ_CMP;
1089 		}
1090 
1091 		/*
1092 		 * Use the vendor specific area to set the TRIM status
1093 		 * for scsi_da
1094 		 */
1095 		if (at->at_identify.support_dsm) {
1096 			rdata->inquiry_data.vendor_specific1[0] =
1097 			    at->at_identify.support_dsm &ATA_SUPPORT_DSM_TRIM;
1098 			rdata->inquiry_data.vendor_specific1[1] =
1099 			    at->at_identify.max_dsm_blocks;
1100 		}
1101 		break;
1102 	case READ_CAPACITY_16:
1103 		if (cdb->read_capacity_16.service_action != SRC16_SERVICE_ACTION) {
1104 			ccbh->status = CAM_REQ_INVALID;
1105 			break;
1106 		}
1107 		if (rdata_len < sizeof(rdata->read_capacity_data_16)) {
1108 			ccbh->status = CAM_CCB_LEN_ERR;
1109 			break;
1110 		}
1111 		/* fall through */
1112 	case READ_CAPACITY:
1113 		if (rdata_len < sizeof(rdata->read_capacity_data)) {
1114 			ccbh->status = CAM_CCB_LEN_ERR;
1115 			break;
1116 		}
1117 
1118 		capacity = at->at_capacity;
1119 
1120 		bzero(rdata, rdata_len);
1121 		if (cdb->generic.opcode == READ_CAPACITY) {
1122 			rdata_len = sizeof(rdata->read_capacity_data);
1123 			if (capacity > 0xFFFFFFFFU) {
1124 				/*
1125 				 * Set capacity to 0 so maxsector winds up
1126 				 * being 0xffffffff in CAM in order to trigger
1127 				 * DA_STATE_PROBE2.
1128 				 */
1129 				capacity = 0;
1130 			}
1131 			bzero(&rdata->read_capacity_data, rdata_len);
1132 			scsi_ulto4b((u_int32_t)capacity - 1,
1133 				    rdata->read_capacity_data.addr);
1134 			scsi_ulto4b(512, rdata->read_capacity_data.length);
1135 		} else {
1136 			rdata_len = sizeof(rdata->read_capacity_data_16);
1137 			bzero(&rdata->read_capacity_data_16, rdata_len);
1138 			scsi_u64to8b(capacity - 1,
1139 				     rdata->read_capacity_data_16.addr);
1140 			scsi_ulto4b(512, rdata->read_capacity_data_16.length);
1141 		}
1142 		ccbh->status = CAM_REQ_CMP;
1143 		break;
1144 	case SYNCHRONIZE_CACHE:
1145 		/*
1146 		 * Synchronize cache.  Specification says this can take
1147 		 * greater then 30 seconds so give it at least 45.
1148 		 */
1149 		fis = xa->fis;
1150 		fis->flags = ATA_H2D_FLAGS_CMD;
1151 		fis->command = ATA_C_FLUSH_CACHE;
1152 		fis->device = 0;
1153 		if (xa->timeout < 45000)
1154 			xa->timeout = 45000;
1155 		xa->datalen = 0;
1156 		xa->flags = 0;
1157 		xa->complete = ahci_ata_complete_disk_synchronize_cache;
1158 		break;
1159 	case TRIM:
1160 		fis = xa->fis;
1161 		fis->command = ATA_C_DATA_SET_MANAGEMENT;
1162 		fis->features = (u_int8_t)ATA_SF_DSM_TRIM;
1163 		fis->features_exp = (u_int8_t)(ATA_SF_DSM_TRIM>> 8);
1164 
1165 		xa->flags = ATA_F_WRITE;
1166 		fis->flags = ATA_H2D_FLAGS_CMD;
1167 
1168 		xa->data = csio->data_ptr;
1169 		xa->datalen = csio->dxfer_len;
1170 		xa->timeout = ccbh->timeout*50;	/* milliseconds */
1171 
1172 		fis->sector_count =(u_int8_t)(xa->datalen/512);
1173 		fis->sector_count_exp =(u_int8_t)((xa->datalen/512)>>8);
1174 
1175 		lba = 0;
1176 		fis->lba_low = (u_int8_t)lba;
1177 		fis->lba_mid = (u_int8_t)(lba >> 8);
1178 		fis->lba_high = (u_int8_t)(lba >> 16);
1179 		fis->lba_low_exp = (u_int8_t)(lba >> 24);
1180 		fis->lba_mid_exp = (u_int8_t)(lba >> 32);
1181 		fis->lba_high_exp = (u_int8_t)(lba >> 40);
1182 
1183 		fis->device = ATA_H2D_DEVICE_LBA;
1184 		xa->data = csio->data_ptr;
1185 
1186 		xa->complete = ahci_ata_complete_disk_rw;
1187 		ccbh->status = CAM_REQ_INPROG;
1188 		break;
1189 	case TEST_UNIT_READY:
1190 	case START_STOP_UNIT:
1191 	case PREVENT_ALLOW:
1192 		/*
1193 		 * Just silently return success
1194 		 */
1195 		ccbh->status = CAM_REQ_CMP;
1196 		rdata_len = 0;
1197 		break;
1198 	case ATA_PASS_12:
1199 		atp12 = &cdb->ata_pass_12;
1200 		fis = xa->fis;
1201 		/*
1202 		 * Figure out the flags to be used, depending on the direction of the
1203 		 * CAM request.
1204 		 */
1205 		switch (ccbh->flags & CAM_DIR_MASK) {
1206 		case CAM_DIR_IN:
1207 			xa->flags = ATA_F_READ;
1208 			break;
1209 		case CAM_DIR_OUT:
1210 			xa->flags = ATA_F_WRITE;
1211 			break;
1212 		default:
1213 			xa->flags = 0;
1214 		}
1215 		xa->flags |= ATA_F_POLL | ATA_F_EXCLUSIVE;
1216 		xa->data = csio->data_ptr;
1217 		xa->datalen = csio->dxfer_len;
1218 		xa->complete = ahci_ata_complete_disk_rw;
1219 		xa->timeout = ccbh->timeout;
1220 
1221 		/*
1222 		 * Populate the fis from the information we received through CAM
1223 		 * ATA passthrough.
1224 		 */
1225 		fis->flags = ATA_H2D_FLAGS_CMD;	/* maybe also atp12->flags ? */
1226 		fis->features = atp12->features;
1227 		fis->sector_count = atp12->sector_count;
1228 		fis->lba_low = atp12->lba_low;
1229 		fis->lba_mid = atp12->lba_mid;
1230 		fis->lba_high = atp12->lba_high;
1231 		fis->device = atp12->device;	/* maybe always 0? */
1232 		fis->command = atp12->command;
1233 		fis->control = atp12->control;
1234 
1235 		/*
1236 		 * Mark as in progress so it is sent to the device.
1237 		 */
1238 		ccbh->status = CAM_REQ_INPROG;
1239 		break;
1240 	case ATA_PASS_16:
1241 		atp16 = &cdb->ata_pass_16;
1242 		fis = xa->fis;
1243 		/*
1244 		 * Figure out the flags to be used, depending on the direction of the
1245 		 * CAM request.
1246 		 */
1247 		switch (ccbh->flags & CAM_DIR_MASK) {
1248 		case CAM_DIR_IN:
1249 			xa->flags = ATA_F_READ;
1250 			break;
1251 		case CAM_DIR_OUT:
1252 			xa->flags = ATA_F_WRITE;
1253 			break;
1254 		default:
1255 			xa->flags = 0;
1256 		}
1257 		xa->flags |= ATA_F_POLL | ATA_F_EXCLUSIVE;
1258 		xa->data = csio->data_ptr;
1259 		xa->datalen = csio->dxfer_len;
1260 		xa->complete = ahci_ata_complete_disk_rw;
1261 		xa->timeout = ccbh->timeout;
1262 
1263 		/*
1264 		 * Populate the fis from the information we received through CAM
1265 		 * ATA passthrough.
1266 		 */
1267 		fis->flags = ATA_H2D_FLAGS_CMD;	/* maybe also atp16->flags ? */
1268 		fis->features = atp16->features;
1269 		fis->features_exp = atp16->features_ext;
1270 		fis->sector_count = atp16->sector_count;
1271 		fis->sector_count_exp = atp16->sector_count_ext;
1272 		fis->lba_low = atp16->lba_low;
1273 		fis->lba_low_exp = atp16->lba_low_ext;
1274 		fis->lba_mid = atp16->lba_mid;
1275 		fis->lba_mid_exp = atp16->lba_mid_ext;
1276 		fis->lba_high = atp16->lba_high;
1277 		fis->lba_mid_exp = atp16->lba_mid_ext;
1278 		fis->device = atp16->device;	/* maybe always 0? */
1279 		fis->command = atp16->command;
1280 
1281 		/*
1282 		 * Mark as in progress so it is sent to the device.
1283 		 */
1284 		ccbh->status = CAM_REQ_INPROG;
1285 		break;
1286 	default:
1287 		switch(cdb->generic.opcode) {
1288 		case READ_6:
1289 			lba = scsi_3btoul(cdb->rw_6.addr) & 0x1FFFFF;
1290 			count = cdb->rw_6.length ? cdb->rw_6.length : 0x100;
1291 			xa->flags = ATA_F_READ;
1292 			break;
1293 		case READ_10:
1294 			lba = scsi_4btoul(cdb->rw_10.addr);
1295 			count = scsi_2btoul(cdb->rw_10.length);
1296 			xa->flags = ATA_F_READ;
1297 			break;
1298 		case READ_12:
1299 			lba = scsi_4btoul(cdb->rw_12.addr);
1300 			count = scsi_4btoul(cdb->rw_12.length);
1301 			xa->flags = ATA_F_READ;
1302 			break;
1303 		case READ_16:
1304 			lba = scsi_8btou64(cdb->rw_16.addr);
1305 			count = scsi_4btoul(cdb->rw_16.length);
1306 			xa->flags = ATA_F_READ;
1307 			break;
1308 		case WRITE_6:
1309 			lba = scsi_3btoul(cdb->rw_6.addr) & 0x1FFFFF;
1310 			count = cdb->rw_6.length ? cdb->rw_6.length : 0x100;
1311 			xa->flags = ATA_F_WRITE;
1312 			break;
1313 		case WRITE_10:
1314 			lba = scsi_4btoul(cdb->rw_10.addr);
1315 			count = scsi_2btoul(cdb->rw_10.length);
1316 			xa->flags = ATA_F_WRITE;
1317 			break;
1318 		case WRITE_12:
1319 			lba = scsi_4btoul(cdb->rw_12.addr);
1320 			count = scsi_4btoul(cdb->rw_12.length);
1321 			xa->flags = ATA_F_WRITE;
1322 			break;
1323 		case WRITE_16:
1324 			lba = scsi_8btou64(cdb->rw_16.addr);
1325 			count = scsi_4btoul(cdb->rw_16.length);
1326 			xa->flags = ATA_F_WRITE;
1327 			break;
1328 		default:
1329 			ccbh->status = CAM_REQ_INVALID;
1330 			break;
1331 		}
1332 		if (ccbh->status != CAM_REQ_INPROG)
1333 			break;
1334 
1335 		fis = xa->fis;
1336 		fis->flags = ATA_H2D_FLAGS_CMD;
1337 		fis->lba_low = (u_int8_t)lba;
1338 		fis->lba_mid = (u_int8_t)(lba >> 8);
1339 		fis->lba_high = (u_int8_t)(lba >> 16);
1340 		fis->device = ATA_H2D_DEVICE_LBA;
1341 
1342 		/*
1343 		 * NCQ only for direct-attached disks, do not currently
1344 		 * try to use NCQ with port multipliers.
1345 		 */
1346 		if (at->at_ncqdepth > 1 &&
1347 		    ap->ap_type == ATA_PORT_T_DISK &&
1348 		    (ap->ap_sc->sc_cap & AHCI_REG_CAP_SNCQ) &&
1349 		    (ccbh->flags & CAM_POLLED) == 0) {
1350 			/*
1351 			 * Use NCQ - always uses 48 bit addressing
1352 			 */
1353 			xa->flags |= ATA_F_NCQ;
1354 			fis->command = (xa->flags & ATA_F_WRITE) ?
1355 					ATA_C_WRITE_FPDMA : ATA_C_READ_FPDMA;
1356 			fis->lba_low_exp = (u_int8_t)(lba >> 24);
1357 			fis->lba_mid_exp = (u_int8_t)(lba >> 32);
1358 			fis->lba_high_exp = (u_int8_t)(lba >> 40);
1359 			fis->sector_count = xa->tag << 3;
1360 			fis->features = (u_int8_t)count;
1361 			fis->features_exp = (u_int8_t)(count >> 8);
1362 		} else if (count > 0x100 || lba > 0x0FFFFFFFU) {
1363 			/*
1364 			 * Use LBA48
1365 			 */
1366 			fis->command = (xa->flags & ATA_F_WRITE) ?
1367 					ATA_C_WRITEDMA_EXT : ATA_C_READDMA_EXT;
1368 			fis->lba_low_exp = (u_int8_t)(lba >> 24);
1369 			fis->lba_mid_exp = (u_int8_t)(lba >> 32);
1370 			fis->lba_high_exp = (u_int8_t)(lba >> 40);
1371 			fis->sector_count = (u_int8_t)count;
1372 			fis->sector_count_exp = (u_int8_t)(count >> 8);
1373 		} else {
1374 			/*
1375 			 * Use LBA
1376 			 *
1377 			 * NOTE: 256 sectors is supported, stored as 0.
1378 			 */
1379 			fis->command = (xa->flags & ATA_F_WRITE) ?
1380 					ATA_C_WRITEDMA : ATA_C_READDMA;
1381 			fis->device |= (u_int8_t)(lba >> 24) & 0x0F;
1382 			fis->sector_count = (u_int8_t)count;
1383 		}
1384 
1385 		xa->data = csio->data_ptr;
1386 		xa->datalen = csio->dxfer_len;
1387 		xa->complete = ahci_ata_complete_disk_rw;
1388 		xa->timeout = ccbh->timeout;	/* milliseconds */
1389 #if 0
1390 		if (xa->timeout > 10000)	/* XXX - debug */
1391 			xa->timeout = 10000;
1392 #endif
1393 		if (ccbh->flags & CAM_POLLED)
1394 			xa->flags |= ATA_F_POLL;
1395 		break;
1396 	}
1397 
1398 	/*
1399 	 * If the request is still in progress the xa and FIS have
1400 	 * been set up (except for the PM target), and must be dispatched.
1401 	 * Otherwise the request was completed.
1402 	 */
1403 	if (ccbh->status == CAM_REQ_INPROG) {
1404 		KKASSERT(xa->complete != NULL);
1405 		xa->atascsi_private = ccb;
1406 		ccb->ccb_h.sim_priv.entries[0].ptr = ap;
1407 		ahci_os_lock_port(ap);
1408 		xa->fis->flags |= at->at_target;
1409 		ahci_ata_cmd(xa);
1410 		ahci_os_unlock_port(ap);
1411 	} else {
1412 		ahci_ata_put_xfer(xa);
1413 		xpt_done(ccb);
1414 	}
1415 }
1416 
1417 /*
1418  * Convert the SCSI command in ccb to an ata_xfer command in xa
1419  * for ATA_PORT_T_ATAPI operations.  Set the completion function
1420  * to convert the response back, then dispatch to the OpenBSD AHCI
1421  * layer.
1422  */
1423 static
1424 void
1425 ahci_xpt_scsi_atapi_io(struct ahci_port *ap, struct ata_port *atx,
1426 			union ccb *ccb)
1427 {
1428 	struct ccb_hdr *ccbh;
1429 	struct ccb_scsiio *csio;
1430 	struct ata_xfer *xa;
1431 	struct ata_fis_h2d *fis;
1432 	scsi_cdb_t cdbs;
1433 	scsi_cdb_t cdbd;
1434 	int flags;
1435 	struct ata_port	*at;
1436 
1437 	ccbh = &ccb->csio.ccb_h;
1438 	csio = &ccb->csio;
1439 	at = atx ? atx : ap->ap_ata[0];
1440 
1441 	switch (ccbh->flags & CAM_DIR_MASK) {
1442 	case CAM_DIR_IN:
1443 		flags = ATA_F_PACKET | ATA_F_READ;
1444 		break;
1445 	case CAM_DIR_OUT:
1446 		flags = ATA_F_PACKET | ATA_F_WRITE;
1447 		break;
1448 	case CAM_DIR_NONE:
1449 		flags = ATA_F_PACKET;
1450 		break;
1451 	default:
1452 		ccbh->status = CAM_REQ_INVALID;
1453 		xpt_done(ccb);
1454 		return;
1455 		/* NOT REACHED */
1456 	}
1457 
1458 	/*
1459 	 * Special handling to get the rfis back into host memory while
1460 	 * still allowing the chip to run commands in parallel to
1461 	 * ATAPI devices behind a PM.
1462 	 */
1463 	flags |= ATA_F_AUTOSENSE;
1464 
1465 	/*
1466 	 * The command has to fit in the packet command buffer.
1467 	 */
1468 	if (csio->cdb_len < 6 || csio->cdb_len > 16) {
1469 		ccbh->status = CAM_CCB_LEN_ERR;
1470 		xpt_done(ccb);
1471 		return;
1472 	}
1473 
1474 	/*
1475 	 * Initialize the XA and FIS.  It is unclear how much of
1476 	 * this has to mimic the equivalent ATA command.
1477 	 *
1478 	 * XXX not passing NULL at for direct attach!
1479 	 */
1480 	xa = ahci_ata_get_xfer(ap, atx);
1481 	fis = xa->fis;
1482 
1483 	fis->flags = ATA_H2D_FLAGS_CMD | at->at_target;
1484 	fis->command = ATA_C_PACKET;
1485 	fis->device = ATA_H2D_DEVICE_LBA;
1486 	fis->sector_count = xa->tag << 3;
1487 	if (flags & (ATA_F_READ | ATA_F_WRITE)) {
1488 		if (flags & ATA_F_WRITE) {
1489 			fis->features = ATA_H2D_FEATURES_DMA |
1490 				       ATA_H2D_FEATURES_DIR_WRITE;
1491 		} else {
1492 			fis->features = ATA_H2D_FEATURES_DMA |
1493 				       ATA_H2D_FEATURES_DIR_READ;
1494 		}
1495 	} else {
1496 		fis->lba_mid = 0;
1497 		fis->lba_high = 0;
1498 	}
1499 	fis->control = ATA_FIS_CONTROL_4BIT;
1500 
1501 	xa->flags = flags;
1502 	xa->data = csio->data_ptr;
1503 	xa->datalen = csio->dxfer_len;
1504 	xa->timeout = ccbh->timeout;	/* milliseconds */
1505 
1506 	if (ccbh->flags & CAM_POLLED)
1507 		xa->flags |= ATA_F_POLL;
1508 
1509 	/*
1510 	 * Copy the cdb to the packetcmd buffer in the FIS using a
1511 	 * convenient pointer in the xa.
1512 	 *
1513 	 * Zero-out any trailing bytes in case the ATAPI device cares.
1514 	 */
1515 	cdbs = (void *)((ccbh->flags & CAM_CDB_POINTER) ?
1516 			csio->cdb_io.cdb_ptr : csio->cdb_io.cdb_bytes);
1517 	bcopy(cdbs, xa->packetcmd, csio->cdb_len);
1518 	if (csio->cdb_len < 16)
1519 		bzero(xa->packetcmd + csio->cdb_len, 16 - csio->cdb_len);
1520 
1521 #if 0
1522 	kprintf("opcode %d cdb_len %d dxfer_len %d\n",
1523 		cdbs->generic.opcode,
1524 		csio->cdb_len, csio->dxfer_len);
1525 #endif
1526 
1527 	/*
1528 	 * Some ATAPI commands do not actually follow the SCSI standard.
1529 	 */
1530 	cdbd = (void *)xa->packetcmd;
1531 
1532 	switch(cdbd->generic.opcode) {
1533 	case REQUEST_SENSE:
1534 		/*
1535 		 * Force SENSE requests to the ATAPI sense length.
1536 		 *
1537 		 * It is unclear if this is needed or not.
1538 		 */
1539 		if (cdbd->sense.length == SSD_FULL_SIZE) {
1540 			if (bootverbose) {
1541 				kprintf("%s: Shortening sense request\n",
1542 					PORTNAME(ap));
1543 			}
1544 			cdbd->sense.length = offsetof(struct scsi_sense_data,
1545 						      extra_bytes[0]);
1546 		}
1547 		break;
1548 	case INQUIRY:
1549 		/*
1550 		 * Some ATAPI devices can't handle long inquiry lengths,
1551 		 * don't ask me why.  Truncate the inquiry length.
1552 		 */
1553 		if (cdbd->inquiry.page_code == 0 &&
1554 		    cdbd->inquiry.length > SHORT_INQUIRY_LENGTH) {
1555 			cdbd->inquiry.length = SHORT_INQUIRY_LENGTH;
1556 		}
1557 		break;
1558 	case READ_6:
1559 	case WRITE_6:
1560 		/*
1561 		 * Convert *_6 to *_10 commands.  Most ATAPI devices
1562 		 * cannot handle the SCSI READ_6 and WRITE_6 commands.
1563 		 */
1564 		cdbd->rw_10.opcode |= 0x20;
1565 		cdbd->rw_10.byte2 = 0;
1566 		cdbd->rw_10.addr[0] = cdbs->rw_6.addr[0] & 0x1F;
1567 		cdbd->rw_10.addr[1] = cdbs->rw_6.addr[1];
1568 		cdbd->rw_10.addr[2] = cdbs->rw_6.addr[2];
1569 		cdbd->rw_10.addr[3] = 0;
1570 		cdbd->rw_10.reserved = 0;
1571 		cdbd->rw_10.length[0] = 0;
1572 		cdbd->rw_10.length[1] = cdbs->rw_6.length;
1573 		cdbd->rw_10.control = cdbs->rw_6.control;
1574 		break;
1575 	default:
1576 		break;
1577 	}
1578 
1579 	/*
1580 	 * And dispatch
1581 	 */
1582 	xa->complete = ahci_atapi_complete_cmd;
1583 	xa->atascsi_private = ccb;
1584 	ccb->ccb_h.sim_priv.entries[0].ptr = ap;
1585 	ahci_os_lock_port(ap);
1586 	ahci_ata_cmd(xa);
1587 	ahci_os_unlock_port(ap);
1588 }
1589 
1590 /*
1591  * Simulate page inquiries for disk attachments.
1592  */
1593 static
1594 void
1595 ahci_xpt_page_inquiry(struct ahci_port *ap, struct ata_port *at, union ccb *ccb)
1596 {
1597 	union {
1598 		struct scsi_vpd_supported_page_list	list;
1599 		struct scsi_vpd_unit_serial_number	serno;
1600 		struct scsi_vpd_unit_devid		devid;
1601 		char					buf[256];
1602 	} *page;
1603 	scsi_cdb_t cdb;
1604 	int i;
1605 	int j;
1606 	int len;
1607 
1608 	page = kmalloc(sizeof(*page), M_DEVBUF, M_WAITOK | M_ZERO);
1609 
1610 	cdb = (void *)((ccb->ccb_h.flags & CAM_CDB_POINTER) ?
1611 			ccb->csio.cdb_io.cdb_ptr : ccb->csio.cdb_io.cdb_bytes);
1612 
1613 	switch(cdb->inquiry.page_code) {
1614 	case SVPD_SUPPORTED_PAGE_LIST:
1615 		i = 0;
1616 		page->list.device = T_DIRECT;
1617 		page->list.page_code = SVPD_SUPPORTED_PAGE_LIST;
1618 		page->list.list[i++] = SVPD_SUPPORTED_PAGE_LIST;
1619 		page->list.list[i++] = SVPD_UNIT_SERIAL_NUMBER;
1620 		page->list.list[i++] = SVPD_UNIT_DEVID;
1621 		page->list.length = i;
1622 		len = offsetof(struct scsi_vpd_supported_page_list, list[3]);
1623 		break;
1624 	case SVPD_UNIT_SERIAL_NUMBER:
1625 		i = 0;
1626 		j = sizeof(at->at_identify.serial);
1627 		for (i = 0; i < j && at->at_identify.serial[i] == ' '; ++i)
1628 			;
1629 		while (j > i && at->at_identify.serial[j-1] == ' ')
1630 			--j;
1631 		page->serno.device = T_DIRECT;
1632 		page->serno.page_code = SVPD_UNIT_SERIAL_NUMBER;
1633 		page->serno.length = j - i;
1634 		bcopy(at->at_identify.serial + i,
1635 		      page->serno.serial_num, j - i);
1636 		len = offsetof(struct scsi_vpd_unit_serial_number,
1637 			       serial_num[j-i]);
1638 		break;
1639 	case SVPD_UNIT_DEVID:
1640 		/* fall through for now */
1641 	default:
1642 		ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
1643 		len = 0;
1644 		break;
1645 	}
1646 	if (ccb->ccb_h.status == CAM_REQ_INPROG) {
1647 		if (len <= ccb->csio.dxfer_len) {
1648 			ccb->ccb_h.status = CAM_REQ_CMP;
1649 			bzero(ccb->csio.data_ptr, ccb->csio.dxfer_len);
1650 			bcopy(page, ccb->csio.data_ptr, len);
1651 			ccb->csio.resid = ccb->csio.dxfer_len - len;
1652 		} else {
1653 			ccb->ccb_h.status = CAM_CCB_LEN_ERR;
1654 		}
1655 	}
1656 	kfree(page, M_DEVBUF);
1657 }
1658 
1659 /*
1660  * Completion function for ATA_PORT_T_DISK cache synchronization.
1661  */
1662 static
1663 void
1664 ahci_ata_complete_disk_synchronize_cache(struct ata_xfer *xa)
1665 {
1666 	union ccb *ccb = xa->atascsi_private;
1667 	struct ccb_hdr *ccbh = &ccb->ccb_h;
1668 	struct ahci_port *ap = ccb->ccb_h.sim_priv.entries[0].ptr;
1669 
1670 	switch(xa->state) {
1671 	case ATA_S_COMPLETE:
1672 		ccbh->status = CAM_REQ_CMP;
1673 		ccb->csio.scsi_status = SCSI_STATUS_OK;
1674 		break;
1675 	case ATA_S_ERROR:
1676 		kprintf("%s: synchronize_cache: error\n",
1677 			ATANAME(ap, xa->at));
1678 		ccbh->status = CAM_SCSI_STATUS_ERROR | CAM_AUTOSNS_VALID;
1679 		ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND;
1680 		ahci_ata_dummy_sense(&ccb->csio.sense_data);
1681 		break;
1682 	case ATA_S_TIMEOUT:
1683 		kprintf("%s: synchronize_cache: timeout\n",
1684 			ATANAME(ap, xa->at));
1685 		ccbh->status = CAM_CMD_TIMEOUT;
1686 		break;
1687 	default:
1688 		kprintf("%s: synchronize_cache: unknown state %d\n",
1689 			ATANAME(ap, xa->at), xa->state);
1690 		panic("%s: Unknown state", ATANAME(ap, xa->at));
1691 		ccbh->status = CAM_REQ_CMP_ERR;
1692 		break;
1693 	}
1694 	ahci_ata_put_xfer(xa);
1695 	/*ahci_os_unlock_port(ap); ILLEGAL SEE NOTE-1 AT TOP */
1696 	xpt_done(ccb);
1697 	/*ahci_os_lock_port(ap);*/
1698 }
1699 
1700 /*
1701  * Completion function for ATA_PORT_T_DISK I/O
1702  */
1703 static
1704 void
1705 ahci_ata_complete_disk_rw(struct ata_xfer *xa)
1706 {
1707 	union ccb *ccb = xa->atascsi_private;
1708 	struct ccb_hdr *ccbh = &ccb->ccb_h;
1709 	struct ahci_port *ap = ccb->ccb_h.sim_priv.entries[0].ptr;
1710 
1711 	switch(xa->state) {
1712 	case ATA_S_COMPLETE:
1713 		ccbh->status = CAM_REQ_CMP;
1714 		ccb->csio.scsi_status = SCSI_STATUS_OK;
1715 		break;
1716 	case ATA_S_ERROR:
1717 		kprintf("%s: disk_rw: error\n", ATANAME(ap, xa->at));
1718 		ccbh->status = CAM_SCSI_STATUS_ERROR | CAM_AUTOSNS_VALID;
1719 		ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND;
1720 		ahci_ata_dummy_sense(&ccb->csio.sense_data);
1721 		break;
1722 	case ATA_S_TIMEOUT:
1723 		kprintf("%s: disk_rw: timeout\n", ATANAME(ap, xa->at));
1724 		ccbh->status = CAM_CMD_TIMEOUT;
1725 		ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND;
1726 		ahci_ata_dummy_sense(&ccb->csio.sense_data);
1727 		break;
1728 	default:
1729 		kprintf("%s: disk_rw: unknown state %d\n",
1730 			ATANAME(ap, xa->at), xa->state);
1731 		panic("%s: Unknown state", ATANAME(ap, xa->at));
1732 		ccbh->status = CAM_REQ_CMP_ERR;
1733 		break;
1734 	}
1735 	ccb->csio.resid = xa->resid;
1736 	ahci_ata_put_xfer(xa);
1737 	/*ahci_os_unlock_port(ap); ILLEGAL SEE NOTE-1 AT TOP */
1738 	xpt_done(ccb);
1739 	/*ahci_os_lock_port(ap);*/
1740 }
1741 
1742 /*
1743  * Completion function for ATA_PORT_T_ATAPI I/O
1744  *
1745  * Sense data is returned in the rfis.
1746  */
1747 static
1748 void
1749 ahci_atapi_complete_cmd(struct ata_xfer *xa)
1750 {
1751 	union ccb *ccb = xa->atascsi_private;
1752 	struct ccb_hdr *ccbh = &ccb->ccb_h;
1753 	struct ahci_port *ap = ccb->ccb_h.sim_priv.entries[0].ptr;
1754 	scsi_cdb_t cdb;
1755 
1756 	cdb = (void *)((ccb->ccb_h.flags & CAM_CDB_POINTER) ?
1757 			ccb->csio.cdb_io.cdb_ptr : ccb->csio.cdb_io.cdb_bytes);
1758 
1759 	switch(xa->state) {
1760 	case ATA_S_COMPLETE:
1761 		ccbh->status = CAM_REQ_CMP;
1762 		ccb->csio.scsi_status = SCSI_STATUS_OK;
1763 		break;
1764 	case ATA_S_ERROR:
1765 		ccbh->status = CAM_SCSI_STATUS_ERROR;
1766 		ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND;
1767 		ahci_ata_atapi_sense(&xa->rfis, &ccb->csio.sense_data);
1768 		break;
1769 	case ATA_S_TIMEOUT:
1770 		kprintf("%s: cmd %d: timeout\n",
1771 			PORTNAME(ap), cdb->generic.opcode);
1772 		ccbh->status = CAM_CMD_TIMEOUT;
1773 		ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND;
1774 		ahci_ata_dummy_sense(&ccb->csio.sense_data);
1775 		break;
1776 	default:
1777 		kprintf("%s: cmd %d: unknown state %d\n",
1778 			PORTNAME(ap), cdb->generic.opcode, xa->state);
1779 		panic("%s: Unknown state", PORTNAME(ap));
1780 		ccbh->status = CAM_REQ_CMP_ERR;
1781 		break;
1782 	}
1783 	ccb->csio.resid = xa->resid;
1784 	xa->atascsi_private = NULL;
1785 	ahci_ata_put_xfer(xa);
1786 	/*ahci_os_unlock_port(ap); ILLEGAL SEE NOTE-1 AT TOP */
1787 	xpt_done(ccb);
1788 	/*ahci_os_lock_port(ap);*/
1789 }
1790 
1791 /*
1792  * Construct dummy sense data for errors on DISKs
1793  */
1794 static
1795 void
1796 ahci_ata_dummy_sense(struct scsi_sense_data *sense_data)
1797 {
1798 	sense_data->error_code = SSD_ERRCODE_VALID | SSD_CURRENT_ERROR;
1799 	sense_data->segment = 0;
1800 	sense_data->flags = SSD_KEY_MEDIUM_ERROR;
1801 	sense_data->info[0] = 0;
1802 	sense_data->info[1] = 0;
1803 	sense_data->info[2] = 0;
1804 	sense_data->info[3] = 0;
1805 	sense_data->extra_len = 0;
1806 }
1807 
1808 /*
1809  * Construct atapi sense data for errors on ATAPI
1810  *
1811  * The ATAPI sense data is stored in the passed rfis and must be converted
1812  * to SCSI sense data.
1813  */
1814 static
1815 void
1816 ahci_ata_atapi_sense(struct ata_fis_d2h *rfis,
1817 		     struct scsi_sense_data *sense_data)
1818 {
1819 	sense_data->error_code = SSD_ERRCODE_VALID | SSD_CURRENT_ERROR;
1820 	sense_data->segment = 0;
1821 	sense_data->flags = (rfis->error & 0xF0) >> 4;
1822 	if (rfis->error & 0x04)
1823 		sense_data->flags |= SSD_KEY_ILLEGAL_REQUEST;
1824 	if (rfis->error & 0x02)
1825 		sense_data->flags |= SSD_EOM;
1826 	if (rfis->error & 0x01)
1827 		sense_data->flags |= SSD_ILI;
1828 	sense_data->info[0] = 0;
1829 	sense_data->info[1] = 0;
1830 	sense_data->info[2] = 0;
1831 	sense_data->info[3] = 0;
1832 	sense_data->extra_len = 0;
1833 }
1834 
1835 static
1836 void
1837 ahci_strip_string(const char **basep, int *lenp)
1838 {
1839 	const char *base = *basep;
1840 	int len = *lenp;
1841 
1842 	while (len && (*base == 0 || *base == ' ')) {
1843 		--len;
1844 		++base;
1845 	}
1846 	while (len && (base[len-1] == 0 || base[len-1] == ' '))
1847 		--len;
1848 	*basep = base;
1849 	*lenp = len;
1850 }
1851