xref: /openbsd/sys/dev/ic/wdc.c (revision 8529ddd3)
1 /*	$OpenBSD: wdc.c,v 1.128 2015/03/09 18:50:08 miod Exp $	*/
2 /*	$NetBSD: wdc.c,v 1.68 1999/06/23 19:00:17 bouyer Exp $	*/
3 /*
4  * Copyright (c) 1998, 2001 Manuel Bouyer.  All rights reserved.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25  */
26 
27 /*-
28  * Copyright (c) 1998 The NetBSD Foundation, Inc.
29  * All rights reserved.
30  *
31  * This code is derived from software contributed to The NetBSD Foundation
32  * by Charles M. Hannum, by Onno van der Linden and by Manuel Bouyer.
33  *
34  * Redistribution and use in source and binary forms, with or without
35  * modification, are permitted provided that the following conditions
36  * are met:
37  * 1. Redistributions of source code must retain the above copyright
38  *    notice, this list of conditions and the following disclaimer.
39  * 2. Redistributions in binary form must reproduce the above copyright
40  *    notice, this list of conditions and the following disclaimer in the
41  *    documentation and/or other materials provided with the distribution.
42  *
43  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
44  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
45  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
46  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
47  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
48  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
49  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
50  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
51  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
52  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
53  * POSSIBILITY OF SUCH DAMAGE.
54  */
55 
56 #include <sys/param.h>
57 #include <sys/systm.h>
58 #include <sys/kernel.h>
59 #include <sys/conf.h>
60 #include <sys/buf.h>
61 #include <sys/device.h>
62 #include <sys/malloc.h>
63 #include <sys/syslog.h>
64 #include <sys/disk.h>
65 #include <sys/pool.h>
66 
67 #include <machine/intr.h>
68 #include <machine/bus.h>
69 
70 #include <dev/ata/atavar.h>
71 #include <dev/ata/atareg.h>
72 #include <dev/ic/wdcreg.h>
73 #include <dev/ic/wdcvar.h>
74 #include <dev/ic/wdcevent.h>
75 
76 #include <scsi/scsi_all.h>
77 #include <scsi/scsiconf.h>
78 
79 #define WDCDELAY  100 /* 100 microseconds */
80 #define WDCNDELAY_RST (WDC_RESET_WAIT * 1000 / WDCDELAY)
81 #if 0
82 /* If you enable this, it will report any delays more than WDCDELAY * N long. */
83 #define WDCNDELAY_DEBUG	50
84 #endif /* 0 */
85 
86 struct pool wdc_xfer_pool;
87 struct scsi_iopool wdc_xfer_iopool;
88 
89 void *	wdc_xfer_get(void *);
90 void	wdc_xfer_put(void *, void *);
91 
92 void  __wdcerror(struct channel_softc *, char *);
93 int   __wdcwait_reset(struct channel_softc *, int);
94 void  __wdccommand_done(struct channel_softc *, struct wdc_xfer *);
95 void  __wdccommand_start(struct channel_softc *, struct wdc_xfer *);
96 int   __wdccommand_intr(struct channel_softc *, struct wdc_xfer *, int);
97 int   wdprint(void *, const char *);
98 void  wdc_kill_pending(struct channel_softc *);
99 
100 #define DEBUG_INTR    0x01
101 #define DEBUG_XFERS   0x02
102 #define DEBUG_STATUS  0x04
103 #define DEBUG_FUNCS   0x08
104 #define DEBUG_PROBE   0x10
105 #define DEBUG_STATUSX 0x20
106 #define DEBUG_SDRIVE  0x40
107 #define DEBUG_DETACH  0x80
108 
109 #ifdef WDCDEBUG
110 #ifndef WDCDEBUG_MASK
111 #define WDCDEBUG_MASK 0x00
112 #endif
113 int wdcdebug_mask = WDCDEBUG_MASK;
114 int wdc_nxfer = 0;
115 #define WDCDEBUG_PRINT(args, level) do {	\
116 	if ((wdcdebug_mask & (level)) != 0)	\
117 		printf args;			\
118 } while (0)
119 #else
120 #define WDCDEBUG_PRINT(args, level)
121 #endif /* WDCDEBUG */
122 
123 int at_poll = AT_POLL;
124 
125 int wdc_floating_bus(struct channel_softc *, int);
126 int wdc_preata_drive(struct channel_softc *, int);
127 int wdc_ata_present(struct channel_softc *, int);
128 
129 struct cfdriver wdc_cd = {
130 	NULL, "wdc", DV_DULL
131 };
132 
133 struct channel_softc_vtbl wdc_default_vtbl = {
134 	wdc_default_read_reg,
135 	wdc_default_write_reg,
136 	wdc_default_lba48_write_reg,
137 	wdc_default_read_raw_multi_2,
138 	wdc_default_write_raw_multi_2,
139 	wdc_default_read_raw_multi_4,
140 	wdc_default_write_raw_multi_4
141 };
142 
143 #ifdef WDCDEBUG
144 static char *wdc_log_buf = NULL;
145 static unsigned int wdc_tail = 0;
146 static unsigned int wdc_head = 0;
147 static unsigned int wdc_log_cap = 16 * 1024;
148 static int chp_idx = 1;
149 
150 void
151 wdc_log(struct channel_softc *chp, enum wdcevent_type type,
152     unsigned int size, char val[])
153 {
154 	unsigned int request_size;
155 	char *ptr;
156 	int log_size;
157 	unsigned int head = wdc_head;
158 	unsigned int tail = wdc_tail;
159 
160 #ifdef DIAGNOSTIC
161 	if (head < 0 || head > wdc_log_cap ||
162 	    tail < 0 || tail > wdc_log_cap) {
163 		printf ("wdc_log: head %x wdc_tail %x\n", head,
164 		    tail);
165 		return;
166 	}
167 
168 	if (size > wdc_log_cap / 2) {
169 		printf ("wdc_log: type %d size %x\n", type, size);
170 		return;
171 	}
172 #endif
173 
174 	if (wdc_log_buf == NULL) {
175 		wdc_log_buf = malloc(wdc_log_cap, M_DEVBUF, M_NOWAIT);
176 		if (wdc_log_buf == NULL)
177 			return;
178 	}
179 	if (chp->ch_log_idx == 0)
180 		chp->ch_log_idx = chp_idx++;
181 
182 	request_size = size + 2;
183 
184 	/* Check how many bytes are left */
185 	log_size = head - tail;
186 	if (log_size < 0) log_size += wdc_log_cap;
187 
188 	if (log_size + request_size >= wdc_log_cap) {
189 		int nb = 0;
190 		int rec_size;
191 
192 		while (nb <= (request_size * 2)) {
193 			if (wdc_log_buf[tail] == 0)
194 				rec_size = 1;
195 			else
196 				rec_size = (wdc_log_buf[tail + 1] & 0x1f) + 2;
197 			tail = (tail + rec_size) % wdc_log_cap;
198 			nb += rec_size;
199 		}
200 	}
201 
202 	/* Avoid wrapping in the middle of a request */
203 	if (head + request_size >= wdc_log_cap) {
204 		memset(&wdc_log_buf[head], 0, wdc_log_cap - head);
205 		head = 0;
206 	}
207 
208 	ptr = &wdc_log_buf[head];
209 	*ptr++ = type & 0xff;
210 	*ptr++ = ((chp->ch_log_idx & 0x7) << 5) | (size & 0x1f);
211 	memcpy(ptr, val, size);
212 
213 	wdc_head = (head + request_size) % wdc_log_cap;
214 	wdc_tail = tail;
215 }
216 
217 char *wdc_get_log(unsigned int *, unsigned int *);
218 
219 char *
220 wdc_get_log(unsigned int * size, unsigned int *left)
221 {
222 	int  log_size;
223 	char *retbuf = NULL;
224 	int  nb, tocopy;
225 	int  s;
226 	unsigned int head = wdc_head;
227 	unsigned int tail = wdc_tail;
228 
229 	s = splbio();
230 
231 	log_size = (head - tail);
232 	if (left != NULL)
233 		*left = 0;
234 
235 	if (log_size < 0)
236 		log_size += wdc_log_cap;
237 
238 	tocopy = log_size;
239 	if ((u_int)tocopy > *size)
240 		tocopy = *size;
241 
242 	if (wdc_log_buf == NULL) {
243 		*size = 0;
244 		*left = 0;
245 		goto out;
246 	}
247 
248 #ifdef DIAGNOSTIC
249 	if (head < 0 || head > wdc_log_cap ||
250 	    tail < 0 || tail > wdc_log_cap) {
251 		printf ("wdc_log: head %x tail %x\n", head,
252 		    tail);
253 		*size = 0;
254 		*left = 0;
255 		goto out;
256 	}
257 #endif
258 
259 	retbuf = malloc(tocopy, M_TEMP, M_NOWAIT);
260 	if (retbuf == NULL) {
261 		*size = 0;
262 		*left = log_size;
263 		goto out;
264 	}
265 
266 	nb = 0;
267 	for (;;) {
268 		int rec_size;
269 
270 		if (wdc_log_buf[tail] == 0)
271 			rec_size = 1;
272 		else
273 			rec_size = (wdc_log_buf[tail + 1] & 0x1f) + 2;
274 
275 		if ((nb + rec_size) >= tocopy)
276 			break;
277 
278 		memcpy(&retbuf[nb], &wdc_log_buf[tail], rec_size);
279 		tail = (tail + rec_size) % wdc_log_cap;
280 		nb += rec_size;
281 	}
282 
283 	wdc_tail = tail;
284 	*size = nb;
285 	*left = log_size - nb;
286 
287  out:
288 	splx(s);
289 	return (retbuf);
290 }
291 #endif /* WDCDEBUG */
292 
293 u_int8_t
294 wdc_default_read_reg(struct channel_softc *chp, enum wdc_regs reg)
295 {
296 #ifdef DIAGNOSTIC
297 	if (reg & _WDC_WRONLY) {
298 		printf ("wdc_default_read_reg: reading from a write-only register %d\n", reg);
299 	}
300 #endif /* DIAGNOSTIC */
301 
302 	if (reg & _WDC_AUX)
303 		return (bus_space_read_1(chp->ctl_iot, chp->ctl_ioh,
304 		    reg & _WDC_REGMASK));
305 	else
306 		return (bus_space_read_1(chp->cmd_iot, chp->cmd_ioh,
307 		    reg & _WDC_REGMASK));
308 }
309 
310 void
311 wdc_default_write_reg(struct channel_softc *chp, enum wdc_regs reg, u_int8_t val)
312 {
313 #ifdef DIAGNOSTIC
314 	if (reg & _WDC_RDONLY) {
315 		printf ("wdc_default_write_reg: writing to a read-only register %d\n", reg);
316 	}
317 #endif /* DIAGNOSTIC */
318 
319 	if (reg & _WDC_AUX)
320 		bus_space_write_1(chp->ctl_iot, chp->ctl_ioh,
321 		    reg & _WDC_REGMASK, val);
322 	else
323 		bus_space_write_1(chp->cmd_iot, chp->cmd_ioh,
324 		    reg & _WDC_REGMASK, val);
325 }
326 
327 void
328 wdc_default_lba48_write_reg(struct channel_softc *chp, enum wdc_regs reg,
329     u_int16_t val)
330 {
331 	/* All registers are two byte deep FIFOs. */
332 	CHP_WRITE_REG(chp, reg, val >> 8);
333 	CHP_WRITE_REG(chp, reg, val);
334 }
335 
336 void
337 wdc_default_read_raw_multi_2(struct channel_softc *chp, void *data,
338     unsigned int nbytes)
339 {
340 	if (data == NULL) {
341 		unsigned int i;
342 
343 		for (i = 0; i < nbytes; i += 2) {
344 			bus_space_read_2(chp->cmd_iot, chp->cmd_ioh, 0);
345 		}
346 
347 		return;
348 	}
349 
350 	bus_space_read_raw_multi_2(chp->cmd_iot, chp->cmd_ioh, 0,
351 	    data, nbytes);
352 }
353 
354 
355 void
356 wdc_default_write_raw_multi_2(struct channel_softc *chp, void *data,
357     unsigned int nbytes)
358 {
359 	if (data == NULL) {
360 		unsigned int i;
361 
362 		for (i = 0; i < nbytes; i += 2) {
363 			bus_space_write_2(chp->cmd_iot, chp->cmd_ioh, 0, 0);
364 		}
365 
366 		return;
367 	}
368 
369 	bus_space_write_raw_multi_2(chp->cmd_iot, chp->cmd_ioh, 0,
370 	    data, nbytes);
371 }
372 
373 
374 void
375 wdc_default_write_raw_multi_4(struct channel_softc *chp, void *data,
376     unsigned int nbytes)
377 {
378 	if (data == NULL) {
379 		unsigned int i;
380 
381 		for (i = 0; i < nbytes; i += 4) {
382 			bus_space_write_4(chp->cmd_iot, chp->cmd_ioh, 0, 0);
383 		}
384 
385 		return;
386 	}
387 
388 	bus_space_write_raw_multi_4(chp->cmd_iot, chp->cmd_ioh, 0,
389 	    data, nbytes);
390 }
391 
392 
393 void
394 wdc_default_read_raw_multi_4(struct channel_softc *chp, void *data,
395     unsigned int nbytes)
396 {
397 	if (data == NULL) {
398 		unsigned int i;
399 
400 		for (i = 0; i < nbytes; i += 4) {
401 			bus_space_read_4(chp->cmd_iot, chp->cmd_ioh, 0);
402 		}
403 
404 		return;
405 	}
406 
407 	bus_space_read_raw_multi_4(chp->cmd_iot, chp->cmd_ioh, 0,
408 	    data, nbytes);
409 }
410 
411 int
412 wdprint(void *aux, const char *pnp)
413 {
414 	struct ata_atapi_attach *aa_link = aux;
415 	if (pnp)
416 		printf("drive at %s", pnp);
417 	printf(" channel %d drive %d", aa_link->aa_channel,
418 	    aa_link->aa_drv_data->drive);
419 	return (UNCONF);
420 }
421 
422 void
423 wdc_disable_intr(struct channel_softc *chp)
424 {
425 	CHP_WRITE_REG(chp, wdr_ctlr, WDCTL_IDS);
426 }
427 
428 void
429 wdc_enable_intr(struct channel_softc *chp)
430 {
431 	CHP_WRITE_REG(chp, wdr_ctlr, WDCTL_4BIT);
432 }
433 
434 void
435 wdc_set_drive(struct channel_softc *chp, int drive)
436 {
437 	CHP_WRITE_REG(chp, wdr_sdh, (drive << 4) | WDSD_IBM);
438 	WDC_LOG_SET_DRIVE(chp, drive);
439 }
440 
441 int
442 wdc_floating_bus(struct channel_softc *chp, int drive)
443 {
444 	u_int8_t cumulative_status, status;
445 	int      iter;
446 
447 	wdc_set_drive(chp, drive);
448 	delay(10);
449 
450 	/* Stolen from Phoenix BIOS Drive Autotyping document */
451 	cumulative_status = 0;
452 	for (iter = 0; iter < 100; iter++) {
453 		CHP_WRITE_REG(chp, wdr_seccnt, 0x7f);
454 		delay (1);
455 
456 		status = CHP_READ_REG(chp, wdr_status);
457 
458 		/* The other bits are meaningless if BSY is set */
459 		if (status & WDCS_BSY)
460 			continue;
461 
462 		cumulative_status |= status;
463 
464 #define BAD_BIT_COMBO  (WDCS_DRDY | WDCS_DSC | WDCS_DRQ | WDCS_ERR)
465 		if ((cumulative_status & BAD_BIT_COMBO) == BAD_BIT_COMBO)
466 			return 1;
467 	}
468 
469 
470 	return 0;
471 }
472 
473 int
474 wdc_preata_drive(struct channel_softc *chp, int drive)
475 {
476 	if (wdc_floating_bus(chp, drive)) {
477 		WDCDEBUG_PRINT(("%s:%d:%d: floating bus detected\n",
478 		    chp->wdc->sc_dev.dv_xname,
479 		    chp->channel, drive), DEBUG_PROBE);
480 		return 0;
481 	}
482 
483 	wdc_set_drive(chp, drive);
484 	delay(100);
485 	if (wdcwait(chp, WDCS_DRDY | WDCS_DRQ, WDCS_DRDY, 10000) != 0) {
486 		WDCDEBUG_PRINT(("%s:%d:%d: not ready\n",
487 		    chp->wdc->sc_dev.dv_xname,
488 		    chp->channel, drive), DEBUG_PROBE);
489 		return 0;
490 	}
491 
492 	CHP_WRITE_REG(chp, wdr_command, WDCC_RECAL);
493 	WDC_LOG_ATA_CMDSHORT(chp, WDCC_RECAL);
494 	if (wdcwait(chp, WDCS_DRDY | WDCS_DRQ, WDCS_DRDY, 10000) != 0) {
495 		WDCDEBUG_PRINT(("%s:%d:%d: WDCC_RECAL failed\n",
496 		    chp->wdc->sc_dev.dv_xname,
497 		    chp->channel, drive), DEBUG_PROBE);
498 		return 0;
499 	}
500 
501 	return 1;
502 }
503 
504 int
505 wdc_ata_present(struct channel_softc *chp, int drive)
506 {
507 	int time_to_done;
508 	int retry_cnt = 0;
509 
510 	wdc_set_drive(chp, drive);
511 	delay(10);
512 
513 retry:
514 	/*
515 	   You're actually supposed to wait up to 10 seconds
516 	   for DRDY. However, as a practical matter, most
517 	   drives assert DRDY very quickly after dropping BSY.
518 
519 	   The 10 seconds wait is sub-optimal because, according
520 	   to the ATA standard, the master should reply with 00
521 	   for any reads to a non-existent slave.
522 	*/
523 	time_to_done = wdc_wait_for_status(chp,
524 	    (WDCS_DRDY | WDCS_DSC | WDCS_DRQ),
525 	    (WDCS_DRDY | WDCS_DSC), 1000);
526 	if (time_to_done == -1) {
527 		if (retry_cnt == 0 && chp->ch_status == 0x00) {
528 			/* At least one flash card needs to be kicked */
529 			wdccommandshort(chp, drive, WDCC_CHECK_PWR);
530 			retry_cnt++;
531 			goto retry;
532 		}
533 		WDCDEBUG_PRINT(("%s:%d:%d: DRDY test timed out with status"
534 		    " %02x\n",
535 		    chp->wdc ? chp->wdc->sc_dev.dv_xname : "wdcprobe",
536 		    chp->channel, drive, chp->ch_status),
537 		    DEBUG_PROBE);
538 		return 0;
539 	}
540 
541 	if ((chp->ch_status & 0xfc) != (WDCS_DRDY | WDCS_DSC)) {
542 		WDCDEBUG_PRINT(("%s:%d:%d: status test for 0x50 failed with"
543 		    " %02x\n",
544 		    chp->wdc ? chp->wdc->sc_dev.dv_xname : "wdcprobe",
545 		    chp->channel, drive, chp->ch_status),
546 		    DEBUG_PROBE);
547 
548 		return 0;
549 	}
550 
551 	WDCDEBUG_PRINT(("%s:%d:%d: waiting for ready %d msec\n",
552 	    chp->wdc ? chp->wdc->sc_dev.dv_xname : "wdcprobe",
553 	    chp->channel, drive, time_to_done), DEBUG_PROBE);
554 
555 	/*
556 	 * Test register writability
557 	 */
558 	CHP_WRITE_REG(chp, wdr_cyl_lo, 0xaa);
559 	CHP_WRITE_REG(chp, wdr_cyl_hi, 0x55);
560 	CHP_WRITE_REG(chp, wdr_seccnt, 0xff);
561 	DELAY(10);
562 
563 	if (CHP_READ_REG(chp, wdr_cyl_lo) != 0xaa &&
564 	    CHP_READ_REG(chp, wdr_cyl_hi) != 0x55) {
565 		WDCDEBUG_PRINT(("%s:%d:%d: register writability failed\n",
566 		    chp->wdc ? chp->wdc->sc_dev.dv_xname : "wdcprobe",
567 		    chp->channel, drive), DEBUG_PROBE);
568 		return 0;
569 	}
570 
571 	return 1;
572 }
573 
574 
575 /*
576  * Test to see controller with at least one attached drive is there.
577  * Returns a bit for each possible drive found (0x01 for drive 0,
578  * 0x02 for drive 1).
579  * Logic:
580  * - If a status register is at 0x7f or 0xff, assume there is no drive here
581  *   (ISA has pull-up resistors).  Similarly if the status register has
582  *   the value we last wrote to the bus (for IDE interfaces without pullups).
583  *   If no drive at all -> return.
584  * - reset the controller, wait for it to complete (may take up to 31s !).
585  *   If timeout -> return.
586  * - test ATA/ATAPI signatures. If at last one drive found -> return.
587  * - try an ATA command on the master.
588  */
589 
590 int
591 wdcprobe(struct channel_softc *chp)
592 {
593 	u_int8_t st0, st1, sc, sn, cl, ch;
594 	u_int8_t ret_value = 0x03;
595 	u_int8_t drive;
596 #ifdef WDCDEBUG
597 	int savedmask = wdcdebug_mask;
598 #endif
599 
600 	if (chp->_vtbl == 0) {
601 		int s = splbio();
602 		chp->_vtbl = &wdc_default_vtbl;
603 		splx(s);
604 	}
605 
606 #ifdef WDCDEBUG
607 	if ((chp->ch_flags & WDCF_VERBOSE_PROBE) ||
608 	    (chp->wdc &&
609 	    (chp->wdc->sc_dev.dv_cfdata->cf_flags & WDC_OPTION_PROBE_VERBOSE)))
610 		wdcdebug_mask |= DEBUG_PROBE;
611 #endif /* WDCDEBUG */
612 
613 	if (chp->wdc == NULL ||
614 	    (chp->wdc->cap & WDC_CAPABILITY_NO_EXTRA_RESETS) == 0) {
615 		/* Sample the statuses of drive 0 and 1 into st0 and st1 */
616 		wdc_set_drive(chp, 0);
617 		delay(10);
618 		st0 = CHP_READ_REG(chp, wdr_status);
619 		WDC_LOG_STATUS(chp, st0);
620 		wdc_set_drive(chp, 1);
621 		delay(10);
622 		st1 = CHP_READ_REG(chp, wdr_status);
623 		WDC_LOG_STATUS(chp, st1);
624 
625 		WDCDEBUG_PRINT(("%s:%d: before reset, st0=0x%b, st1=0x%b\n",
626 		    chp->wdc ? chp->wdc->sc_dev.dv_xname : "wdcprobe",
627 		    chp->channel, st0, WDCS_BITS, st1, WDCS_BITS),
628 		    DEBUG_PROBE);
629 
630 		if (st0 == 0xff || st0 == WDSD_IBM)
631 			ret_value &= ~0x01;
632 		if (st1 == 0xff || st1 == (WDSD_IBM | 0x10))
633 			ret_value &= ~0x02;
634 		if (ret_value == 0)
635 			return 0;
636 	}
637 
638 	/* reset the channel */
639 	wdc_do_reset(chp);
640 
641 	ret_value = __wdcwait_reset(chp, ret_value);
642 	WDCDEBUG_PRINT(("%s:%d: after reset, ret_value=0x%d\n",
643 	    chp->wdc ? chp->wdc->sc_dev.dv_xname : "wdcprobe", chp->channel,
644 	    ret_value), DEBUG_PROBE);
645 
646 	if (ret_value == 0)
647 		return 0;
648 
649 	/*
650 	 * Use signatures to find potential ATAPI drives
651 	 */
652 	for (drive = 0; drive < 2; drive++) {
653  		if ((ret_value & (0x01 << drive)) == 0)
654 			continue;
655 		wdc_set_drive(chp, drive);
656 		delay(10);
657 		/* Save registers contents */
658 		st0 = CHP_READ_REG(chp, wdr_status);
659 		sc = CHP_READ_REG(chp, wdr_seccnt);
660 		sn = CHP_READ_REG(chp, wdr_sector);
661 		cl = CHP_READ_REG(chp, wdr_cyl_lo);
662 		ch = CHP_READ_REG(chp, wdr_cyl_hi);
663 		WDC_LOG_REG(chp, wdr_cyl_lo, (ch << 8) | cl);
664 
665 		WDCDEBUG_PRINT(("%s:%d:%d: after reset, st=0x%b, sc=0x%x"
666 		    " sn=0x%x cl=0x%x ch=0x%x\n",
667 		    chp->wdc ? chp->wdc->sc_dev.dv_xname : "wdcprobe",
668 		    chp->channel, drive, st0, WDCS_BITS, sc, sn, cl, ch),
669 		    DEBUG_PROBE);
670 		/*
671 		 * This is a simplification of the test in the ATAPI
672 		 * spec since not all drives seem to set the other regs
673 		 * correctly.
674 		 */
675 		if (cl == 0x14 && ch == 0xeb)
676 			chp->ch_drive[drive].drive_flags |= DRIVE_ATAPI;
677 	}
678 
679 	/*
680 	 * Detect ATA drives by poking around the registers
681 	 */
682 	for (drive = 0; drive < 2; drive++) {
683  		if ((ret_value & (0x01 << drive)) == 0)
684 			continue;
685 		if (chp->ch_drive[drive].drive_flags & DRIVE_ATAPI)
686 			continue;
687 
688 		wdc_disable_intr(chp);
689 		/* ATA detect */
690 		if (wdc_ata_present(chp, drive)) {
691 			chp->ch_drive[drive].drive_flags |= DRIVE_ATA;
692 			if (chp->wdc == NULL ||
693 			    (chp->wdc->cap & WDC_CAPABILITY_PREATA) != 0)
694 				chp->ch_drive[drive].drive_flags |= DRIVE_OLD;
695 		} else {
696 			ret_value &= ~(1 << drive);
697 		}
698 		wdc_enable_intr(chp);
699 	}
700 
701 #ifdef WDCDEBUG
702 	wdcdebug_mask = savedmask;
703 #endif
704 	return (ret_value);
705 }
706 
707 struct channel_queue *
708 wdc_alloc_queue(void)
709 {
710 	static int inited = 0;
711 	struct channel_queue *queue;
712 
713 	/* Initialize global data. */
714 	if (inited == 0) {
715 		/* Initialize the wdc_xfer pool. */
716 		pool_init(&wdc_xfer_pool, sizeof(struct wdc_xfer), 0,
717 		    0, 0, "wdcxfer", NULL);
718 		pool_setipl(&wdc_xfer_pool, IPL_BIO);
719 		scsi_iopool_init(&wdc_xfer_iopool, NULL,
720 		    wdc_xfer_get, wdc_xfer_put);
721 		inited = 1;
722 	}
723 
724 	queue = malloc(sizeof(*queue), M_DEVBUF, M_NOWAIT);
725 	if (queue != NULL) {
726 		TAILQ_INIT(&queue->sc_xfer);
727 	}
728 	return (queue);
729 }
730 
731 void
732 wdc_free_queue(struct channel_queue *queue)
733 {
734 	free(queue, M_DEVBUF, 0);
735 }
736 
737 void
738 wdcattach(struct channel_softc *chp)
739 {
740 	int i;
741 	struct ata_atapi_attach aa_link;
742 #ifdef WDCDEBUG
743 	int    savedmask = wdcdebug_mask;
744 #endif
745 
746 	if (!cold)
747 		at_poll = AT_WAIT;
748 
749 	if (chp->wdc->reset == NULL)
750 		chp->wdc->reset = wdc_do_reset;
751 
752 	timeout_set(&chp->ch_timo, wdctimeout, chp);
753 
754 	if (!chp->_vtbl)
755 		chp->_vtbl = &wdc_default_vtbl;
756 
757 	for (i = 0; i < 2; i++) {
758 		chp->ch_drive[i].chnl_softc = chp;
759 		chp->ch_drive[i].drive = i;
760 	}
761 
762 	if (chp->wdc->drv_probe != NULL) {
763 		chp->wdc->drv_probe(chp);
764 	} else {
765 		if (wdcprobe(chp) == 0)
766 			/* If no drives, abort attach here. */
767 			return;
768 	}
769 
770 	/* ATAPI drives need settling time. Give them 250ms */
771 	if ((chp->ch_drive[0].drive_flags & DRIVE_ATAPI) ||
772 	    (chp->ch_drive[1].drive_flags & DRIVE_ATAPI)) {
773 		delay(250 * 1000);
774 	}
775 
776 #ifdef WDCDEBUG
777 	if (chp->wdc->sc_dev.dv_cfdata->cf_flags & WDC_OPTION_PROBE_VERBOSE)
778 		wdcdebug_mask |= DEBUG_PROBE;
779 
780 	if ((chp->ch_drive[0].drive_flags & DRIVE_ATAPI) ||
781 	    (chp->ch_drive[1].drive_flags & DRIVE_ATAPI)) {
782 		wdcdebug_mask = DEBUG_PROBE;
783 	}
784 #endif /* WDCDEBUG */
785 
786 	for (i = 0; i < 2; i++) {
787 		struct ata_drive_datas *drvp = &chp->ch_drive[i];
788 
789 		/* If controller can't do 16bit flag the drives as 32bit */
790 		if ((chp->wdc->cap &
791 		    (WDC_CAPABILITY_DATA16 | WDC_CAPABILITY_DATA32)) ==
792 		    WDC_CAPABILITY_DATA32)
793 			drvp->drive_flags |= DRIVE_CAP32;
794 
795 		if ((drvp->drive_flags & DRIVE) == 0)
796 			continue;
797 
798 		if (i == 1 && ((chp->ch_drive[0].drive_flags & DRIVE) == 0))
799 			chp->ch_flags |= WDCF_ONESLAVE;
800 		/*
801 		 * Wait a bit, some devices are weird just after a reset.
802 		 * Then issue a IDENTIFY command, to try to detect slave ghost.
803 		 */
804 		delay(5000);
805 		if (ata_get_params(&chp->ch_drive[i], at_poll, &drvp->id) ==
806 		    CMD_OK) {
807 			/* If IDENTIFY succeeded, this is not an OLD ctrl */
808 			drvp->drive_flags &= ~DRIVE_OLD;
809 		} else {
810 			bzero(&drvp->id, sizeof(struct ataparams));
811 			drvp->drive_flags &=
812 			    ~(DRIVE_ATA | DRIVE_ATAPI);
813 			WDCDEBUG_PRINT(("%s:%d:%d: IDENTIFY failed\n",
814 			    chp->wdc->sc_dev.dv_xname,
815 			    chp->channel, i), DEBUG_PROBE);
816 
817 			if ((drvp->drive_flags & DRIVE_OLD) &&
818 			    !wdc_preata_drive(chp, i))
819 				drvp->drive_flags &= ~DRIVE_OLD;
820 		}
821 	}
822 
823 	WDCDEBUG_PRINT(("wdcattach: ch_drive_flags 0x%x 0x%x\n",
824 	    chp->ch_drive[0].drive_flags, chp->ch_drive[1].drive_flags),
825 	    DEBUG_PROBE);
826 
827 	/* If no drives, abort here */
828 	if ((chp->ch_drive[0].drive_flags & DRIVE) == 0 &&
829 	    (chp->ch_drive[1].drive_flags & DRIVE) == 0)
830 		goto exit;
831 
832 	for (i = 0; i < 2; i++) {
833 		if ((chp->ch_drive[i].drive_flags & DRIVE) == 0) {
834 			continue;
835 		}
836 		bzero(&aa_link, sizeof(struct ata_atapi_attach));
837 		if (chp->ch_drive[i].drive_flags & DRIVE_ATAPI)
838 			aa_link.aa_type = T_ATAPI;
839 		else
840 			aa_link.aa_type = T_ATA;
841 		aa_link.aa_channel = chp->channel;
842 		aa_link.aa_openings = 1;
843 		aa_link.aa_drv_data = &chp->ch_drive[i];
844 		config_found(&chp->wdc->sc_dev, (void *)&aa_link, wdprint);
845 	}
846 
847 	/*
848 	 * reset drive_flags for unattached devices, reset state for attached
849 	 *  ones
850 	 */
851 	for (i = 0; i < 2; i++) {
852 		if (chp->ch_drive[i].drive_name[0] == 0)
853 			chp->ch_drive[i].drive_flags = 0;
854 	}
855 
856 exit:
857 #ifdef WDCDEBUG
858 	wdcdebug_mask = savedmask;
859 #endif
860 	return;	/* for the ``exit'' label above */
861 }
862 
863 /*
864  * Start I/O on a controller, for the given channel.
865  * The first xfer may be not for our channel if the channel queues
866  * are shared.
867  */
868 void
869 wdcstart(struct channel_softc *chp)
870 {
871 	struct wdc_xfer *xfer;
872 
873 	splassert(IPL_BIO);
874 
875 	/* is there a xfer ? */
876 	if ((xfer = TAILQ_FIRST(&chp->ch_queue->sc_xfer)) == NULL) {
877 		return;
878 	}
879 
880 	/* adjust chp, in case we have a shared queue */
881 	chp = xfer->chp;
882 
883 	if ((chp->ch_flags & WDCF_ACTIVE) != 0 ) {
884 		return; /* channel already active */
885 	}
886 #ifdef DIAGNOSTIC
887 	if ((chp->ch_flags & WDCF_IRQ_WAIT) != 0)
888 		panic("wdcstart: channel waiting for irq");
889 #endif /* DIAGNOSTIC */
890 
891 	WDCDEBUG_PRINT(("wdcstart: xfer %p channel %d drive %d\n", xfer,
892 	    chp->channel, xfer->drive), DEBUG_XFERS);
893 	chp->ch_flags |= WDCF_ACTIVE;
894 	if (chp->ch_drive[xfer->drive].drive_flags & DRIVE_RESET) {
895 		chp->ch_drive[xfer->drive].drive_flags &= ~DRIVE_RESET;
896 		chp->ch_drive[xfer->drive].state = 0;
897 	}
898 	xfer->c_start(chp, xfer);
899 }
900 
901 int
902 wdcdetach(struct channel_softc *chp, int flags)
903 {
904 	int s, rv;
905 
906 	s = splbio();
907 	chp->dying = 1;
908 
909 	wdc_kill_pending(chp);
910 	timeout_del(&chp->ch_timo);
911 
912 	rv = config_detach_children((struct device *)chp->wdc, flags);
913 	splx(s);
914 
915 	return (rv);
916 }
917 
918 /*
919  * Interrupt routine for the controller.  Acknowledge the interrupt, check for
920  * errors on the current operation, mark it done if necessary, and start the
921  * next request.  Also check for a partially done transfer, and continue with
922  * the next chunk if so.
923  */
924 int
925 wdcintr(void *arg)
926 {
927 	struct channel_softc *chp = arg;
928 	struct wdc_xfer *xfer;
929 	u_int8_t st = 0;
930 	int ret = 0;
931 
932 	if ((chp->ch_flags & WDCF_IRQ_WAIT) == 0) {
933 		/* Acknowledge interrupt by reading status */
934 		if (chp->_vtbl == 0)
935 			st = bus_space_read_1(chp->cmd_iot, chp->cmd_ioh,
936 			    wdr_status & _WDC_REGMASK);
937 		else
938 			st = CHP_READ_REG(chp, wdr_status);
939 		if (st == 0xff)
940 			return (-1);
941 
942 		WDCDEBUG_PRINT(("wdcintr: inactive controller\n"), DEBUG_INTR);
943 		return ret;
944 	}
945 
946 	WDCDEBUG_PRINT(("wdcintr\n"), DEBUG_INTR);
947 	xfer = TAILQ_FIRST(&chp->ch_queue->sc_xfer);
948 	if (chp->ch_flags & WDCF_DMA_WAIT) {
949 		chp->wdc->dma_status =
950 		    (*chp->wdc->dma_finish)(chp->wdc->dma_arg, chp->channel,
951 		    xfer->drive, 0);
952 		if (chp->wdc->dma_status == 0xff)
953 			return (-1);
954 		if (chp->wdc->dma_status & WDC_DMAST_NOIRQ) {
955 			/* IRQ not for us, not detected by DMA engine */
956 			return 0;
957 		}
958 		chp->ch_flags &= ~WDCF_DMA_WAIT;
959 	}
960 
961 	chp->ch_flags &= ~WDCF_IRQ_WAIT;
962 	ret = xfer->c_intr(chp, xfer, 1);
963 	if (ret == 0)	/* irq was not for us, still waiting for irq */
964 		chp->ch_flags |= WDCF_IRQ_WAIT;
965 	return (ret);
966 }
967 
968 /* Put all disk in RESET state */
969 void
970 wdc_reset_channel(struct ata_drive_datas *drvp, int nowait)
971 {
972 	struct channel_softc *chp = drvp->chnl_softc;
973 	int drive;
974 
975 	WDCDEBUG_PRINT(("ata_reset_channel %s:%d for drive %d\n",
976 	    chp->wdc->sc_dev.dv_xname, chp->channel, drvp->drive),
977 	    DEBUG_FUNCS);
978 	(void) wdcreset(chp, nowait ? NOWAIT : VERBOSE);
979 	for (drive = 0; drive < 2; drive++) {
980 		chp->ch_drive[drive].state = 0;
981 	}
982 }
983 
984 int
985 wdcreset(struct channel_softc *chp, int flags)
986 {
987 	int drv_mask1, drv_mask2;
988 
989 	if (!chp->_vtbl)
990 		chp->_vtbl = &wdc_default_vtbl;
991 
992 	chp->wdc->reset(chp);
993 
994 	if (flags & NOWAIT)
995 		return 0;
996 
997 	drv_mask1 = (chp->ch_drive[0].drive_flags & DRIVE) ? 0x01:0x00;
998 	drv_mask1 |= (chp->ch_drive[1].drive_flags & DRIVE) ? 0x02:0x00;
999 	drv_mask2 = __wdcwait_reset(chp, drv_mask1);
1000 
1001 	if ((flags & VERBOSE) && drv_mask2 != drv_mask1) {
1002 		printf("%s channel %d: reset failed for",
1003 		    chp->wdc->sc_dev.dv_xname, chp->channel);
1004 		if ((drv_mask1 & 0x01) != 0 && (drv_mask2 & 0x01) == 0)
1005 			printf(" drive 0");
1006 		if ((drv_mask1 & 0x02) != 0 && (drv_mask2 & 0x02) == 0)
1007 			printf(" drive 1");
1008 		printf("\n");
1009 	}
1010 
1011 	return (drv_mask1 != drv_mask2) ? 1 : 0;
1012 }
1013 
1014 void
1015 wdc_do_reset(struct channel_softc *chp)
1016 {
1017 	wdc_set_drive(chp, 0);
1018 	DELAY(10);
1019 	CHP_WRITE_REG(chp, wdr_ctlr, WDCTL_4BIT | WDCTL_RST);
1020 	delay(10000);
1021 	CHP_WRITE_REG(chp, wdr_ctlr, WDCTL_4BIT);
1022 	delay(10000);
1023 }
1024 
1025 int
1026 __wdcwait_reset(struct channel_softc *chp, int drv_mask)
1027 {
1028 	int timeout;
1029 	u_int8_t st0, er0, st1, er1;
1030 
1031 	/* wait for BSY to deassert */
1032 	for (timeout = 0; timeout < WDCNDELAY_RST; timeout++) {
1033 		wdc_set_drive(chp, 0);
1034 		delay(10);
1035 		st0 = CHP_READ_REG(chp, wdr_status);
1036 		er0 = CHP_READ_REG(chp, wdr_error);
1037 		wdc_set_drive(chp, 1);
1038 		delay(10);
1039 		st1 = CHP_READ_REG(chp, wdr_status);
1040 		er1 = CHP_READ_REG(chp, wdr_error);
1041 
1042 		if ((drv_mask & 0x01) == 0) {
1043 			/* no master */
1044 			if ((drv_mask & 0x02) != 0 && (st1 & WDCS_BSY) == 0) {
1045 				/* No master, slave is ready, it's done */
1046 				goto end;
1047 			}
1048 		} else if ((drv_mask & 0x02) == 0) {
1049 			/* no slave */
1050 			if ((drv_mask & 0x01) != 0 && (st0 & WDCS_BSY) == 0) {
1051 				/* No slave, master is ready, it's done */
1052 				goto end;
1053 			}
1054 		} else {
1055 			/* Wait for both master and slave to be ready */
1056 			if ((st0 & WDCS_BSY) == 0 && (st1 & WDCS_BSY) == 0) {
1057 				goto end;
1058 			}
1059 		}
1060 		delay(WDCDELAY);
1061 	}
1062 	/* Reset timed out. Maybe it's because drv_mask was not right */
1063 	if (st0 & WDCS_BSY)
1064 		drv_mask &= ~0x01;
1065 	if (st1 & WDCS_BSY)
1066 		drv_mask &= ~0x02;
1067 end:
1068 	WDCDEBUG_PRINT(("%s:%d: wdcwait_reset() end, st0=0x%b, er0=0x%x, "
1069 	    "st1=0x%b, er1=0x%x, reset time=%d msec\n",
1070 	    chp->wdc ? chp->wdc->sc_dev.dv_xname : "wdcprobe", chp->channel,
1071 	    st0, WDCS_BITS, er0, st1, WDCS_BITS, er1,
1072 	    timeout * WDCDELAY / 1000), DEBUG_PROBE);
1073 
1074 	return drv_mask;
1075 }
1076 
1077 /*
1078  * Wait for a drive to be !BSY, and have mask in its status register.
1079  * return -1 for a timeout after "timeout" ms.
1080  */
1081 int
1082 wdc_wait_for_status(struct channel_softc *chp, int mask, int bits, int timeout)
1083 {
1084 	u_char status;
1085 	int time = 0;
1086 
1087 	WDCDEBUG_PRINT(("wdcwait %s:%d\n", chp->wdc ?chp->wdc->sc_dev.dv_xname
1088 	    :"none", chp->channel), DEBUG_STATUS);
1089 	chp->ch_error = 0;
1090 
1091 	timeout = timeout * 1000 / WDCDELAY; /* delay uses microseconds */
1092 
1093 	for (;;) {
1094 		chp->ch_status = status = CHP_READ_REG(chp, wdr_status);
1095 		WDC_LOG_STATUS(chp, chp->ch_status);
1096 
1097 		if (status == 0xff) {
1098 			if ((chp->ch_flags & WDCF_ONESLAVE)) {
1099 				wdc_set_drive(chp, 1);
1100 				chp->ch_status = status =
1101 				    CHP_READ_REG(chp, wdr_status);
1102 				WDC_LOG_STATUS(chp, chp->ch_status);
1103 			}
1104 		}
1105 		if ((status & WDCS_BSY) == 0 && (status & mask) == bits)
1106 			break;
1107 		if (++time > timeout) {
1108 			WDCDEBUG_PRINT(("wdcwait: timeout, status 0x%b "
1109 			    "error 0x%x\n", status, WDCS_BITS,
1110 			    CHP_READ_REG(chp, wdr_error)),
1111 			    DEBUG_STATUSX | DEBUG_STATUS);
1112 			return -1;
1113 		}
1114 		delay(WDCDELAY);
1115 	}
1116 	if (status & WDCS_ERR) {
1117 		chp->ch_error = CHP_READ_REG(chp, wdr_error);
1118 		WDC_LOG_ERROR(chp, chp->ch_error);
1119 
1120 		WDCDEBUG_PRINT(("wdcwait: error %x\n", chp->ch_error),
1121 			       DEBUG_STATUSX | DEBUG_STATUS);
1122 	}
1123 
1124 #ifdef WDCNDELAY_DEBUG
1125 	/* After autoconfig, there should be no long delays. */
1126 	if (!cold && time > WDCNDELAY_DEBUG) {
1127 		struct wdc_xfer *xfer = TAILQ_FIRST(&chp->ch_queue->sc_xfer);
1128 		if (xfer == NULL)
1129 			printf("%s channel %d: warning: busy-wait took %dus\n",
1130 			    chp->wdc->sc_dev.dv_xname, chp->channel,
1131 			    WDCDELAY * time);
1132 		else
1133 			printf("%s:%d:%d: warning: busy-wait took %dus\n",
1134 			    chp->wdc->sc_dev.dv_xname, chp->channel,
1135 			    xfer->drive,
1136 			    WDCDELAY * time);
1137 	}
1138 #endif /* WDCNDELAY_DEBUG */
1139 	return time;
1140 }
1141 
1142 /*
1143  * Busy-wait for DMA to complete
1144  */
1145 int
1146 wdc_dmawait(struct channel_softc *chp, struct wdc_xfer *xfer, int timeout)
1147 {
1148 	int time;
1149 	for (time = 0; time < timeout * 1000 / WDCDELAY; time++) {
1150 		chp->wdc->dma_status =
1151 		    (*chp->wdc->dma_finish)(chp->wdc->dma_arg,
1152 		    chp->channel, xfer->drive, 0);
1153 		if ((chp->wdc->dma_status & WDC_DMAST_NOIRQ) == 0)
1154 			return 0;
1155 		if (chp->wdc->dma_status == 0xff) {
1156 			chp->dying = 1;
1157 			return -1;
1158 		}
1159 		delay(WDCDELAY);
1160 	}
1161 	/* timeout, force a DMA halt */
1162 	chp->wdc->dma_status = (*chp->wdc->dma_finish)(chp->wdc->dma_arg,
1163 	    chp->channel, xfer->drive, 1);
1164 	return 1;
1165 }
1166 
1167 void
1168 wdctimeout(void *arg)
1169 {
1170 	struct channel_softc *chp = (struct channel_softc *)arg;
1171 	struct wdc_xfer *xfer;
1172 	int s;
1173 
1174 	WDCDEBUG_PRINT(("wdctimeout\n"), DEBUG_FUNCS);
1175 
1176 	s = splbio();
1177 	xfer = TAILQ_FIRST(&chp->ch_queue->sc_xfer);
1178 
1179 	/* Did we lose a race with the interrupt? */
1180 	if (xfer == NULL ||
1181 	    !timeout_triggered(&chp->ch_timo)) {
1182 		splx(s);
1183 		return;
1184 	}
1185 	if ((chp->ch_flags & WDCF_IRQ_WAIT) != 0) {
1186 		__wdcerror(chp, "timeout");
1187 		printf("\ttype: %s\n", (xfer->c_flags & C_ATAPI) ?
1188 		    "atapi":"ata");
1189 		printf("\tc_bcount: %d\n", xfer->c_bcount);
1190 		printf("\tc_skip: %d\n", xfer->c_skip);
1191 		if (chp->ch_flags & WDCF_DMA_WAIT) {
1192 			chp->wdc->dma_status =
1193 			    (*chp->wdc->dma_finish)(chp->wdc->dma_arg,
1194 			    chp->channel, xfer->drive, 1);
1195 			chp->ch_flags &= ~WDCF_DMA_WAIT;
1196 		}
1197 		/*
1198 		 * Call the interrupt routine. If we just missed and interrupt,
1199 		 * it will do what's needed. Else, it will take the needed
1200 		 * action (reset the device).
1201 		 */
1202 		xfer->c_flags |= C_TIMEOU;
1203 		chp->ch_flags &= ~WDCF_IRQ_WAIT;
1204 		xfer->c_intr(chp, xfer, 1);
1205 	} else
1206 		__wdcerror(chp, "missing untimeout");
1207 	splx(s);
1208 }
1209 
1210 /*
1211  * Probe drive's capabilities, for use by the controller later.
1212  * Assumes drvp points to an existing drive.
1213  * XXX this should be a controller-indep function
1214  */
1215 void
1216 wdc_probe_caps(struct ata_drive_datas *drvp, struct ataparams *params)
1217 {
1218 	struct channel_softc *chp = drvp->chnl_softc;
1219 	struct wdc_softc *wdc = chp->wdc;
1220 	int i, valid_mode_found;
1221 	int cf_flags = drvp->cf_flags;
1222 
1223 	if ((wdc->cap & (WDC_CAPABILITY_DATA16 | WDC_CAPABILITY_DATA32)) ==
1224 	    (WDC_CAPABILITY_DATA16 | WDC_CAPABILITY_DATA32)) {
1225 		struct ataparams params2;
1226 
1227 		/*
1228 		 * Controller claims 16 and 32 bit transfers.
1229 		 * Re-do an IDENTIFY with 32-bit transfers,
1230 		 * and compare results.
1231 		 */
1232 		drvp->drive_flags |= DRIVE_CAP32;
1233 		ata_get_params(drvp, at_poll, &params2);
1234 		if (bcmp(params, &params2, sizeof(struct ataparams)) != 0) {
1235 			/* Not good. fall back to 16bits */
1236 			drvp->drive_flags &= ~DRIVE_CAP32;
1237 		}
1238 	}
1239 #if 0 /* Some ultra-DMA drives claims to only support ATA-3. sigh */
1240 	if (params->atap_ata_major > 0x01 &&
1241 	    params->atap_ata_major != 0xffff) {
1242 		for (i = 14; i > 0; i--) {
1243 			if (params->atap_ata_major & (1 << i)) {
1244 				printf("%sATA version %d\n", sep, i);
1245 				drvp->ata_vers = i;
1246 				break;
1247 			}
1248 		}
1249 	} else
1250 #endif /* 0 */
1251 	/* Use PIO mode 3 as a default value for ATAPI devices */
1252 	if (drvp->drive_flags & DRIVE_ATAPI)
1253 		drvp->PIO_mode = 3;
1254 
1255 	WDCDEBUG_PRINT(("wdc_probe_caps: wdc_cap 0x%x cf_flags 0x%x\n",
1256 	    wdc->cap, cf_flags), DEBUG_PROBE);
1257 
1258 	valid_mode_found = 0;
1259 
1260 	WDCDEBUG_PRINT(("%s: atap_oldpiotiming=%d\n", __func__,
1261 	    params->atap_oldpiotiming), DEBUG_PROBE);
1262 	/*
1263 	 * ATA-4 compliant devices contain PIO mode
1264 	 * number in atap_oldpiotiming.
1265 	 */
1266 	if (params->atap_oldpiotiming <= 2) {
1267 		drvp->PIO_cap = params->atap_oldpiotiming;
1268 		valid_mode_found = 1;
1269 		drvp->drive_flags |= DRIVE_MODE;
1270 	} else if (params->atap_oldpiotiming > 180) {
1271 		/*
1272 		 * ATA-2 compliant devices contain cycle
1273 		 * time in atap_oldpiotiming.
1274 		 * A device with a cycle time of 180ns
1275 		 * or less is at least PIO mode 3 and
1276 		 * should be reporting that in
1277 		 * atap_piomode_supp, so ignore it here.
1278 		 */
1279 		if (params->atap_oldpiotiming <= 240) {
1280 			drvp->PIO_cap = 2;
1281 		} else {
1282 			drvp->PIO_cap = 1;
1283 		}
1284 		valid_mode_found = 1;
1285 		drvp->drive_flags |= DRIVE_MODE;
1286 	}
1287 	if (valid_mode_found)
1288 		drvp->PIO_mode = drvp->PIO_cap;
1289 
1290 	WDCDEBUG_PRINT(("%s: atap_extensions=0x%x, atap_piomode_supp=0x%x, "
1291 	    "atap_dmamode_supp=0x%x, atap_udmamode_supp=0x%x\n",
1292 	    __func__, params->atap_extensions, params->atap_piomode_supp,
1293 	    params->atap_dmamode_supp, params->atap_udmamode_supp),
1294 	    DEBUG_PROBE);
1295 
1296 	/*
1297 	 * It's not in the specs, but it seems that some drive
1298 	 * returns 0xffff in atap_extensions when this field is invalid
1299 	 */
1300 	if (params->atap_extensions != 0xffff &&
1301 	    (params->atap_extensions & WDC_EXT_MODES)) {
1302 		/*
1303 		 * XXX some drives report something wrong here (they claim to
1304 		 * support PIO mode 8 !). As mode is coded on 3 bits in
1305 		 * SET FEATURE, limit it to 7 (so limit i to 4).
1306 		 * If higher mode than 7 is found, abort.
1307 		 */
1308 		for (i = 7; i >= 0; i--) {
1309 			if ((params->atap_piomode_supp & (1 << i)) == 0)
1310 				continue;
1311 			if (i > 4)
1312 				return;
1313 
1314 			valid_mode_found = 1;
1315 
1316 			if ((wdc->cap & WDC_CAPABILITY_MODE) == 0) {
1317 				drvp->PIO_cap = i + 3;
1318 				continue;
1319 			}
1320 
1321 			/*
1322 			 * See if mode is accepted.
1323 			 * If the controller can't set its PIO mode,
1324 			 * assume the BIOS set it up correctly
1325 			 */
1326 			if (ata_set_mode(drvp, 0x08 | (i + 3),
1327 			    at_poll) != CMD_OK)
1328 				continue;
1329 
1330 			/*
1331 			 * If controller's driver can't set its PIO mode,
1332 			 * set the highest one the controller supports
1333 			 */
1334 			if (wdc->PIO_cap >= i + 3) {
1335 				drvp->PIO_mode = i + 3;
1336 				drvp->PIO_cap = i + 3;
1337 				break;
1338 			}
1339 		}
1340 		if (!valid_mode_found) {
1341 			/*
1342 			 * We didn't find a valid PIO mode.
1343 			 * Assume the values returned for DMA are buggy too
1344 			 */
1345 			return;
1346 		}
1347 		drvp->drive_flags |= DRIVE_MODE;
1348 
1349 		/* Some controllers don't support ATAPI DMA */
1350 		if ((drvp->drive_flags & DRIVE_ATAPI) &&
1351 		    (wdc->cap & WDC_CAPABILITY_NO_ATAPI_DMA))
1352 			return;
1353 
1354 		valid_mode_found = 0;
1355 		for (i = 7; i >= 0; i--) {
1356 			if ((params->atap_dmamode_supp & (1 << i)) == 0)
1357 				continue;
1358 			if ((wdc->cap & WDC_CAPABILITY_DMA) &&
1359 			    (wdc->cap & WDC_CAPABILITY_MODE))
1360 				if (ata_set_mode(drvp, 0x20 | i, at_poll)
1361 				    != CMD_OK)
1362 					continue;
1363 
1364 			valid_mode_found = 1;
1365 
1366 			if (wdc->cap & WDC_CAPABILITY_DMA) {
1367 				if ((wdc->cap & WDC_CAPABILITY_MODE) &&
1368 				    wdc->DMA_cap < i)
1369 					continue;
1370 				drvp->DMA_mode = i;
1371 				drvp->DMA_cap = i;
1372 				drvp->drive_flags |= DRIVE_DMA;
1373 			}
1374 			break;
1375 		}
1376 		if (params->atap_extensions & WDC_EXT_UDMA_MODES) {
1377 			for (i = 7; i >= 0; i--) {
1378 				if ((params->atap_udmamode_supp & (1 << i))
1379 				    == 0)
1380 					continue;
1381 				if ((wdc->cap & WDC_CAPABILITY_MODE) &&
1382 				    (wdc->cap & WDC_CAPABILITY_UDMA))
1383 					if (ata_set_mode(drvp, 0x40 | i,
1384 					    at_poll) != CMD_OK)
1385 						continue;
1386 				if (wdc->cap & WDC_CAPABILITY_UDMA) {
1387 					if ((wdc->cap & WDC_CAPABILITY_MODE) &&
1388 					    wdc->UDMA_cap < i)
1389 						continue;
1390 					drvp->UDMA_mode = i;
1391 					drvp->UDMA_cap = i;
1392 					drvp->drive_flags |= DRIVE_UDMA;
1393 				}
1394 				break;
1395 			}
1396 		}
1397 	}
1398 
1399 	/* Try to guess ATA version here, if it didn't get reported */
1400 	if (drvp->ata_vers == 0) {
1401 		if (drvp->drive_flags & DRIVE_UDMA)
1402 			drvp->ata_vers = 4; /* should be at last ATA-4 */
1403 		else if (drvp->PIO_cap > 2)
1404 			drvp->ata_vers = 2; /* should be at last ATA-2 */
1405 	}
1406 	if (cf_flags & ATA_CONFIG_PIO_SET) {
1407 		drvp->PIO_mode =
1408 		    (cf_flags & ATA_CONFIG_PIO_MODES) >> ATA_CONFIG_PIO_OFF;
1409 		drvp->drive_flags |= DRIVE_MODE;
1410 	}
1411 	if ((wdc->cap & WDC_CAPABILITY_DMA) == 0) {
1412 		/* don't care about DMA modes */
1413 		return;
1414 	}
1415 	if (cf_flags & ATA_CONFIG_DMA_SET) {
1416 		if ((cf_flags & ATA_CONFIG_DMA_MODES) ==
1417 		    ATA_CONFIG_DMA_DISABLE) {
1418 			drvp->drive_flags &= ~DRIVE_DMA;
1419 		} else {
1420 			drvp->DMA_mode = (cf_flags & ATA_CONFIG_DMA_MODES) >>
1421 			    ATA_CONFIG_DMA_OFF;
1422 			drvp->drive_flags |= DRIVE_DMA | DRIVE_MODE;
1423 		}
1424 	}
1425 	if ((wdc->cap & WDC_CAPABILITY_UDMA) == 0) {
1426 		/* don't care about UDMA modes */
1427 		return;
1428 	}
1429 	if (cf_flags & ATA_CONFIG_UDMA_SET) {
1430 		if ((cf_flags & ATA_CONFIG_UDMA_MODES) ==
1431 		    ATA_CONFIG_UDMA_DISABLE) {
1432 			drvp->drive_flags &= ~DRIVE_UDMA;
1433 		} else {
1434 			drvp->UDMA_mode = (cf_flags & ATA_CONFIG_UDMA_MODES) >>
1435 			    ATA_CONFIG_UDMA_OFF;
1436 			drvp->drive_flags |= DRIVE_UDMA | DRIVE_MODE;
1437 		}
1438 	}
1439 }
1440 
1441 void
1442 wdc_output_bytes(struct ata_drive_datas *drvp, void *bytes, unsigned int buflen)
1443 {
1444 	struct channel_softc *chp = drvp->chnl_softc;
1445 	unsigned int off = 0;
1446 	unsigned int len = buflen, roundlen;
1447 
1448 	if (drvp->drive_flags & DRIVE_CAP32) {
1449 		roundlen = len & ~3;
1450 
1451 		CHP_WRITE_RAW_MULTI_4(chp,
1452 		    (void *)((u_int8_t *)bytes + off), roundlen);
1453 
1454 		off += roundlen;
1455 		len -= roundlen;
1456 	}
1457 
1458 	if (len > 0) {
1459 		roundlen = (len + 1) & ~0x1;
1460 
1461 		CHP_WRITE_RAW_MULTI_2(chp,
1462 		    (void *)((u_int8_t *)bytes + off), roundlen);
1463 	}
1464 }
1465 
1466 void
1467 wdc_input_bytes(struct ata_drive_datas *drvp, void *bytes, unsigned int buflen)
1468 {
1469 	struct channel_softc *chp = drvp->chnl_softc;
1470 	unsigned int off = 0;
1471 	unsigned int len = buflen, roundlen;
1472 
1473 	if (drvp->drive_flags & DRIVE_CAP32) {
1474 		roundlen = len & ~3;
1475 
1476 		CHP_READ_RAW_MULTI_4(chp,
1477 		    (void *)((u_int8_t *)bytes + off), roundlen);
1478 
1479 		off += roundlen;
1480 		len -= roundlen;
1481 	}
1482 
1483 	if (len > 0) {
1484 		roundlen = (len + 1) & ~0x1;
1485 
1486 		CHP_READ_RAW_MULTI_2(chp,
1487 		    (void *)((u_int8_t *)bytes + off), roundlen);
1488 	}
1489 }
1490 
1491 void
1492 wdc_print_caps(struct ata_drive_datas *drvp)
1493 {
1494 	/* This is actually a lie until we fix the _probe_caps
1495 	   algorithm. Don't print out lies */
1496 #if 0
1497  	printf("%s: can use ", drvp->drive_name);
1498 
1499 	if (drvp->drive_flags & DRIVE_CAP32) {
1500 		printf("32-bit");
1501 	} else
1502 		printf("16-bit");
1503 
1504 	printf(", PIO mode %d", drvp->PIO_cap);
1505 
1506 	if (drvp->drive_flags & DRIVE_DMA) {
1507 		printf(", DMA mode %d", drvp->DMA_cap);
1508 	}
1509 
1510 	if (drvp->drive_flags & DRIVE_UDMA) {
1511 		printf(", Ultra-DMA mode %d", drvp->UDMA_cap);
1512 	}
1513 
1514 	printf("\n");
1515 #endif /* 0 */
1516 }
1517 
1518 void
1519 wdc_print_current_modes(struct channel_softc *chp)
1520 {
1521 	int drive;
1522 	struct ata_drive_datas *drvp;
1523 
1524 	for (drive = 0; drive < 2; drive++) {
1525 		drvp = &chp->ch_drive[drive];
1526 		if ((drvp->drive_flags & DRIVE) == 0)
1527 			continue;
1528 
1529 		printf("%s(%s:%d:%d):",
1530  		    drvp->drive_name,
1531 		    chp->wdc->sc_dev.dv_xname, chp->channel, drive);
1532 
1533 		if ((chp->wdc->cap & WDC_CAPABILITY_MODE) == 0 &&
1534 		    !(drvp->cf_flags & ATA_CONFIG_PIO_SET))
1535 			printf(" using BIOS timings");
1536 		else
1537 			printf(" using PIO mode %d", drvp->PIO_mode);
1538 		if (drvp->drive_flags & DRIVE_DMA)
1539 			printf(", DMA mode %d", drvp->DMA_mode);
1540 		if (drvp->drive_flags & DRIVE_UDMA)
1541 			printf(", Ultra-DMA mode %d", drvp->UDMA_mode);
1542 		printf("\n");
1543 	}
1544 }
1545 
1546 /*
1547  * downgrade the transfer mode of a drive after an error. return 1 if
1548  * downgrade was possible, 0 otherwise.
1549  */
1550 int
1551 wdc_downgrade_mode(struct ata_drive_datas *drvp)
1552 {
1553 	struct channel_softc *chp = drvp->chnl_softc;
1554 	struct wdc_softc *wdc = chp->wdc;
1555 	int cf_flags = drvp->cf_flags;
1556 
1557 	/* if drive or controller don't know its mode, we can't do much */
1558 	if ((drvp->drive_flags & DRIVE_MODE) == 0 ||
1559 	    (wdc->cap & WDC_CAPABILITY_MODE) == 0)
1560 		return 0;
1561 	/* current drive mode was set by a config flag, let it this way */
1562 	if ((cf_flags & ATA_CONFIG_PIO_SET) ||
1563 	    (cf_flags & ATA_CONFIG_DMA_SET) ||
1564 	    (cf_flags & ATA_CONFIG_UDMA_SET))
1565 		return 0;
1566 
1567 	/*
1568 	 * We'd ideally like to use an Ultra DMA mode since they have the
1569 	 * protection of a CRC. So we try each Ultra DMA mode and see if
1570 	 * we can find any working combo
1571 	 */
1572 	if ((drvp->drive_flags & DRIVE_UDMA) && drvp->UDMA_mode > 0) {
1573 		drvp->UDMA_mode = drvp->UDMA_mode - 1;
1574 		printf("%s: transfer error, downgrading to Ultra-DMA mode %d\n",
1575 		    drvp->drive_name, drvp->UDMA_mode);
1576 	} else 	if ((drvp->drive_flags & DRIVE_UDMA) &&
1577 	    (drvp->drive_flags & DRIVE_DMAERR) == 0) {
1578 		/*
1579 		 * If we were using ultra-DMA, don't downgrade to
1580 		 * multiword DMA if we noticed a CRC error. It has
1581 		 * been noticed that CRC errors in ultra-DMA lead to
1582 		 * silent data corruption in multiword DMA.  Data
1583 		 * corruption is less likely to occur in PIO mode.
1584 		 */
1585 		drvp->drive_flags &= ~DRIVE_UDMA;
1586 		drvp->drive_flags |= DRIVE_DMA;
1587 		drvp->DMA_mode = drvp->DMA_cap;
1588 		printf("%s: transfer error, downgrading to DMA mode %d\n",
1589 		    drvp->drive_name, drvp->DMA_mode);
1590 	} else if (drvp->drive_flags & (DRIVE_DMA | DRIVE_UDMA)) {
1591 		drvp->drive_flags &= ~(DRIVE_DMA | DRIVE_UDMA);
1592 		drvp->PIO_mode = drvp->PIO_cap;
1593 		printf("%s: transfer error, downgrading to PIO mode %d\n",
1594 		    drvp->drive_name, drvp->PIO_mode);
1595 	} else /* already using PIO, can't downgrade */
1596 		return 0;
1597 
1598 	wdc->set_modes(chp);
1599 	/* reset the channel, which will schedule all drives for setup */
1600 	wdc_reset_channel(drvp, 0);
1601 	return 1;
1602 }
1603 
1604 int
1605 wdc_exec_command(struct ata_drive_datas *drvp, struct wdc_command *wdc_c)
1606 {
1607 	struct channel_softc *chp = drvp->chnl_softc;
1608 	struct wdc_xfer *xfer;
1609 	int s, ret;
1610 
1611 	WDCDEBUG_PRINT(("wdc_exec_command %s:%d:%d\n",
1612 	    chp->wdc->sc_dev.dv_xname, chp->channel, drvp->drive),
1613 	    DEBUG_FUNCS);
1614 
1615 	/* set up an xfer and queue. Wait for completion */
1616 	xfer = wdc_get_xfer(wdc_c->flags & AT_WAIT ? WDC_CANSLEEP :
1617 	    WDC_NOSLEEP);
1618 	if (xfer == NULL) {
1619 		return WDC_TRY_AGAIN;
1620 	}
1621 
1622 	if (wdc_c->flags & AT_POLL)
1623 		xfer->c_flags |= C_POLL;
1624 	xfer->drive = drvp->drive;
1625 	xfer->databuf = wdc_c->data;
1626 	xfer->c_bcount = wdc_c->bcount;
1627 	xfer->cmd = wdc_c;
1628 	xfer->c_start = __wdccommand_start;
1629 	xfer->c_intr = __wdccommand_intr;
1630 	xfer->c_kill_xfer = __wdccommand_done;
1631 
1632 	s = splbio();
1633 	wdc_exec_xfer(chp, xfer);
1634 #ifdef DIAGNOSTIC
1635 	if ((wdc_c->flags & AT_POLL) != 0 &&
1636 	    (wdc_c->flags & AT_DONE) == 0)
1637 		panic("wdc_exec_command: polled command not done");
1638 #endif /* DIAGNOSTIC */
1639 	if (wdc_c->flags & AT_DONE) {
1640 		ret = WDC_COMPLETE;
1641 	} else {
1642 		if (wdc_c->flags & AT_WAIT) {
1643 			WDCDEBUG_PRINT(("wdc_exec_command sleeping\n"),
1644 				       DEBUG_FUNCS);
1645 
1646 			while ((wdc_c->flags & AT_DONE) == 0) {
1647 				tsleep(wdc_c, PRIBIO, "wdccmd", 0);
1648 			}
1649 			ret = WDC_COMPLETE;
1650 		} else {
1651 			ret = WDC_QUEUED;
1652 		}
1653 	}
1654 	splx(s);
1655 	return ret;
1656 }
1657 
1658 void
1659 __wdccommand_start(struct channel_softc *chp, struct wdc_xfer *xfer)
1660 {
1661 	int drive = xfer->drive;
1662 	struct wdc_command *wdc_c = xfer->cmd;
1663 
1664 	WDCDEBUG_PRINT(("__wdccommand_start %s:%d:%d\n",
1665 	    chp->wdc->sc_dev.dv_xname, chp->channel, xfer->drive),
1666 	    DEBUG_FUNCS);
1667 
1668 	/*
1669 	 * Disable interrupts if we're polling
1670 	 */
1671 	if (xfer->c_flags & C_POLL) {
1672 		wdc_disable_intr(chp);
1673 	}
1674 
1675 	wdc_set_drive(chp, drive);
1676 	DELAY(1);
1677 
1678 	/*
1679 	 * For resets, we don't really care to make sure that
1680 	 * the bus is free
1681 	 */
1682 	if (wdc_c->r_command != ATAPI_SOFT_RESET) {
1683 		if (wdcwait(chp, wdc_c->r_st_bmask | WDCS_DRQ,
1684 		    wdc_c->r_st_bmask, wdc_c->timeout) != 0) {
1685 			goto timeout;
1686 		}
1687 	} else
1688 		DELAY(10);
1689 
1690 	wdccommand(chp, drive, wdc_c->r_command, wdc_c->r_cyl, wdc_c->r_head,
1691 	    wdc_c->r_sector, wdc_c->r_count, wdc_c->r_precomp);
1692 
1693 	if ((wdc_c->flags & AT_WRITE) == AT_WRITE) {
1694 		/* wait at least 400ns before reading status register */
1695 		DELAY(10);
1696 		if (wait_for_unbusy(chp, wdc_c->timeout) != 0)
1697 			goto timeout;
1698 
1699 		if ((chp->ch_status & (WDCS_DRQ | WDCS_ERR)) == WDCS_ERR) {
1700 			__wdccommand_done(chp, xfer);
1701 			return;
1702 		}
1703 
1704 		if (wait_for_drq(chp, wdc_c->timeout) != 0)
1705 			goto timeout;
1706 
1707 		wdc_output_bytes(&chp->ch_drive[drive],
1708 		    wdc_c->data, wdc_c->bcount);
1709 	}
1710 
1711 	if ((wdc_c->flags & AT_POLL) == 0) {
1712 		chp->ch_flags |= WDCF_IRQ_WAIT; /* wait for interrupt */
1713 		timeout_add_msec(&chp->ch_timo, wdc_c->timeout);
1714 		return;
1715 	}
1716 
1717 	/*
1718 	 * Polled command. Wait for drive ready or drq. Done in intr().
1719 	 * Wait for at last 400ns for status bit to be valid.
1720 	 */
1721 	delay(10);
1722 	__wdccommand_intr(chp, xfer, 0);
1723 	return;
1724 
1725 timeout:
1726 	wdc_c->flags |= AT_TIMEOU;
1727 	__wdccommand_done(chp, xfer);
1728 }
1729 
1730 int
1731 __wdccommand_intr(struct channel_softc *chp, struct wdc_xfer *xfer, int irq)
1732 {
1733 	struct ata_drive_datas *drvp = &chp->ch_drive[xfer->drive];
1734 	struct wdc_command *wdc_c = xfer->cmd;
1735 	int bcount = wdc_c->bcount;
1736 	char *data = wdc_c->data;
1737 
1738 	WDCDEBUG_PRINT(("__wdccommand_intr %s:%d:%d\n",
1739 	    chp->wdc->sc_dev.dv_xname, chp->channel, xfer->drive), DEBUG_INTR);
1740 	if (wdcwait(chp, wdc_c->r_st_pmask, wdc_c->r_st_pmask,
1741 	    (irq == 0) ? wdc_c->timeout : 0)) {
1742 		if (chp->dying) {
1743 			__wdccommand_done(chp, xfer);
1744 			return -1;
1745 		}
1746 		if (irq && (xfer->c_flags & C_TIMEOU) == 0)
1747 			return 0; /* IRQ was not for us */
1748 		wdc_c->flags |= AT_TIMEOU;
1749 		goto out;
1750 	}
1751 	if (chp->wdc->cap & WDC_CAPABILITY_IRQACK)
1752 		chp->wdc->irqack(chp);
1753 	if (wdc_c->flags & AT_READ) {
1754 		if ((chp->ch_status & WDCS_DRQ) == 0) {
1755 			wdc_c->flags |= AT_TIMEOU;
1756 			goto out;
1757 		}
1758 		wdc_input_bytes(drvp, data, bcount);
1759 		/* Should we wait for device to indicate idle? */
1760 	}
1761 out:
1762 	__wdccommand_done(chp, xfer);
1763 	WDCDEBUG_PRINT(("__wdccommand_intr returned\n"), DEBUG_INTR);
1764 	return 1;
1765 }
1766 
1767 void
1768 __wdccommand_done(struct channel_softc *chp, struct wdc_xfer *xfer)
1769 {
1770 	struct wdc_command *wdc_c = xfer->cmd;
1771 
1772 	WDCDEBUG_PRINT(("__wdccommand_done %s:%d:%d %02x\n",
1773 	    chp->wdc->sc_dev.dv_xname, chp->channel, xfer->drive,
1774 	    chp->ch_status), DEBUG_FUNCS);
1775 	if (chp->dying)
1776 		goto killit;
1777 	if (chp->ch_status & WDCS_DWF)
1778 		wdc_c->flags |= AT_DF;
1779 	if (chp->ch_status & WDCS_ERR) {
1780 		wdc_c->flags |= AT_ERROR;
1781 		wdc_c->r_error = chp->ch_error;
1782 	}
1783 	wdc_c->flags |= AT_DONE;
1784 	if ((wdc_c->flags & AT_READREG) != 0 &&
1785 	    (wdc_c->flags & (AT_ERROR | AT_DF)) == 0) {
1786 		wdc_c->r_head = CHP_READ_REG(chp, wdr_sdh);
1787 		wdc_c->r_cyl = CHP_READ_REG(chp, wdr_cyl_hi) << 8;
1788 		wdc_c->r_cyl |= CHP_READ_REG(chp, wdr_cyl_lo);
1789 		wdc_c->r_sector = CHP_READ_REG(chp, wdr_sector);
1790 		wdc_c->r_count = CHP_READ_REG(chp, wdr_seccnt);
1791 		wdc_c->r_error = CHP_READ_REG(chp, wdr_error);
1792 		wdc_c->r_precomp = wdc_c->r_error;
1793 		/* XXX CHP_READ_REG(chp, wdr_precomp); - precomp
1794 		   isn't a readable register */
1795 	}
1796 
1797 killit:
1798 	if (xfer->c_flags & C_POLL) {
1799 		wdc_enable_intr(chp);
1800 	} else
1801 		timeout_del(&chp->ch_timo);
1802 
1803 	wdc_free_xfer(chp, xfer);
1804 	WDCDEBUG_PRINT(("__wdccommand_done before callback\n"), DEBUG_INTR);
1805 
1806 	if (chp->dying)
1807 		return;
1808 
1809 	if (wdc_c->flags & AT_WAIT)
1810 		wakeup(wdc_c);
1811 	else
1812 		if (wdc_c->callback)
1813 			wdc_c->callback(wdc_c->callback_arg);
1814 	wdcstart(chp);
1815 	WDCDEBUG_PRINT(("__wdccommand_done returned\n"), DEBUG_INTR);
1816 }
1817 
1818 /*
1819  * Send a command. The drive should be ready.
1820  * Assumes interrupts are blocked.
1821  */
1822 void
1823 wdccommand(struct channel_softc *chp, u_int8_t drive, u_int8_t command,
1824     u_int16_t cylin, u_int8_t head, u_int8_t sector, u_int8_t count,
1825     u_int8_t precomp)
1826 {
1827 	WDCDEBUG_PRINT(("wdccommand %s:%d:%d: command=0x%x cylin=%d head=%d "
1828 	    "sector=%d count=%d precomp=%d\n", chp->wdc->sc_dev.dv_xname,
1829 	    chp->channel, drive, command, cylin, head, sector, count, precomp),
1830 	    DEBUG_FUNCS);
1831 	WDC_LOG_ATA_CMDLONG(chp, head, precomp, cylin, cylin >> 8, sector,
1832 	    count, command);
1833 
1834 	/* Select drive, head, and addressing mode. */
1835 	CHP_WRITE_REG(chp, wdr_sdh, WDSD_IBM | (drive << 4) | head);
1836 
1837 	/* Load parameters. wdr_features(ATA/ATAPI) = wdr_precomp(ST506) */
1838 	CHP_WRITE_REG(chp, wdr_precomp, precomp);
1839 	CHP_WRITE_REG(chp, wdr_cyl_lo, cylin);
1840 	CHP_WRITE_REG(chp, wdr_cyl_hi, cylin >> 8);
1841 	CHP_WRITE_REG(chp, wdr_sector, sector);
1842 	CHP_WRITE_REG(chp, wdr_seccnt, count);
1843 
1844 	/* Send command. */
1845 	CHP_WRITE_REG(chp, wdr_command, command);
1846 }
1847 
1848 /*
1849  * Send a 48-bit addressing command. The drive should be ready.
1850  * Assumes interrupts are blocked.
1851  */
1852 void
1853 wdccommandext(struct channel_softc *chp, u_int8_t drive, u_int8_t command,
1854     u_int64_t blkno, u_int16_t count)
1855 {
1856 	WDCDEBUG_PRINT(("wdccommandext %s:%d:%d: command=0x%x blkno=%llu "
1857 	    "count=%d\n", chp->wdc->sc_dev.dv_xname,
1858 	    chp->channel, drive, command, blkno, count),
1859 	    DEBUG_FUNCS);
1860 	WDC_LOG_ATA_CMDEXT(chp, blkno >> 40, blkno >> 16, blkno >> 32,
1861 	    blkno >> 8, blkno >> 24, blkno, count >> 8, count, command);
1862 
1863 	/* Select drive and LBA mode. */
1864 	CHP_WRITE_REG(chp, wdr_sdh, (drive << 4) | WDSD_LBA);
1865 
1866 	/* Load parameters. */
1867 	CHP_LBA48_WRITE_REG(chp, wdr_lba_hi,
1868 	    ((blkno >> 32) & 0xff00) | ((blkno >> 16) & 0xff));
1869 	CHP_LBA48_WRITE_REG(chp, wdr_lba_mi,
1870 	    ((blkno >> 24) & 0xff00) | ((blkno >> 8) & 0xff));
1871 	CHP_LBA48_WRITE_REG(chp, wdr_lba_lo,
1872 	    ((blkno >> 16) & 0xff00) | (blkno & 0xff));
1873 	CHP_LBA48_WRITE_REG(chp, wdr_seccnt, count);
1874 
1875 	/* Send command. */
1876 	CHP_WRITE_REG(chp, wdr_command, command);
1877 }
1878 
1879 /*
1880  * Simplified version of wdccommand().  Unbusy/ready/drq must be
1881  * tested by the caller.
1882  */
1883 void
1884 wdccommandshort(struct channel_softc *chp, int drive, int command)
1885 {
1886 
1887 	WDCDEBUG_PRINT(("wdccommandshort %s:%d:%d command 0x%x\n",
1888 	    chp->wdc->sc_dev.dv_xname, chp->channel, drive, command),
1889 	    DEBUG_FUNCS);
1890 	WDC_LOG_ATA_CMDSHORT(chp, command);
1891 
1892 	/* Select drive. */
1893 	CHP_WRITE_REG(chp, wdr_sdh, WDSD_IBM | (drive << 4));
1894 	CHP_WRITE_REG(chp, wdr_command, command);
1895 }
1896 
1897 /* Add a command to the queue and start controller. Must be called at splbio */
1898 
1899 void
1900 wdc_exec_xfer(struct channel_softc *chp, struct wdc_xfer *xfer)
1901 {
1902 	WDCDEBUG_PRINT(("wdc_exec_xfer %p flags 0x%x channel %d drive %d\n",
1903 	    xfer, xfer->c_flags, chp->channel, xfer->drive), DEBUG_XFERS);
1904 
1905 	/* complete xfer setup */
1906 	xfer->chp = chp;
1907 
1908 	/*
1909 	 * If we are a polled command, and the list is not empty,
1910 	 * we are doing a dump. Drop the list to allow the polled command
1911 	 * to complete, we're going to reboot soon anyway.
1912 	 */
1913 	if ((xfer->c_flags & C_POLL) != 0 &&
1914 	    !TAILQ_EMPTY(&chp->ch_queue->sc_xfer)) {
1915 		TAILQ_INIT(&chp->ch_queue->sc_xfer);
1916 	}
1917 	/* insert at the end of command list */
1918 	TAILQ_INSERT_TAIL(&chp->ch_queue->sc_xfer,xfer , c_xferchain);
1919 	WDCDEBUG_PRINT(("wdcstart from wdc_exec_xfer, flags 0x%x\n",
1920 	    chp->ch_flags), DEBUG_XFERS);
1921 	wdcstart(chp);
1922 }
1923 
1924 void *
1925 wdc_xfer_get(void *null)
1926 {
1927 	return (pool_get(&wdc_xfer_pool, PR_NOWAIT | PR_ZERO));
1928 }
1929 
1930 void
1931 wdc_scrub_xfer(struct wdc_xfer *xfer)
1932 {
1933 	memset(xfer, 0, sizeof(*xfer));
1934 	xfer->c_flags = C_SCSIXFER;
1935 }
1936 
1937 void
1938 wdc_xfer_put(void *null, void *xxfer)
1939 {
1940 	struct wdc_xfer *xfer = xxfer;
1941 	int put = 0;
1942 	int s;
1943 
1944 	s = splbio();
1945 	if (ISSET(xfer->c_flags, C_SCSIXFER))
1946 		CLR(xfer->c_flags, C_SCSIXFER);
1947 	else
1948 		put = 1;
1949 	splx(s);
1950 
1951 	if (put)
1952 		pool_put(&wdc_xfer_pool, xfer);
1953 }
1954 
1955 struct wdc_xfer *
1956 wdc_get_xfer(int flags)
1957 {
1958 	return (scsi_io_get(&wdc_xfer_iopool,
1959 	    ISSET(flags, WDC_NOSLEEP) ? SCSI_NOSLEEP : 0));
1960 }
1961 
1962 void
1963 wdc_free_xfer(struct channel_softc *chp, struct wdc_xfer *xfer)
1964 {
1965 	int put = 0;
1966 	int s;
1967 
1968 	if (xfer->c_flags & C_PRIVATEXFER) {
1969 		chp->ch_flags &= ~WDCF_ACTIVE;
1970 		TAILQ_REMOVE(&chp->ch_queue->sc_xfer, xfer, c_xferchain);
1971 		return;
1972 	}
1973 
1974 	s = splbio();
1975 	chp->ch_flags &= ~WDCF_ACTIVE;
1976 	TAILQ_REMOVE(&chp->ch_queue->sc_xfer, xfer, c_xferchain);
1977 	if (ISSET(xfer->c_flags, C_SCSIXFER))
1978 		CLR(xfer->c_flags, C_SCSIXFER);
1979 	else
1980 		put = 1;
1981 	splx(s);
1982 
1983 	if (put)
1984 		scsi_io_put(&wdc_xfer_iopool, xfer);
1985 }
1986 
1987 
1988 /*
1989  * Kill off all pending xfers for a channel_softc.
1990  *
1991  * Must be called at splbio().
1992  */
1993 void
1994 wdc_kill_pending(struct channel_softc *chp)
1995 {
1996 	struct wdc_xfer *xfer;
1997 
1998 	while ((xfer = TAILQ_FIRST(&chp->ch_queue->sc_xfer)) != NULL) {
1999 		chp = xfer->chp;
2000 		(*xfer->c_kill_xfer)(chp, xfer);
2001 	}
2002 }
2003 
2004 void
2005 __wdcerror(struct channel_softc *chp, char *msg)
2006 {
2007 	struct wdc_xfer *xfer = TAILQ_FIRST(&chp->ch_queue->sc_xfer);
2008 	if (xfer == NULL)
2009 		printf("%s:%d: %s\n", chp->wdc->sc_dev.dv_xname, chp->channel,
2010 		    msg);
2011 	else
2012 		printf("%s(%s:%d:%d): %s\n",
2013 		    chp->ch_drive[xfer->drive].drive_name,
2014 		    chp->wdc->sc_dev.dv_xname,
2015 		    chp->channel, xfer->drive, msg);
2016 }
2017 
2018 /*
2019  * the bit bucket
2020  */
2021 void
2022 wdcbit_bucket(struct channel_softc *chp, int size)
2023 {
2024 	CHP_READ_RAW_MULTI_2(chp, NULL, size);
2025 }
2026 
2027 
2028 #include <sys/ataio.h>
2029 #include <sys/file.h>
2030 
2031 int wdc_ioc_ata_cmd(struct ata_drive_datas *, atareq_t *);
2032 
2033 int
2034 wdc_ioc_ata_cmd(struct ata_drive_datas *drvp, atareq_t *atareq)
2035 {
2036 	struct wdc_command wdc_c;
2037 	int err = 0;
2038 
2039 	/*
2040 	 * Make sure a timeout was supplied in the ioctl request
2041 	 */
2042 	if (atareq->timeout == 0)
2043 		return (EINVAL);
2044 
2045 	if (atareq->datalen > MAXPHYS)
2046 		return (EINVAL);
2047 
2048 	bzero(&wdc_c, sizeof(wdc_c));
2049 
2050 	if (atareq->datalen > 0) {
2051 		wdc_c.data = dma_alloc(atareq->datalen, PR_NOWAIT | PR_ZERO);
2052 		if (wdc_c.data == NULL) {
2053 			err = ENOMEM;
2054 			goto err;
2055 		}
2056 		wdc_c.bcount = atareq->datalen;
2057 	}
2058 
2059 	wdc_c.flags = AT_WAIT;
2060 	if (atareq->flags & ATACMD_READ)
2061 		wdc_c.flags |= AT_READ;
2062 	if (atareq->flags & ATACMD_WRITE) {
2063 		if (atareq->datalen > 0) {
2064 			err = copyin(atareq->databuf, wdc_c.data,
2065 			    atareq->datalen);
2066 			if (err != 0)
2067 				goto err;
2068 		}
2069 		wdc_c.flags |= AT_WRITE;
2070 	}
2071 	if (atareq->flags & ATACMD_READREG)
2072 		wdc_c.flags |= AT_READREG;
2073 
2074 	wdc_c.timeout = atareq->timeout;
2075 	wdc_c.r_command = atareq->command;
2076 	wdc_c.r_head = atareq->head & 0x0f;
2077 	wdc_c.r_cyl = atareq->cylinder;
2078 	wdc_c.r_sector = atareq->sec_num;
2079 	wdc_c.r_count = atareq->sec_count;
2080 	wdc_c.r_precomp = atareq->features;
2081 	if (drvp->drive_flags & DRIVE_ATAPI) {
2082 		if (wdc_c.r_command == WDCC_IDENTIFY)
2083 			wdc_c.r_command = ATAPI_IDENTIFY_DEVICE;
2084 	} else {
2085 		wdc_c.r_st_bmask = WDCS_DRDY;
2086 		wdc_c.r_st_pmask = WDCS_DRDY;
2087 	}
2088 
2089 	if (wdc_exec_command(drvp, &wdc_c) != WDC_COMPLETE) {
2090 		atareq->retsts = ATACMD_ERROR;
2091 		goto copyout;
2092 	}
2093 
2094 	if (wdc_c.flags & (AT_ERROR | AT_TIMEOU | AT_DF)) {
2095 		if (wdc_c.flags & AT_ERROR) {
2096 			atareq->retsts = ATACMD_ERROR;
2097 			atareq->error = wdc_c.r_error;
2098 		} else if (wdc_c.flags & AT_DF)
2099 			atareq->retsts = ATACMD_DF;
2100 		else
2101 			atareq->retsts = ATACMD_TIMEOUT;
2102 	} else {
2103 		atareq->retsts = ATACMD_OK;
2104 		if (atareq->flags & ATACMD_READREG) {
2105 			atareq->head = wdc_c.r_head;
2106 			atareq->cylinder = wdc_c.r_cyl;
2107 			atareq->sec_num = wdc_c.r_sector;
2108 			atareq->sec_count = wdc_c.r_count;
2109 			atareq->features = wdc_c.r_precomp;
2110 			atareq->error = wdc_c.r_error;
2111 		}
2112 	}
2113 
2114 copyout:
2115 	if (atareq->datalen > 0 && atareq->flags & ATACMD_READ) {
2116 		err = copyout(wdc_c.data, atareq->databuf, atareq->datalen);
2117 		if (err != 0)
2118 			goto err;
2119 	}
2120 
2121 err:
2122 	if (wdc_c.data)
2123 		dma_free(wdc_c.data, atareq->datalen);
2124 	return (err);
2125 }
2126 
2127 int
2128 wdc_ioctl(struct ata_drive_datas *drvp, u_long xfer, caddr_t addr, int flag,
2129     struct proc *p)
2130 {
2131 	int error = 0;
2132 
2133 	switch (xfer) {
2134 #ifdef WDCDEBUG
2135 	case ATAIOGETTRACE: {
2136 		atagettrace_t *agt = (atagettrace_t *)addr;
2137 		unsigned int size = 0;
2138 		char *log_to_copy;
2139 
2140 		size = agt->buf_size;
2141 		if (size > 65536) {
2142 			size = 65536;
2143 		}
2144 
2145 		log_to_copy = wdc_get_log(&size, &agt->bytes_left);
2146 
2147 		if (log_to_copy != NULL) {
2148 			error = copyout(log_to_copy, agt->buf, size);
2149 			free(log_to_copy, M_TEMP, 0);
2150 		}
2151 
2152 		agt->bytes_copied = size;
2153 		break;
2154 	}
2155 #endif /* WDCDEBUG */
2156 
2157 	case ATAIOCCOMMAND: {
2158 		atareq_t *atareq = (atareq_t *)addr;
2159 
2160 		/*
2161 		 * Make sure this command is (relatively) safe first
2162 		 */
2163 		if ((flag & FWRITE) == 0 && atareq->flags & ATACMD_WRITE)
2164 			error = EPERM;
2165 		else
2166 			error = wdc_ioc_ata_cmd(drvp, atareq);
2167 		break;
2168 	}
2169 
2170 	default:
2171 		error = ENOTTY;
2172 		goto exit;
2173 	}
2174 
2175 exit:
2176 	return (error);
2177 }
2178