46 static void ata_sort_queue(struct ata_channel *ch, struct ata_request *request);
47 static void atawritereorder(struct ata_channel *ch);
51 ata_queue_init(struct ata_channel *ch)  in ata_queue_init()  argument
53     TAILQ_INIT(&ch->ata_queue);  in ata_queue_init()
54     ch->reorder = 0;  in ata_queue_init()
55     ch->transition = NULL;  in ata_queue_init()
65     struct ata_channel *ch = device_get_softc(device_get_parent(dev));  in ata_drop_requests()  local
68     lockmgr(&ch->queue_mtx, LK_EXCLUSIVE);  in ata_drop_requests()
69     TAILQ_FOREACH_MUTABLE(request, &ch->ata_queue, chain, tmp) {  in ata_drop_requests()
70 	TAILQ_REMOVE(&ch->ata_queue, request, chain);  in ata_drop_requests()
73     lockmgr(&ch->queue_mtx, LK_RELEASE);  in ata_drop_requests()
79     struct ata_channel *ch;  in ata_queue_request()  local
91     ch = device_get_softc(request->parent);  in ata_queue_request()
93     callout_init_lk(&request->callout, &ch->state_mtx);  in ata_queue_request()
98     if ((ch->state & ATA_STALL_QUEUE) && (request->flags & ATA_R_CONTROL)) {  in ata_queue_request()
99 	lockmgr(&ch->state_mtx, LK_EXCLUSIVE);  in ata_queue_request()
100 	ch->running = request;  in ata_queue_request()
101 	if (ch->hw.begin_transaction(request) == ATA_OP_FINISHED) {  in ata_queue_request()
102 	    ch->running = NULL;  in ata_queue_request()
105 	    lockmgr(&ch->state_mtx, LK_RELEASE);  in ata_queue_request()
110 	lockmgr(&ch->state_mtx, LK_RELEASE);  in ata_queue_request()
114 	lockmgr(&ch->queue_mtx, LK_EXCLUSIVE);  in ata_queue_request()
116 	    TAILQ_INSERT_HEAD(&ch->ata_queue, request, chain);  in ata_queue_request()
118 	    ata_sort_queue(ch, request);  in ata_queue_request()
120 	    TAILQ_INSERT_TAIL(&ch->ata_queue, request, chain);  in ata_queue_request()
121 	    ch->transition = NULL;  in ata_queue_request()
123 	lockmgr(&ch->queue_mtx, LK_RELEASE);  in ata_queue_request()
125 	ata_start(ch->dev);  in ata_queue_request()
217     struct ata_channel *ch = device_get_softc(dev);  in ata_start()  local
223     lockmgr(&ch->queue_mtx, LK_EXCLUSIVE);  in ata_start()
224     if ((request = TAILQ_FIRST(&ch->ata_queue))) {  in ata_start()
227 	if (ATA_LOCKING(dev, ATA_LF_LOCK) == ch->unit) {  in ata_start()
240 	    lockmgr(&ch->state_mtx, LK_EXCLUSIVE);  in ata_start()
241 	    if (ch->state == ATA_IDLE && !dependencies) {  in ata_start()
244 		if (ch->transition == request)  in ata_start()
245 		    ch->transition = TAILQ_NEXT(request, chain);  in ata_start()
246 		TAILQ_REMOVE(&ch->ata_queue, request, chain);  in ata_start()
247 		ch->running = request;  in ata_start()
248 		ch->state = ATA_ACTIVE;  in ata_start()
250 		if (ch->hw.begin_transaction(request) == ATA_OP_FINISHED) {  in ata_start()
251 		    ch->running = NULL;  in ata_start()
252 		    ch->state = ATA_IDLE;  in ata_start()
253 		    lockmgr(&ch->state_mtx, LK_RELEASE);  in ata_start()
254 		    lockmgr(&ch->queue_mtx, LK_RELEASE);  in ata_start()
264 		    lockmgr(&ch->state_mtx, LK_RELEASE);  in ata_start()
265 		    lockmgr(&ch->queue_mtx, LK_RELEASE);  in ata_start()
266 		    while (!ata_interrupt(ch))  in ata_start()
271 	    lockmgr(&ch->state_mtx, LK_RELEASE);  in ata_start()
274     lockmgr(&ch->queue_mtx, LK_RELEASE);  in ata_start()
280     struct ata_channel *ch = device_get_softc(request->parent);  in ata_finish()  local
287 	(ch->state & ATA_STALL_QUEUE) || (request->flags & ATA_R_DIRECT)) {  in ata_finish()
304     struct ata_channel *ch = device_get_softc(request->parent);  in ata_completed()  local
329 	if (ch && !ata_reinit(ch->dev) && !request->result &&  in ata_completed()
534     if (ch)  in ata_completed()
535 	ata_start(ch->dev);  in ata_completed()
541     struct ata_channel *ch = device_get_softc(request->parent);  in ata_timeout()  local
555     if (ch->state == ATA_ACTIVE) {  in ata_timeout()
557 	lockmgr(&ch->state_mtx, LK_RELEASE);  in ata_timeout()
558 	ATA_LOCKING(ch->dev, ATA_LF_UNLOCK);  in ata_timeout()
560 	lockmgr(&ch->state_mtx, LK_EXCLUSIVE);  in ata_timeout()
567     struct ata_channel *ch = device_get_softc(device_get_parent(dev));  in ata_fail_requests()  local
573     lockmgr(&ch->queue_mtx, LK_EXCLUSIVE);  in ata_fail_requests()
574     lockmgr(&ch->state_mtx, LK_EXCLUSIVE);  in ata_fail_requests()
577     if ((request = ch->running) && (!dev || request->dev == dev)) {  in ata_fail_requests()
579 	ch->running = NULL;  in ata_fail_requests()
585     TAILQ_FOREACH_MUTABLE(request, &ch->ata_queue, chain, tmp) {  in ata_fail_requests()
587 	    if (ch->transition == request)  in ata_fail_requests()
588 		ch->transition = TAILQ_NEXT(request, chain);  in ata_fail_requests()
589 	    TAILQ_REMOVE(&ch->ata_queue, request, chain);  in ata_fail_requests()
595     lockmgr(&ch->state_mtx, LK_RELEASE);  in ata_fail_requests()
596     lockmgr(&ch->queue_mtx, LK_RELEASE);  in ata_fail_requests()
634 ata_sort_queue(struct ata_channel *ch, struct ata_request *request)  in ata_sort_queue()  argument
637 	if (ch->transition) {  in ata_sort_queue()
641 	    TAILQ_INSERT_BEFORE(ch->transition, request, chain);  in ata_sort_queue()
642 	    if (++ch->reorder >= bioq_reorder_minor_interval) {  in ata_sort_queue()
643 		ch->reorder = 0;  in ata_sort_queue()
644 		atawritereorder(ch);  in ata_sort_queue()
651 	    TAILQ_INSERT_TAIL(&ch->ata_queue, request, chain);  in ata_sort_queue()
659 	TAILQ_INSERT_TAIL(&ch->ata_queue, request, chain);  in ata_sort_queue()
660 	if (ch->transition == NULL)  in ata_sort_queue()
661 		ch->transition = request;  in ata_sort_queue()
664 	ch->transition = NULL;  in ata_sort_queue()
665 	ch->reorder = 0;  in ata_sort_queue()
675 atawritereorder(struct ata_channel *ch)  in atawritereorder()  argument
682     next_offset = ata_get_lba(ch->transition);  in atawritereorder()
683     while ((req = ch->transition) != NULL &&  in atawritereorder()
687 	ch->transition = TAILQ_NEXT(req, chain);  in atawritereorder()